|
import requests
from lxml import etree
url = 'https://www.runoob.com/python3/python3-examples.html'
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36'}
res = requests.get(url,headers = headers)
html = etree.HTML(res.content.decode())
hrefs = html.xpath('//div[@id="content"]/ul/li/a/@href')
baseurl = 'https://www.runoob.com/'
hrefall = [baseurl+h if h.startswith('/python3/') else baseurl+'python3/'+h for h in hrefs]
titles = []
cons = []
for href in hrefall:
? ? res = requests.get(href,headers = headers)
? ? html = etree.HTML(res.content.decode())
? ? title = html.xpath('//div[@id="content"]/h1/text()')
? ? con = html.xpath('//div[@id="content"]/p[2]/text()')
? ? titles.append(title[0] if title else '')
? ? cons.append(con[0] if con else '')
cons = [con.strip() for con in cons]
result = list(enumerate(zip(titles,cons),1))
with open('python.csv','w',encoding='utf-8') as fp:
for r in result:
fp.write(str(r[0])+'、'+r[1][0]+'\n'+r[1][1]+'\n\n')
?
|