爬取古诗词
闲来无事，就写了个代码爬取古诗词名句来看看。我们用到的是这个网站 链接: https://www.xungushici.com/。刚刚开始学习Python爬虫，有些不对的地方还请大佬指出。
获取内容
首先我们看一下这个内容的基本架构。分别有注释、故事赏析等等。
分别获取它的位置。
对其进行提取
def get_body(url):
respond = requests.get(url)
html = parsel.Selector(respond.text)
title = html.css('body > div > div > div > div > div > h3::text').get()
origin = html.css('body > div > div > div> div > div> p > a::text').get()
try: #因为部分诗词没有注释和故事，所以在这里进行报错
note = html.css('body > div > div > div> div:nth-child(2) > div > p:nth-child(4)::text').get()
stories = html.css('body > div > div > div > div:nth-child(3) > div > p::text').getall()
story = "\n".join(stories) #对列表进行拼接
except:
pass
appreciates = html.css('body > div > div > div > div:nth-child(3) > div > p::text').getall()
appreciate = '\n'.join(appreciates)
进行文件保存
with open('title + '.txt', mode='a', encoding='utf-8') as f:
f.write(title)
f.write('\n')
f.write('出自于:'+'\n'+origin+'\n')
try: #因为部分诗词没有注释和故事，所以在这里进行报错
f.write('注释:'+'\n'+note+'\n')
f.write('故事:'+'\n'+story+'\n')
except:
pass
f.write('赏析:'+'\n'+appreciate+'\n')
f.close()
实现所有网页提取和翻页
得到每一个诗词的位置
进行编写
def get_url(url):
respond = requests.get(url)
html = parsel.Selector(respond.text)
url_links = html.css('body > div > div > div > div > div > ul > li > a::attr(href)').getall()
for url_link in url_links:
url_link = 'https://www.xungushici.com'+url_link
print(url_link)
get_body(url_link)
再分析每一页
不难得出规律
进行编写
def get_page():
url = 'https://www.xungushici.com/mingjus/p'
for i in range(1,3):
html = 'https://www.xungushici.com/mingjus/p%d'%i
get_url(html)
完整代码
# -*- coding = utf-8 -*-
# @Time : 2021/7/3 21:02
# @File : 尝试爬取寻古诗词网.py
# @Software : PyCharm
import requests
import parsel
def get_page():
url = 'https://www.xungushici.com/mingjus/p'
for i in range(1,3):
html = 'https://www.xungushici.com/mingjus/p%d'%i
get_url(html)
def get_url(url):
respond = requests.get(url)
html = parsel.Selector(respond.text)
url_links = html.css('body > div > div > div > div > div > ul > li > a::attr(href)').getall()
for url_link in url_links:
url_link = 'https://www.xungushici.com'+url_link
print(url_link)
get_body(url_link)
def get_body(url):
respond = requests.get(url)
html = parsel.Selector(respond.text)
title = html.css('body > div > div > div > div > div > h3::text').get()
origin = html.css('body > div > div > div> div > div> p > a::text').get()
try:
note = html.css('body > div > div > div> div:nth-child(2) > div > p:nth-child(4)::text').get()
stories = html.css('body > div > div > div > div:nth-child(3) > div > p::text').getall()
story = "\n".join(stories) #对列表进行拼接
except:
pass
appreciates = html.css('body > div > div > div > div:nth-child(3) > div > p::text').getall()
appreciate = '\n'.join(appreciates)
with open(title + '.txt', mode='a', encoding='utf-8') as f:
f.write(title)
f.write('\n')
f.write('出自于:'+'\n'+origin+'\n')
try:
f.write('注释:'+'\n'+note+'\n')
f.write('故事:'+'\n'+story+'\n')
except:
pass
f.write('赏析:'+'\n'+appreciate+'\n')
f.close()
get_page()