[Python知识库]爬取一部小说并写入txt

学习记录：这个爬的太慢了~~~慢，慢，慢........

网址：斗罗大陆5重生唐三最新章节_斗罗大陆5重生唐三全文免费阅读-笔趣阁

找到正文：

?代码：

"""
2022年

CSDN:抄代码抄错的小牛马
"""

import requests
from lxml import etree


def get_url():
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36'
    }
    url = 'https://www.52bqg.net/48_48686/'
    response = requests.get(url=url, headers=headers)
    response.encoding = 'gbk'
    content = response.text
    return content


def down_load(content):
    tree = etree.HTML(content)
    titles = tree.xpath('//div[@id="list"]/dl/dd/a ')[12:]  # 正文开始
    for i in titles:
        # 章节标题
        title = i.xpath('.//text()')[0]  # 索引去[],str
        # 章节链接
        href = i.xpath('./@href')[0]
        url = 'https://www.52bqg.net/48_48686/'
        href = url + href
        response_text = requests.get(href)
        response_text.encoding = 'gbk'
        content_text = response_text.text
        tree = etree.HTML(content_text)
        text = tree.xpath('//div[@id="content"]/text()')
        text = "\n".join(text).replace('笔趣阁 www.52bqg.net，最快更新斗罗大陆5重生唐三 ！', '').replace(
            '新书上传，希望大家可以先收藏、推荐，正式连载将于5月20号。', '').replace('&nbsp', '')

        with open("斗罗大陆.txt", 'a', encoding='utf-8') as f:
            f.write(title + "\n")
            f.write(text + "\n")
            print('%s下载成功' % title)


if __name__ == '__main__':
    # 获取网页的源码（响应数据） response
    content = get_url()
    # 下载并保存数据到本地 + 解析网页
    down_load(content)

运行：

?查看：

.?