[Python知识库]爬虫xpath及实例

xpath的使用方法

tree.xpath(/html/body/div) 逐级定位
tree.xpath(/html//div) 两个//可以跨级定位
tree.xpath(//div[@class=“song”]/p) 可以按照属性值定位
tree.xpath(//div[@class=“song”]/p[3]) 可以定位到具体第几个标签
tree.xpath(//div[@class=“song”]/p[3]/a/text()) 获取直系
tree.xpath(//div[@class=“song”]/p[3]/a//text()) 多一个/可以获取全部
tree.xpath(//div[@class=“song”]/p[3]/a/text())[0]
tree.xpath(//div[@class=“song”]/img/@scr) 可以获取属性值

实例

import requests
from lxml import etree
if __name__  == "__main__":
    url = "https://sanguo.5000yan.com/";
    headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'       
    }
    page_text = requests.get(url = url , headers = headers).text;
    page_text = page_text.encode("ISO-8859-1");
    page_text = page_text.decode("utf-8");
    #print(page_text);
    fp = open("./second.txt",'w',encoding='utf-8');
    tree = etree.HTML(page_text);
    li_list = tree.xpath('//div[@class="sidamingzhu-list-mulu"]/ul/li');
    for li in li_list:
        li_title = li.xpath('./a//text()')[0];
        fp.write(li_title+":\n");
        print(li_title+"写入成功");
        src_list = li.xpath('./a/@href')[0];
        print(src_list)
        page_data = requests.get(url = src_list , headers = headers).text;
        page_data = page_data.encode("ISO-8859-1");
        page_data = page_data.decode("utf-8");
        page_tree = etree.HTML(page_data);
        div_list = page_tree.xpath('//body/div/div[@class="main_life"]//div');
        for div in div_list:
            page_content = div.xpath('./div//text()')[0];
            print(page_content);
            fp.write(page_content);