[开发测试]python+selenium抓取网页新闻列表元素，保存到excel文档中

python+selenium:抓取网页新闻列表元素，并记录标题和链接地址，保存到excel文档中

#!/usr/local/bin/python3
# !python3.9
# -*- coding: UTF-8 -*-
'''
@File:xwzx1.py
@Time:2021-09-09 18:32
@Author:user
抓取网页新闻列表元素，并记录标题和链接地址，保存到excel文档中
'''
from selenium import webdriver
from os.path import join
import os
import sys
from time import strftime
from time import sleep
import xlrd
import xlwt

excel_dir=os.path.dirname(os.path.abspath(__file__))
now_time=strftime("%Y-%m-%d-%H-%M")


class text_xwzx():
    def __init__(self):

        self.url="http://www.sinopecgroup.com/group/xwzx/"
        self.filename=now_time+".xls"
        self.path=join(excel_dir,'data/')

    def getUTitle(self,driver,excel,row=0,col=0,rowlen=0):

        # 先找到ul->li，再找li->span.title->a,根据标签<a>属性，得到title和url
        ul = driver.find_element_by_css_selector("ul.w_newslistpage_list")
        atitles = ul.find_elements_by_css_selector("li>span.title>a")

        for title in atitles:
            atitle = title.text
            aurl = title.get_attribute("href")
            #当有分页时，先取得第一页的列表数量，作为第二页的开始，依次小于当前页列表的总数量
            #思路：1)当为首页时，row从0到列表总数；若为第二/三/四页时，把首页列表总数量，做为开始数，遍历当前页列表的总数；
            if(rowlen >= len(atitles)):
                row += 1
            else:
                row += 1

            excel.write(row, col, atitle)
            excel.write(row, col+1, aurl)
        return row

    def nextPage(self,driver):
        nextpage = driver.find_element_by_link_text("下一页")
        nextpage.click()
        sleep(10)

    def getData(self,modular):
        driver = webdriver.Chrome()
        driver.get(self.url+modular)
        sleep(10)

        #使用xlwt模块操作Excel的方法
        book = xlwt.Workbook(encoding = 'utf-8')
        # 先创建一个worksheet
        sh = book.add_sheet(modular)
        xwzx=text_xwzx()
        # 写入excel
        # 参数对应 row-行, col-列, 值
        rowlen=xwzx.getUTitle(driver,sh)

        print("rowlen=" , rowlen)
        xwzx.nextPage(driver)
        rowlen3=xwzx.getUTitle(driver,sh,rowlen,0,rowlen)
        print("rowlen3=" , rowlen3)
        xwzx.nextPage(driver)
        rowlen4=xwzx.getUTitle(driver, sh, rowlen3, 0, rowlen3)

        print("rowlen4=" , rowlen4)


        path_xls = self.path+self.filename
        print(path_xls)
        #保存文件，文件给文件一个名字
        book.save(path_xls)

if __name__ == '__main__':
    data=text_xwzx()
    data.getData('dqgz')

?