[Python知识库]python通过selenium绕过反扒系统跨网页批量获取股票财务信息

完整代码：

import xlwings as xw
import requests
from bs4 import BeautifulSoup
from datetime import datetime
import json
import xlwt
import xlwings as xw
from selenium import webdriver
import time
import pandas as pd
import csv
import re
from selenium.webdriver import Chrome, ChromeOptions


# item_list=[]
df = pd.DataFrame()
def data_a(html):  # 获取基础信息1
    # with open('rrBand.html', 'r', encoding='utf-8') as f:
    # html = BeautifulSoup(f, 'lxml')
    # html.list = html.find_all('div', attrs={'class': 'container-sm float-left stock__main'})
    # print(html.list)
    df = pd.DataFrame()
    # print(html)
    for i, item in enumerate(html):
        # print(item)
        # print(html.list_a)
        try:
            bandNanme = item.find_all('div', attrs={'class', 'stock-name'})[0].text.strip('()')[0:].strip(')')
            df['序号'] = '',
            df['股票'] = item.find_all('div', attrs={'class', 'stock-name'})[0].text.strip('()')[0:bandNanme.find('(')].strip(
                        ')'),
            df['代码'] = item.find_all('div', attrs={'class', 'stock-name'})[0].text.strip('')[5:].strip(')').replace(':',''),
                    # print(df[['股票','代码']])
            df['股价'] = item.find_all('div', attrs={'class', 'stock-current'})[0].text.strip('￥'),
            print(df['股价'])
            html.list_a = item.find_all('table', attrs={'class', 'quote-info'})

            for i,item_a in enumerate(html.list_a):
                # print(item_a.find_all('span'))
                # for i in range(21):
                df['总市值(亿)'] = item_a.find_all('span')[19].text.strip('亿'),
                df['总股本(亿)'] = '',
                # df['营业额'] = '',
                df['EPS每股收益'] = item_a.find_all('span')[16].text.strip(''),
                # df['分红'] = '',
                df['分红率'] = '',
                df['营市比'] = '',
                # print(item_a.find_all('span')[i].text,i,sep=',')#_a.find_all('span')[18].text.strip('亿'),i,sep=','),
                df['PE市盈率'] = item_a.find_all('span')[10].text.strip(''),
                df['PB市净率'] = str((float(df['股价']) / float(item_a.find_all('span')[20].text.strip(''))))[0:4],
                # df['负债率'] = '',
                # print(df['PB市净率'])
                print(str(i),"第一模块写入正常")
        except:
            print(str(i), "第一模块写入异常")
            #     # continue
    return df
# df.to_csv('fundWedb.csv', index=None, encoding='utf-8-sig',sep=',')#mode='a', header=None,index=None,
# print(df[['股价','总市值','EPS每股收益']])

def data_b(html):  # 获取基础信息2
# url='https://xueqiu.com/snowman/S/SH600282/detail#/ZYCWZB'
# print('12')
#     with open('Band.html', 'r', encoding='utf-8') as f:
#         html = BeautifulSoup(f, 'lxml')
#         html.list_b = html.find_all('tbody')
        df = pd.DataFrame()
        bandIncome=[]
        for i, item in enumerate(html):
            # print(item)
            html.list_b_a=item.find_all('tr')
            for i,item in enumerate(html.list_b_a):
                html.list_b_a_a = item.find_all('td')
                # print(item.find_all('td'))
                for i, item in enumerate(html.list_b_a_a):
                    try:
                        html.list = item.find_all('p')[0].contents[0]
                        bandIncome.append(html.list)
                        # print(bandIncome,i,sep=',')
                        # for i, item in enumerate(html.list):
                        # print(item)
                        html.list_b = item.find_all('table', attrs={'class', 'quote-info'})
                        # print(html.list_a)
                        # try:
                        # bandNanme = item.find_all('div', attrs={'class', 'stock-name'})[0].text.strip('()')[0:].strip(')')
                        # df['分红'] = '',
                        # df['负债率'] = '',
                        df['17年利润(亿)'] = '',
                        df['20年利润(亿)'] = '',
                        print(str(i), "第二模块写入正常")
                    except:
                        # continue
                        print(str(i), "第二模块写入异常")
        df['营业额'] =bandIncome[0].strip('亿'),
        df['负债率'] =bandIncome[85].strip(''),
        df['经营现金流'] =bandIncome[50].strip(''),
        df['17年利润(亿)'] =bandIncome[13].strip('亿'),
        df['20年利润(亿)'] = bandIncome[10].strip('亿'),
        df['未分配利润'] = bandIncome[45].strip(''),
        df['公积金'] = bandIncome[40].strip(''),
        df['毛利率'] = bandIncome[75].strip(''),
        df['净利率'] = bandIncome[80].strip(''),
        df['ROA总报酬率'] = bandIncome[65].strip(''),
        df['ROE净收益率'] = bandIncome[60].strip(''),
        df['账款周期'] = bandIncome[120].strip(''),
        df['存货周转'] = bandIncome[150].strip(''),
        df['总资产周转率'] = bandIncome[145].strip(''),
        print(df)
        # for i in range(len(bandIncome)):
        #     print(bandIncome[i],i,sep=',')
        return df
def data_c(html):
    # with open('Band.html','r',encoding='utf-8') as f:
    #     # url='https://xueqiu.com/snowman/S/SH601991/detail#/FHPS'
    #     html=BeautifulSoup(f,'lxml')
    #     html.list=html.find_all('tbody')
    df = pd.DataFrame()
    for i,item in enumerate(html):
        try:
            cut_a=item.find_all('td')[1].text.strip().find('派')
            cut_b = item.find_all('td')[1].text.strip().find('元')
            # print(cut_a,cut_b)
            # print(item)
            print(item.find_all('td')[1].text.strip()[cut_a+1:cut_b])
            df['分红']=item.find_all('td')[1].text.strip()[cut_a+1:cut_b],
            print(df['分红'])
            print(str(i),"第三模块写入正常")
        except:
            print(str(i), "第三模块写入异常")
    return df
def data_d(html):
# with open('Band.html','r',encoding='utf-8') as f:
#     # url='https://xueqiu.com/snowman/S/SH600282/detail#/ZCFZB'
#     html=BeautifulSoup(f,'lxml')
#     html.list=html.find_all('tbody')
    df = pd.DataFrame()
    # print(html)
    for i, item in enumerate(html):
        # try:
        cut_a = item.find_all('td')[7].text.strip('').find('亿')
        # print(cut_a)
        print(item.find_all('td')[7].text.strip('')[0: cut_a-1])
        print(item.find_all('td')[61].text.strip('')[0: cut_a-1])
        df['货币资金']=item.find_all('td')[7].text.strip('')[0: cut_a-1],
        df['存货']=item.find_all('td')[61].text.strip('')[0: cut_a-1],
        # for i in range(300):
        #     print(item.find_all('p')[i],i,sep=',')
        print(str(i), "第四模块写入正常")
        # except:
        #     print(str(i), "第四模块写入正常")
    return df


# 写入csv中
if __name__ == "__main__":
    # 创建一个workbook
    app = xw.App(visible=False, add_book=False)
    wb = app.books.open('fundWebd.xlsx')
    # 创建一个worksheet
    sh = wb.sheets['worksheet']
    rng = [i for i in sh.range("c:c").value if i != None]#单元格内容
    j = sh.range('a1').expand('table').rows.count#序号
    app.display_alerts = False
    app.screen_updating = False
    # rng = sh.range('a1').expand('table')

    # nrows = rng.rows.count
    # a = sh.range(f'a1:a{nrows}').value
    # a = [ i for i in sht.range(a:a).value if i != None]
    # 打开网页
    opt = ChromeOptions()            # 创建Chrome参数对象
    opt.headless =False #True#              # 把Chrome设置成可视化无界面模式，
    driver = Chrome(options=opt)
    # driver = webdriver.Chrome()
    for i in range(len(rng)-1):
        print(str(i),rng[i],'第'+str(i+1)+'只股票开始写入')#rng[i+1]

        bandcode=rng[i+1]#'SH601600'
        xueqiu_url='https://xueqiu.com/S/'+bandcode#雪球网基础数据'https://c.runoob.com/'#很好的ide工具
        xueqiu_url_a='https://xueqiu.com/snowman/S/'+bandcode+'/detail#/ZYCWZB'#主要指标
        xueqiu_url_c= 'https://xueqiu.com/snowman/S/'+bandcode+'/detail#/FHPS'#分红
        xueqiu_url_d = 'https://xueqiu.com/snowman/S/'+bandcode+'/detail#/ZCFZB'  # 存货

        # DFCF_url='http://emweb.eastmoney.com/PC_HSF10/OperationsRequired/Index?type=web&code=SH601600'

        #基础数据1加载
        driver.get(xueqiu_url)#加载网址
        source =driver.page_source#获取网页内容
        html=BeautifulSoup(source,'html.parser')#获取网页内容
        time.sleep(2)#休眠1秒
        html.list = html.find_all('div', attrs={'class': 'container-sm float-left stock__main'})
        df_a = data_a(html.list)  # 执行语句块
        time.sleep(2)  # 休眠1秒

        #基础数据2加载
        driver.get(xueqiu_url_a)  # 加载网址
        # driver.find_elements_by_class_name('btn active').click()
        # driver.find_element_by_xpath(".//*[@id='header']/div[1]/div/form/input[2]").click()
        time.sleep(2)
        driver.find_element_by_xpath(".//div[contains(@class,'stock-info-btn-list')]/span[2]").click()
        # print(driver.find_element_by_xpath(".//div[contains(@class,'stock-info-btn-list')]/span[2]").text)  # /span[contains(@class,'btn')]
        time.sleep(3)  # 休眠4秒
        source = driver.page_source  # 获取网页内容
        html = BeautifulSoup(source, 'html.parser')  # 获取网页内容
        # print(html)
        time.sleep(2)  # 休眠1秒
        html.list_b = html.find_all('tbody')
        df_b = data_b(html.list_b)  # 执行语句块
        time.sleep(1)

        #基础数据三加载
        driver.get(xueqiu_url_c)  # 加载网址
        time.sleep(2)
        source = driver.page_source  # 获取网页内容
        html = BeautifulSoup(source, 'html.parser')  # 获取网页内容
        # print(html)
        time.sleep(2)  # 休眠1秒
        html.list_c = html.find_all('tbody')
        df_c= data_c(html.list_c)  # 执行语句块
        # print(html)
        time.sleep(1)

        # 基础数据四加载
        driver.get(xueqiu_url)  # 加载网址
        driver.get(xueqiu_url_d)  # 加载网址
        time.sleep(4)
        driver.find_element_by_xpath(".//div[contains(@class,'stock-info-btn-list')]/span[2]").click()
        time.sleep(3)
        source = driver.page_source  # 获取网页内容
        html = BeautifulSoup(source, 'html.parser')  # 获取网页内容
        # print(html)
        time.sleep(2)  # 休眠1秒
        html.list_d=html.find_all('tbody')
        df_d= data_d(html.list_d)  # 执行语句块
        df_d.to_json('fundWebdTest.json', orient='records', force_ascii=False)  # ,orient="values")
        time.sleep(1)

        # with open('rrBand.html', 'r', encoding='utf-8') as f:
        #     html = BeautifulSoup(f, 'lxml')
        #     html.list = html.find_all('div', attrs={'class': 'container-sm float-left stock__main'})
        #     df_a = data_a(html.list)  # 执行语句块
        # with open('Band.html', 'r', encoding='utf-8') as f:
        #     html = BeautifulSoup(f, 'lxml')
        #     html.list_b = html.find_all('tbody')
        #     df_b = data_b(html.list_b)  # 执行语句块
        # with open('Band.html', 'r', encoding='utf-8') as f:
        #      html = BeautifulSoup(f, 'lxml')
        #      html.list_b = html.find_all('tbody')
        # with open('Band.html','r',encoding='utf-8') as f:
        #     # url='https://xueqiu.com/snowman/S/SH600282/detail#/ZCFZB'
        #     html=BeautifulSoup(f,'lxml')
        #     html.list=html.find_all('tbody')

        #以下为写入模板
        # df=pd.concat([df_a,df_b],axis=1)#列合并，axis=0表示按行合并df = df_a.append(df_b)
        # print(df_c,'测试')
        df1=pd.concat([df_a,df_b],axis=1)#按列合并
        print(df1)
        df2=pd.concat([df1,df_c],axis=1)#按列合并
        print(df2)
        df3 = pd.concat([df2, df_d], axis=1)  # 按列合并
        print(df3)
        df=pd.concat([df3,df],axis=0)#按行合并,这里很重要
        print(df)
        # item_list.append(df)
        # print(item_list)

        df.to_json('fundWebd.json',orient ='records', force_ascii=False)#,orient="values")
        # with open('fundWebd.json','r',encoding='utf-8') as f:
        #     data = json.load(f)
        # item_list.append(data)
        # with open('.fund.json', 'w', encoding='utf-8')as f:
        #     json.dump(item_list,f, indent=1, ensure_ascii=False)
        with open('fundWebd.json', 'r', encoding='utf-8') as f:
            data = json.load(f)
            # print(data[0]['股票'])
        bandN = ['序号', '股票', '代码', '股价', '总市值(亿)', '总股本(亿)', '营业额', 'EPS每股收益', '分红', '分红率', '营市比', 'PE市盈率',\
                      'PB市净率', '负债率','经营现金流','货币资金','存货','利息费/收','17年利润(亿)','20年利润(亿)','利润复增率','营业额复合增长率',\
                      '季度增长率','现金收入比','PEG','未分配利润','公积金','毛利率','净利率','ROA总报酬率','ROE净收益率','账款周期','存货周转','总资产周转率']
        for i in range(len(data)):#写入数据
            print(len(data))
            sh.cells[i+1,0].value=data[i][bandN[0]]
            sh.cells[i+1, 1].value=data[i][bandN[1]]
            sh.cells[i+1, 2].value=data[i][bandN[2]]
            sh.cells[i+1, 3].value = data[i][bandN[3]]
            sh.cells[i+1, 4].value = data[i][bandN[4]]
            sh.cells[i+1, 5].value = data[i][bandN[5]]
            sh.cells[i+1, 6].value = data[i][bandN[6]]#营业额
            sh.cells[i+1, 7].value = data[i][bandN[7]]
            sh.cells[i+1, 8].value = data[i][bandN[8]]#分红
            # sh.cells[i+1, 9].value = data[i][bandN[9]]#分红率
            # sh.cells[i+1, 10].value = data[i][bandN[10]]#营市比
            sh.cells[i+1, 11].value = data[i][bandN[11]]
            sh.cells[i+1, 12].value = data[i][bandN[12]]
            sh.cells[i+1, 13].value = data[i][bandN[13]]#负债率
            sh.cells[i + 1, 14].value = data[i][bandN[14]]  # 经营现金流
            sh.cells[i + 1, 15].value = data[i][bandN[15]]  # 货币资金
            sh.cells[i + 1, 16].value = data[i][bandN[16]]  # 存货
            # sh.cells[i + 1, 17].value = data[i][bandN[17]]  # 利息费/收
            sh.cells[i + 1, 18].value = data[i][bandN[18]]  # 17年利润(亿)
            sh.cells[i + 1, 19].value = data[i][bandN[19]]  # 20年利润(亿)
            # sh.cells[i + 1, 20].value = data[i][bandN[20]]  # 利润复增率
            # sh.cells[i + 1, 21].value = data[i][bandN[21]]  # 营业额复合增长率
            # sh.cells[i + 1, 22].value = data[i][bandN[22]]  # 季度增长率
            # sh.cells[i + 1, 23].value = data[i][bandN[23]]  # 现金收入比
            # sh.cells[i + 1, 24].value = data[i][bandN[24]]  # PEG
            sh.cells[i + 1, 25].value = data[i][bandN[25]]  # 未分配利润
            sh.cells[i + 1, 26].value = data[i][bandN[26]]  # 公积金
            sh.cells[i + 1, 27].value = data[i][bandN[27]]  # 毛利率
            sh.cells[i + 1, 28].value = data[i][bandN[28]]  # 净利率
            sh.cells[i + 1, 29].value = data[i][bandN[29]]  # ROA总报酬率
            sh.cells[i + 1, 30].value = data[i][bandN[30]]  # ROE净收益率
            sh.cells[i + 1, 31].value = data[i][bandN[31]]  # 账款周期
            sh.cells[i + 1, 32].value = data[i][bandN[32]]  # 存货周转
            sh.cells[i + 1, 33].value = data[i][bandN[33]]  # 总资产周转率
            # sh.cells[i + 1, 32].value = data[i][bandN[32]]  # 存货周转
            # sh.cells[i + 1, 32].value = data[i][bandN[32]]  # 存货周转
            # sh.cells[i + 1, 32].value = data[i][bandN[32]]  # 存货周转
                    # print(i)
    wb.save('fundWebd.xlsx')
    app.quit()
    # 获得当前窗口句柄
    sreach_windows = driver.current_window_handle
    driver.quit()
    # 获得当前所有打开的窗口的句柄
    all_handles = driver.window_handles
    for handle in all_handles:
        driver.switch_to.window(handle)
        driver.close()
        time.sleep(2)
    # driver.close()
    # driver.quit()