[Python知识库]python 爬取2021年《财富》世界500强排行榜2层链接

python ?爬取2021年《财富》世界500强排行榜2层链接

'''
2021年500强财富爬取
QQ：28928247
# '''
# -*- coding: UTF-8 -*-
import requests,random
import os
from lxml import etree
import xlsxwriter
 
class Httprequest(object):
    ua_list = [
        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1',
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36Chrome 17.0',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
        'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0Firefox 4.0.1',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
        'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
        'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
        'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',
    ]
    @property  #把方法变成属性的装饰器
    def random_headers(self):
        return {
            'User-Agent': random.choice(self.ua_list)
        }
 
 
class Get_data(Httprequest):
    def __init__(self):
        self.murl="http://www.fortunechina.com/fortune500/c/2021-08/02/content_394571.htm"
 
    def get_data(self,url):
        html=requests.get(url,headers=self.random_headers,timeout=5).content.decode('utf-8')
        req = etree.HTML(html)
        data=[]
        zc=req.xpath('//div[@class="top"]/div[@class="table"]/table/tr[4]/td[2]/text()')
        gdqy=req.xpath('//div[@class="top"]/div[@class="table"]/table/tr[5]/td[2]/text()')
        jlr=req.xpath('//div[@class="top"]/div[@class="table"]/table/tr[7]/td[3]/text()')
        zcsyl=req.xpath('//div[@class="top"]/div[@class="table"]/table/tr[8]/td[3]/text()')
        data.extend(zc)
        data.extend(gdqy)
        data.extend(jlr)
        data.extend(zcsyl)
        #print(data)
        return data
 
 
 
    def get_mdata(self):
        html=requests.get(self.murl,headers=self.random_headers,timeout=5).content.decode('utf-8')
        #print(html)
        req = etree.HTML(html)
        rankings=req.xpath('//table[@class="wt-table"]/tbody/tr/td[1]/text()')
        companys=req.xpath('//table[@class="wt-table"]/tbody/tr/td[2]/a/text()')
        incomes=req.xpath('//table[@class="wt-table"]/tbody/tr/td[3]/text()')
        profits=req.xpath('//table[@class="wt-table"]/tbody/tr/td[4]/text()')
        hrefs=req.xpath('//table[@class="wt-table"]/tbody/tr/td[2]/a/@href')
        data_list=[]
        for ranking,company,income,profit,href in zip(
                rankings,companys, incomes, profits,hrefs
        ):
            data = [
                ranking,company,income,profit
            ]
            href=href.replace('../','')
            href="http://www.fortunechina.com/{}".format(href)
            data.extend(self.get_data(href))
 
            print(data)
            data_list.append(data)
        print('\n')
 
        self.write_to_mxlsx(data_list)
 
 
    def write_to_mxlsx(self, data_list):
        mypath = os.path.dirname((os.path.abspath(__file__)))
        workbook = xlsxwriter.Workbook(mypath+'\{}.xlsx'.format("2021年《财富》美国500强排行榜"))  
        worksheet = workbook.add_worksheet("2020年《财富》美国500强排行榜")
        title = ['排名', '公司名称（中文）', '营业收入（百万美元）', '利润（百万美元）','资产','股东权益','净利率','资产收益率']  
        worksheet.write_row('A1', title)
        for index, data in enumerate(data_list):
            num0 = str(index + 2)
            row = 'A' + num0
            worksheet.write_row(row, data)
        workbook.close()
 
 
if __name__=="__main__":
    spider=Get_data()
    spider.get_mdata()