[Python知识库]Python爬虫，将爬取到的信息转换为DataFrame形式，并以csv格式存储（以药监总局数据为例）

#动态加载数据,并将数据转换为DataFrame格式，并将其转换为csv格式
import json
import requests
import pandas as pd


def getId():#获取动态数据的id信息
    idurl = r'http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsList'
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'}
    idlist = []
    for page in range(1,20):#获取数据的页数
        page =str(page)#将页数转换为字符串格式
        data = {#参数
            'on': 'true',
            'page': page,
            'pageSize': '15',
            'productName':'',
            'conditionType': '1',
            'applyname':'',
            'applysn':''
        }
        responce = requests.post(url=idurl,headers=headers,data=data).json()
        print(responce)
        print("---------------------------------------")
        for dic in responce['list']:
            idlist.append(dic['ID'])
    return idlist

def getInformation(idlist):#获取动态数据的详细信息
    url = 'http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsById'
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'}
    inforlist = []
    for id in idlist:
        data = {
            'id':id
        }
        responce = requests.post(url=url,data=data,headers=headers).json()
        inforlist.append(responce)
    return inforlist


def getDataFrame(idlist):#持久化存储
    datall = pd.DataFrame()
    for item in idlist:
        df = pd.DataFrame.from_dict(item,orient='index').T
        datall = datall.append(df)
    datall.to_csv('data.csv')


if __name__ == '__main__':
    id = getId()
    #print(id)
    detailinformation = getInformation(id)
    #print(detailinformation)
    getDataFrame(detailinformation)
    print("over!!!!")

爬取数据后的具体结果如上图所示。