[Python知识库]2021-10-21Python爬虫-requests基础，实战-第一血

Python爬虫-requests基础，实战-第一血

# -*- codeing = utf-8 -*-
# @Time : 2021/9/26 16:11
# @Author : luxiaoguo
# @File : requests第一血.py
# @Software : PyCharm
import requests


if __name__ == "__main__":
    # step1：指定url
    url = 'https://www.sogou.com/'

    # step2: 发起请求
    # get方法会返回一个响应对象
    response = requests.get(url=url)

    # step3:获取响应数据。text返回是字符串形式数据
    page_text = response.text
    print(page_text)
    # step4：持久化保存数据
    with open('./sogou.html', 'w',encoding='utf-8')as fp:
        fp.write(page_text)
    print("爬取完毕")

requests实战之破解百度翻译.py

# -*- codeing = utf-8 -*-
# @Time : 2021/9/26 19:27
# @Author : luxiaoguo
# @File : requests实战之破解百度翻译.py
# @Software : PyCharm
import json

import requests

if __name__ == "__main__":
    # 1指定url
    post_url = 'https://fanyi.baidu.com/sug'
    # 2进行UA伪装
    header = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36 Edg/94.0.992.31'

    }
    word = input("enter a word:")
    # post请求参数处理（同get一致）
    data = {
        'kw':word
    }
    # 4发送请求
    response = requests.post(url=post_url,data=data,headers=header)

    # 5获取响应数据：json()方法的是obj(确认返回是json)
    dic_obj = response.json()

    # 持久化保存
    fileName = word + '.json'
    fp = open(fileName,'w', encoding='utf-8')
    json.dump(dic_obj,fp=fp,ensure_ascii=False)

    print("over!")

requests词条搜索.py

# -*- codeing = utf-8 -*-
# @Time : 2021/9/26 16:30
# @Author : luxiaoguo
# @File : requests词条搜索.py
# @Software : PyCharm
# UA: User-Agent(请求载体的身份标识）
# UA伪装：门户网站的服务器检测对应请求的载体身份标识

import requests

if __name__ == "__main__":
    # UA伪装：将对应的User—Agent封装到一个字典中
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36 Edg/94.0.992.31'
    }
    url = 'https://www.sogou.com/web'
    # 处理url携带的参数
    kw = input('enter a word:')
    # 字典键值
    param = {
        'query': kw
    }
    # 对指定的url发起的请求对应的url是携带参数的，并且请求过程中处理了参数
    response = requests.get(url=url, params=param)

    page_text = response.text
    fileName = kw + '.html'
    with open(fileName, 'w', encoding='utf-8')as fp:
        fp.write(page_text)
    print(fileName, "保存成功!!!")

requests实战之肯德基餐厅位置爬取.py

# -*- codeing = utf-8 -*-
# @Time : 2021/9/26 20:22
# @Author : luxiaoguo
# @File : requests实战之肯德基餐厅位置爬取.py
# @Software : PyCharm

import requests

if __name__ == "__main__":
    url = 'http://www.kfc.com.cn/kfccda/storelist/index.aspx'
    header = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36 Edg/94.0.992.31'

    }
    e = int(input('enter a pageSize:'))
    for i in range(e):
        pageIndex = ""

        par = {
            'cname':'',
            ' pid':'',
            'keyword': '北京',
            'pageIndex': i,
            'pageSize':  e,
        }

        response = requests.post(url=url, data=par, headers=header)

        kedeji_text = response.text
        print(kedeji_text)
        with open('./kedeji.text','w', encoding='utf-8')as fp:
            fp.write(kedeji_text)
        print("读取完毕！！")

requests实战之爬取药监总局数据.py

# -*- codeing = utf-8 -*-
# @Time : 2021/9/27 15:17
# @Author : luxiaoguo
# @File : requests实战之爬取药监总局数据.py
# @Software : PyCharm
import requests
import json
if __name__ == '__main__':
    # 批量获取不同产业的url
    url = 'http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsList'

    header = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36 Edg/94.0.992.31'

    }
    # 存储id
    id_list = []
    all_data_list = []
    for i in range(1,6):
        page = str(i)
        data = {
            'on': 'true',
            'page': page,
            'pageSize': '15',
            'productName':'',
            'conditionType': '1',
            'applyname':'',
            'applysn':'',
        }


        json_ids = requests.post(url=url, headers=header, data=data).json()
        # print(json_ids)
        for dic in json_ids['list']:
            id_list.append(dic['ID'])
        # print(id_list)

    # 获取不同企业详情数据

    post_url = 'http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsById'
    for id in id_list:
        data = {
            'id':id
        }
        data_json = requests.post(url=post_url,headers=header,data=data).json()
        # print(data_json)
        all_data_list.append(data_json)

    # 持久化存储all_data_list
    fp = open('./allData.json','w',encoding='utf-8')
    json.dump(all_data_list,fp=fp,ensure_ascii=False)

    print('over!!!')
    print("爬取完毕")

step1：指定url

• url = ‘https://www.sogou.com/’

step2: 发起请求

# get方法会返回一个响应对象
response = requests.get(url=url)

step3:获取响应数据。text返回是字符串形式数据

page_text = response.text
print(page_text)

step4：持久化保存数据