[网络协议]python 爬虫之 requests库

1 代理访问

proxies = {
   # 'type':'type://ip:port'
   # 'type': 'type://username:password@ip:port'
  "http": "http://10.10.1.10:3128",
  "https": "https://10.10.1.10:1080",
}

2 session自动保存cookies

import requests
# 创建一个session对象 
session = requests.Session()

# 用session对象发出请求，保存一些信息

3 ssl验证

import requests
# 禁用安全请求警告
requests.packages.urllib3.disable_warnings()

resp = requests.get(url, verify=False)

?案例

import requests

session = requests.Session()
img_url ='https://passport.ganji.com/ajax.php?dir=captcha&module=login_captcha'
login_url ='https://passport.ganji.com/login.php'
header = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36'
}
img_resp = session.get(img_url,headers=header)
# 下载验证码图片
with open('code_img.jpg','wb') as f:
    f.write(img_resp.content)
code = input('请输入验证码:')
data = {
'username': '12334',
'password': '123',
'setcookie': '14',
'checkCode': code,
'next': '/',
'source': 'passport'
}
login_resp = session.post(login_url,headers=header,data=data)

index_url = 'http://sh.ganji.com/'
index_resp = session.get(index_url,headers=header)
# print(index_resp.text)

url ='http://www.ganji.com/vip/'
resp = session.get(url,headers=header)
print(resp.content.decode())