解决selenium下被反扒的情况
# -*- encoding:utf-8 -*-
import time
import requests
from selenium.webdriver.common.by import By
def get_data(cookies):
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36",
# "Host": "www.iwencai.com",
# 'Upgrade-Insecure-Requests': '1',
'Accept-Language': 'zh-CN,zh;q=0.9',
# 'referer': 'https://www.iwencai.com/',
"Cookie": f"cid={cookies['cid']}; "
f"ComputerID={cookies['ComputerID']}; "
"ver_mark=c; "
"other_uid=Ths_iwencai_Xuangu_9fnth9iibkd8zrtya1yj4ll48wx683s8; "
f"ta_random_userid={cookies['ta_random_userid']}; "
"WafStatus=0; "
f"PHPSESSID={cookies['PHPSESSID']}; "
f"v={cookies['v']}",
"hexin-v": f"{cookies['v']}"
}
print("开始爬取数据".center(30,"*"))
print("请求头为：",headers)
url = 'http://www.iwencai.com/stockpick/load-data?typed=0&preParams=&ts=1&f=1&qs=result_original&selfsectsn=&querytype=stock&searchfilter=&tid=stockpick&' \
'w=股价下跌，超大单净流入大于1000万元，大单净流入大于1000万元，市值小于400亿，非ST,非创业板，非科创板，股价大于10元'
res = requests.get(url, headers=headers)
print("res为：", res)
if res.status_code != 200:
print(res.text)
return
result_data = res.json()
print('爬取数据的结果为：',result_data)
return result_data['data']['result']['result']
def handle_data(result_data):
"""
处理数据
:return:
"""
result_list_data = []
for one_data in result_data:
result_dict_data = {}
# 股票代码
result_dict_data['stock_code'] = one_data[0]
# 股票名称
result_dict_data['stock_name'] = one_data[1]
# 涨跌幅
result_dict_data['stock_upanddown'] = one_data[2]
# 超大单流入
result_dict_data['stock_extea_large_flow'] = one_data[3]
# 大单流入
result_dict_data['stock_large_flow'] = one_data[4]
# 股票市值
result_dict_data['stock_market_value'] = one_data[5]
# 股票价格
result_dict_data['stock_price'] = one_data[7]
result_list_data.append(result_dict_data)
print("处理后的结果数据为：",result_list_data)
return result_list_data
def handle_business(cookies):
"""
处理业务
:return:
"""
# 爬取数据
result_data = get_data(cookies)
if not result_data:
return
# 处理数据为字典的列表
result_list_data = handle_data(result_data)
# print(result_list_data)
# 业务统计，并发送邮件
for one_data in result_list_data:
# 涨跌幅小于5%的，市值小于200亿，特大单净流入大于2000万的
if abs(float(one_data['stock_upanddown'])) > 2 and float(one_data['stock_market_value']) < 20000000000 and float(one_data['stock_extea_large_flow']) >= 20000000:
print('满足跌幅在5%以下的，市值小于200亿，特大单净流入大于2000万的的股票为：',one_data)
# 发送邮件通知
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import os,subprocess,threading
def open_browser():
subprocess.call(
'C:\Program Files\Google\Chrome\Application\chrome.exe --remote-debugging-port=9222 --user-data-dir="C:selenumAutomationProfile"',
timeout=15)
print("打开浏览器结束了")
th = threading.Thread(target=open_browser)
th.start()
options = webdriver.ChromeOptions()
options.add_experimental_option("debuggerAddress", "127.0.0.1:9222")
# 不加载图片
# options.add_argument('blink-settings=imagesEnabled=false')
# 不显示图形界面
# options.add_argument('--headless')
driver = webdriver.Chrome(executable_path=r'D:\Python\Python38\chromedriver.exe',options=options)
driver.get('https://www.iwencai.com')
#
# WebDriverWait(driver,timeout=10).until(EC.presence_of_element_located((By.ID,"auto")))
# driver.find_element(By.ID,"auto").send_keys('跌幅在5%以下的，市值小于200亿，特大单净流入大于2000万')
# driver.find_element(By.ID,'qs-enter').click()
cookies = driver.get_cookies()
print(cookies)
res_cookies = {}
for one_cookie in cookies:
if one_cookie['name'] == 'v':
res_cookies['v'] = one_cookie['value']
elif one_cookie['name'] == 'ta_random_userid':
res_cookies['ta_random_userid'] = one_cookie['value']
elif one_cookie['name'] == 'cid':
res_cookies['cid'] = one_cookie['value']
elif one_cookie['name'] == 'ComputerID':
res_cookies['ComputerID'] = one_cookie['value']
elif one_cookie['name'] == 'PHPSESSID':
res_cookies['PHPSESSID'] = one_cookie['value']
time.sleep(3)
handle_business(res_cookies)
driver.close()
print("关闭页签")
