[开发测试]51job_selenium爬虫

程序

仅仅提供学习使用

import requests
from lxml import etree
from bs4 import BeautifulSoup
import json
from selenium.webdriver.chrome.options import Options   #实现五可视化
from selenium.webdriver import ChromeOptions     #实现规避被检测到的风险
from time import sleep
from selenium.webdriver import Chrome
import pandas as pd
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable--gpu')

#实现规避检测
option = ChromeOptions()
option.add_experimental_option('excludeSwitches',['enable-automation'])
chrome = Chrome(executable_path=r'E:\浏览器下载\chromedriver',options=chrome_options)

url = 'https://search.51job.com/list/000000,000000,0000,32,9,99,+,2,1.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare='
chrome.get(url)
input = chrome.find_elements_by_id('keywordInput')[0]
input.send_keys('java 成都')
search = input.find_element_by_xpath('//*[@id="search_btn"]')
search.click()

text = chrome.page_source
tree = etree.HTML(text)
xinxi = tree.xpath('//script[@type="text/javascript"]')[3]
dict_ = xinxi.xpath('./text()')
num = json.loads(dict_[0][28:])['total_page']
data = []

for _ in range(int(num)-1):
    text = chrome.page_source
    tree = etree.HTML(text)
    xinxi = tree.xpath('//script[@type="text/javascript"]')[3]
    dict_ = xinxi.xpath('./text()')
    # print(dict_[0][28:])
    dict_ = json.loads(dict_[0][28:])['engine_jds']
    for i in dict_:
        data.append([i['job_name'], i['workarea_text'], i['updatedate'], list(i['attribute_text'])[1],list(i['attribute_text'])[2], i['providesalary_text'],list(i['attribute_text'])[-1]])
        # print(i['job_name'], i['workarea_text'], i['updatedate'], list(i['attribute_text'])[1],list(i['attribute_text'])[2], i['providesalary_text'],list(i['attribute_text'])[-1])
    try:
        sleep(1)
        next_ = chrome.find_elements_by_xpath('//li[@class="next"]')[0]
        next_.click()
        chrome.refresh()
    except:
        break

data = pd.DataFrame(data,columns=['招聘岗位','招聘城市','招聘年月','工作经验','学历要求','招聘薪资','招聘数量'])
writer = pd.ExcelWriter(r'C:\Users\24132\Desktop\data_spyder.xlsx')		# 写入Excel文件
data.to_excel(writer, 'page_1', float_format='%.10f')		# ‘page_1’是写入excel的sheet名
writer.save()
writer.close()
# for i in range(1,101):
#     text = chrome.page_source
#
#     tree = etree.HTML(text)
#     xinxi = tree.xpath('//script[@type="text/javascript"]')[3]
#
#     dict_ = xinxi.xpath('./text()')
#     print(dict_[0][28:])
#     dict_ = json.loads(dict_[0][28:])['engine_jds']
#     # job_name workarea_text updatedate attribute_text  providesalary_text
#     for i in dict_:
#         print(i['job_name'], i['workarea_text'], i['updatedate'], list(i['attribute_text']), i['providesalary_text'])

# for i in range(1,101):
#     text = requests.get(url=url.format(i), headers=headers)
#     if text.status_code != 200:
#         break
#     tree = etree.HTML(text.text)
#     xinxi = tree.xpath('//script[@type="text/javascript"]')[2]
#     dict_ = xinxi.xpath('./text()')
#     dict_ = json.loads(dict_[0][29:])['engine_jds']
#     # job_name workarea_text updatedate attribute_text  providesalary_text
#     for i in dict_:
#         print(i['job_name'], i['workarea_text'], i['updatedate'], list(i['attribute_text']), i['providesalary_text'])

总结