[Python知识库]51job数据爬取

import time

from selenium import webdriver
import csv
from selenium.webdriver import ActionChains

fp = open('./51job_data_02.csv', mode='a', newline='', encoding='utf-8')
csv_writer = csv.writer(fp)
a = []


def login(driver):
    num = 1
    global title, area, experience, education, link,date, salary, message

    driver.delete_all_cookies()
    #url = "https://login.51job.com/login.php?lang=c"  # https://www.qcc.com/weblogin?back=%2F
    #driver.get(url)
    time.sleep(10)

    # 点击密码登入
    # driver.find_element_by_xpath('/html/body/div[3]/div[1]/div[2]/div[1]/span[3]/a').click()
    # time.sleep(1)
    #
    # # 输入账号密码
    # time.sleep(5)
    # driver.find_element_by_id('nameNormal').send_keys(username)  # /html/body/div[1]/div[3]/div/div[2]/div[3]/form/div[1]/input
    # driver.find_element_by_xpath('/html/body/div[3]/div[1]/div[2]/form/div[1]/div/input[1]').send_keys('账号')# 18438559658 sf123456
    # driver.find_element_by_xpath('/html/body/div[3]/div[1]/div[2]/form/div[2]/div/input[1]').send_keys('密码')#
    # time.sleep(10)
    # driver.find_element_by_xpath('/html/body/div[3]/div[1]/div[2]/form/div[4]/button').click()  # 点击登录
    # time.sleep(8)
    url = [
        "https://search.51job.com/list/000000,000000,0000,00,0,99,%25E5%25A4%25A7%25E6%2595%25B0%25E6%258D%25AE,2,{0}.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=".format(
            i) for i in range(60, 160)]
    for k2, k in enumerate(url):
        try:
            driver.get(k)
            time.sleep(10)
        except:
            print('****************************')
            time.sleep(30)
            try:
                driver.get(k)
            except:
                continue

        url_3 = []
        for id in range(1, 51):
            o_ = 0
            while o_ < 5:
                try:
                    x = driver.find_element_by_xpath(
                        '/html/body/div[2]/div[3]/div/div[2]/div[4]/div[1]/div[{}]/a'.format(id))
                    url_3.append(x.get_attribute("href"))
                    break
                except:
                    o_ += 1
                    time.sleep(3)
                    continue

        for j in url_3:
            li = []
            po = 1
            try:
                driver.get(j)  # 详细资料爬取
                time.sleep(3)
            except:
                time.sleep(40)
                try:
                    driver.get(j)  # 详细资料爬取
                except:
                    continue
            kl = 0
            while True:
                try:
                    kl += 1
                    title = driver.find_element_by_xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/h1').text
                    # li.extend(title[:5])
                    # li.append(title[5:])
                    #print(j)
                    break
                except:
                    if kl == 2:
                        print('--------*****************----------')
                        time.sleep(2)
                        print('-----------*************************************************-------')
                        po = 0
                        break
                    else:
                        time.sleep(5)
                        continue
            if po == 0:
                continue
            t1 = 0
            while True:
                try:
                    t1 += 1
                    # company = driver.find_element_by_xpath(
                    #     '/html/body/div[3]/div[2]/div[2]/div/div[1]/p[1]/a[1]').get_attribute('title')
                    area = \
                        driver.find_element_by_xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/p').get_attribute(
                            'title').split('|')[0]
                    area = area.strip()
                    experience = \
                        driver.find_element_by_xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/p').get_attribute(
                            'title').split('|')[1]
                    experience = experience.strip()
                    education = \
                        driver.find_element_by_xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/p').get_attribute(
                            'title').split('|')[2]
                    education = education.strip()
                    link=j
                    date = \
                        driver.find_element_by_xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/p').get_attribute(
                            'title').split('|')[3]
                    date = date.strip()
                    date = date.replace('发布', '')
                    salary = driver.find_element_by_xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/strong').text
                    message = '|' + driver.find_element_by_xpath('/html/body/div[2]/div[2]/div[3]/div[1]/div').text+'|'
                    break
                except:
                    time.sleep(3)
                    if t1 == 3:
                        break
            if t1 == 3:
                continue
            li.extend([title, area, experience, education,link,date, salary, message, ])
            #输出发布日期
            print(date)
            #print(title, area, experience, education, date, salary, message, )
            try:
                csv_writer.writerow(li)
                #输出内容
                print(num, k2)
                num += 1
            except:
                pass
            time.sleep(4)
            del li
        time.sleep(5)


def main():
    # while True:
    """
    chromeOptions 是一个配置 chrome 启动是属性的类,就是初始化
    """
    option = webdriver.ChromeOptions()
    # option.add_argument('--headless')
    # option.add_argument('blink-settings=imagesEnabled=false')

    # 不显示页面
    option.add_argument('--headless')
    #option.add_argument('--no-sandbox')
    option.add_argument('--disable-gpu')
    option.add_argument('--disable-dev-shm-usage')

    """
    add_experimental_option 添加实验性质的设置参数
    """
    option.add_experimental_option('excludeSwitches', ['enable-automation'])  # webdriver防检测
    '''
    add_argument 添加启动参数
    '''
    option.add_argument("--disable-blink-features=AutomationControlled")
    option.add_argument("--no-sandbox")
    option.add_argument("--disable-dev-usage")
    option.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})#不加载图片
    """
    Chrome 配置驱动
    """
    driver = webdriver.Chrome(executable_path=r'chromedriver.exe',options=option)
    driver.set_page_load_timeout(40)
    login(driver)
    # jugesd(driver)


if __name__ == '__main__':
    username = 'x'
    password = 'x'
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36'}
    main()
    fp.close()

创作打卡挑战赛

赢取流量/现金/CSDN周边激励大奖