[Python知识库]天气后报数据爬虫pythont

代码部分

#!/usr/bin/python3
# -*- coding: utf-8 -*-
# @Time    : 2021/12/27 15:45
# @Author  : wangshenhui
# @FileName: 衡阳2019天气爬虫代码.py
# @Software: PyCharm
# 拿到页面源代码
# 导入第三方库
import requests
from bs4 import BeautifulSoup
import pandas as pd

# 查看数据格式
def read_data():
    url = 'http://tianqihoubao.com/lishi/hengyang/month/201902.html'
    df = pd.read_html(url, encoding='gbk', header=0)[0]
    return df

# 拼接url（网页链接）
def get_url(City,Year,month):
    base_url = 'http://tianqihoubao.com/lishi/{}/month/{}{}.html'
    url = base_url.format(City, Year, month)
    return url

# 这是个通用爬虫
def get_data(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
    }
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text, 'lxml')
    # print(soup)
    table = soup.find('table')
    # find()函数用于查找元素并返回第一个
    df = pd.read_html(table.prettify(), header=0)[0]
    # prettify()函数修剪数据只保留context部分
    return df

# 保存数据
def save_data(dfs, filename):
    for df in dfs:
        df.to_csv(filename, mode='a', header=None)

# 主函数
if __name__ == '__main__':

    City = input('输入城市：')
    Year = input('输入年份：')
    filename = City + Year
    for month in range(1, 13):
        print("正则爬取第%d月的数据。。。" % month)
        dfs = []
        month = str(month)
        if len(month) == 1:
            month = '0' + month
        url = get_url(City,Year,month)
        df = get_data(url)
        dfs.append(df)
        save_data(dfs, (filename + '.csv'))
    print('完成')

数据展示

天气后报网站
要取随意，点赞自取。