[Python知识库]python 爬取豆瓣电影top250的电影

通过python的urlib和re模块提取豆瓣top250的电影的排行，电影名字，导演名字，电影评分的信息。

from urllib.request import Request,urlopen
import re

# 1.查找网页源代码
def get_page(url):
    # 1.准备请求信息
    r = Request(url=url,
                headers={
                'User-Agent':'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'})
    # 2.打开url
    response = urlopen(r)

    # 3.因为得到的文件用utf-8进行编码，所以用 utf - 8进行解码
    return response.read().decode('utf-8')

# 2.从网页中提取需要的信息
def get_infomation(s):
        # 正则表达式
        obj = re.compile(r'<div class="item">.*?<em class="">(?P<排名>.*?)</em>'
                         r'.*?<span class="title">(?P<电影名>.*?)</span>'
                         r'.*?导演: (?P<导演>.*?) '
                         r'.*?<span class="rating_num" property="v:average">(?P<评分>.*?)</span>'
                         , re.S)
        # 运用正则表达式提取信息
        result = obj.finditer(s)
        lst = []
        for item in result:
            lst.append(item.groupdict())
            # lst.append(item.group())
        return lst
def main():
    result = []
    # 把所有的网页的信息都提取出来
    for i in range(10):
        s = get_page(f'https://movie.douban.com/top250?start={i * 25}')
        result += get_infomation(s)
    with open("test.txt", mode='a', encoding='utf-8') as fp:
        for item in result:
            fp.write(str(item))
            fp.write('\n')
    lst = []
    # 去除字典的’ 和 { } 符号
    with open("test.txt", mode='r', encoding='utf-8') as fp:
        for line in fp:
            k = line.replace('{', "")
            k = k.replace("}", "")
            k = k.replace("'", "")
            lst.append(k)
    with open("test.txt", mode='w', encoding='utf-8') as fp:
        for item in lst:
            fp.write(item)
main()

最后的效果：