? 写了一个简单的Python爬取指定用户微博的内容和图片，目前比较简陋，之前有在github上参考别人写的爬虫，发现现在微博好像使用的是Ajax的方式来渲染数据，这也太方便了，直接请求接口，然后解析数据不就能得到我们想要的数据了吗？？？？
? ok，开始操作
首先，我们进入微博，打开检查，观察数据，这里我用的方法比较蠢，是一个个看返回内容是什么。。。等后面学习到了更高深的技术了在和大家分享吧，
经过手动查看，发现mymblog接口是获取内容的接口，然后操作，我们使用请求数据的第三方库是requests，先将请求头和获取内容的地址还有获取信息的地址定义好
def __init__(self):
self.headers = {
'cookie': '',
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36',
'referer': 'https://weibo.com/u/3214393887'
}
self.user_info_url = 'https://weibo.com/ajax/profile/info?uid=3214393887'
self.content_url = 'https://weibo.com/ajax/statuses/mymblog?uid=5541182601&page={}&feature=0'
self.session = requests.session()
self.user = User()
self.contents = []
这里的请求头可以在浏览器中cv，不再赘述,
这里是我保存微博内容的对象，
class Weibo:
def __init__(self):
self.id = ''
self.user_id = ''
self.screen_name = ''
self.content = ''
self.article_url = ''
self.original_pictures = []
self.retweet_pictures = None
self.original = None
self.video_url = ''
self.publish_place = ''
self.publish_time = ''
self.publish_tool = ''
self.up_num = 0
self.retweet_num = 0
self.comment_num = 0
def __str__(self):
"""打印一条微博"""
result = self.content + '\n'
result += u'微博发布位置：%s\n' % self.publish_place
result += u'发布时间：%s\n' % self.publish_time
result += u'发布工具：%s\n' % self.publish_tool
result += u'点赞数：%d\n' % self.up_num
result += u'转发数：%d\n' % self.retweet_num
result += u'评论数：%d\n' % self.comment_num
# result += u'url：https://weibo.cn/comment/%s\n' % self.id
return result
解析数据的接口没啥好讲的，就是对症下药。。。
def get_data(self, since_id,page):
contentInfo = self.session.get(url=(self.content_url + '&since_id=' + since_id).format(page), headers=self.headers).content.decode()
resultDict = json.loads(contentInfo)
dataDict = resultDict['data']
dataList = dataDict['list']
index = 0
for data in dataList:
index += 1
wb = Weibo()
wb.id = data['id']
wb.user_id = data['user']['id']
wb.content = data['text_raw']
wb.publish_time = data['created_at']
wb.publish_tool = data['source']
wb.screen_name = data['user']['screen_name']
# wb.article_url = data['']
pic_ids = data['pic_ids']
try:
pic_infos = data['pic_infos']
for id in pic_ids:
info = pic_infos[id]
wb.original_pictures.append(info['original']['url'])
except:
print('没有图片')
#获取图片url
self.contents.append(wb)
if 'since_id' in dataDict:
page += 1
return dataDict['since_id'],page
else:
return ''
额，这里其实是有个问题的，这个微博的内容好像是下拉触发换页的，然后返回内容里面有一个since_id字段应该是用来判断是否还有下一页的，然后你在发请求的时候需要指定page参数的值所以，所以需要+1,
数据下来后，可以保存到Excel或者数据库，这里为了直观是存入Excel的，目前还有蛮多功能没有实现，比如视频下载，评论拉取和点赞数，后面慢慢完善吧
def save_as_excel(self):
wb = openpyxl.Workbook()
sheet = wb.get_active_sheet()
headers = ['时间', '发布工具', '内容', '图片', '发布地点', '点赞数', '评论']
index = 0
for header in headers:
index += 1
sheet.cell(row=1, column=index).value = header
row = 1
for weibo in self.contents:
row += 1
sheet.cell(row=row, column=1).value = weibo.publish_time
sheet.cell(row=row, column=2).value = weibo.publish_tool
sheet.cell(row=row, column=3).value = weibo.content
pics = ''
for pic in weibo.original_pictures:
pics += pic + '\n'
sheet.cell(row=row, column=4).value = pics
sheet.cell(row=row, column=5).value = weibo.publish_place
sheet.cell(row=row, column=6).value = ''
sheet.cell(row=row, column=7).value = ''
wb.save('小七.xlsx')
下载图片,根据前面获取到的链接下载图片，还是使用reuquest模块进行下载
def down_load_pic(self):
for content in self.contents:
text = content.content
imgs = content.original_pictures
index = 0
for img in imgs:
index += 1
filename = hashlib.md5((text+str(index)).encode()).hexdigest() + '.jpg'
dowload = self.session.get(url=img, headers=self.headers)
with open('./imgs/'+filename, 'wb') as f:
print('./imgs/'+filename)
f.write(dowload.content)
最后，可以继续完善
import requests
import lxml
from user import User
import json
from weibo import Weibo
import hashlib
import openpyxl
import time
class WeiBo(object):
def __init__(self, url):
self.url = url
self.headers = {
'cookie': '',
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36',
'referer': 'https://weibo.com/u/3214393887'
}
self.user_info_url = 'https://weibo.com/ajax/profile/info?uid=5541182601'
self.content_url = 'https://weibo.com/ajax/statuses/mymblog?uid=5541182601&page={}&feature=0'
self.session = requests.session()
self.user = User()
self.contents = []
def get_data(self, since_id,page):
contentInfo = self.session.get(url=(self.content_url + '&since_id=' + since_id).format(page), headers=self.headers).content.decode()
resultDict = json.loads(contentInfo)
dataDict = resultDict['data']
dataList = dataDict['list']
index = 0
for data in dataList:
index += 1
wb = Weibo()
wb.id = data['id']
wb.user_id = data['user']['id']
wb.content = data['text_raw']
wb.publish_time = data['created_at']
wb.publish_tool = data['source']
wb.screen_name = data['user']['screen_name']
# wb.article_url = data['']
pic_ids = data['pic_ids']
try:
pic_infos = data['pic_infos']
for id in pic_ids:
info = pic_infos[id]
wb.original_pictures.append(info['original']['url'])
except:
print('没有图片')
#获取图片url
self.contents.append(wb)
if 'since_id' in dataDict:
page += 1
return dataDict['since_id'],page
else:
return ''
def save_as_excel(self):
wb = openpyxl.Workbook()
sheet = wb.get_active_sheet()
headers = ['时间', '发布工具', '内容', '图片', '发布地点', '点赞数', '评论']
index = 0
for header in headers:
index += 1
sheet.cell(row=1, column=index).value = header
row = 1
for weibo in self.contents:
row += 1
sheet.cell(row=row, column=1).value = weibo.publish_time
sheet.cell(row=row, column=2).value = weibo.publish_tool
sheet.cell(row=row, column=3).value = weibo.content
pics = ''
for pic in weibo.original_pictures:
pics += pic + '\n'
sheet.cell(row=row, column=4).value = pics
sheet.cell(row=row, column=5).value = weibo.publish_place
sheet.cell(row=row, column=6).value = ''
sheet.cell(row=row, column=7).value = ''
wb.save('小七.xlsx')
def get_user_info(self):
userInfo = self.session.get(url=self.user_info_url, headers=self.headers).content.decode()
resultDict = json.loads(userInfo)
dataDict = resultDict['data']
userDict = dataDict['user']
self.user.id = userDict['id']
self.user.nickname = userDict['screen_name']
self.user.avator_img = userDict['profile_image_url'] + userDict['profile_url']
self.user.gender = '女' if userDict['gender'] == 'f' else '男'
self.user.description = userDict['description']
self.user.location = userDict['location']
self.user.following = userDict['followers_count']
self.user.followers = userDict['friends_count']
self.user.weibo_num = userDict['statuses_count']
def down_load_pic(self):
for content in self.contents:
text = content.content
imgs = content.original_pictures
index = 0
for img in imgs:
index += 1
filename = hashlib.md5((text+str(index)).encode()).hexdigest() + '.jpg'
dowload = self.session.get(url=img, headers=self.headers)
with open('./imgs/'+filename, 'wb') as f:
print('./imgs/'+filename)
f.write(dowload.content)
def parse_data(self):
#解析内容
"""时间 发布工具 内容 图片 发布地点 点赞数 评论"""
def run(self):
# 获取用户信息
self.get_user_info()
# print(self.user)
# 获取微博内容
since_id,page = self.get_data('',1)
while since_id != '' and page < 5:
since_id,page = self.get_data(since_id,page)
# print(self.contents)
self.save_as_excel()
#下载图片
# self.down_load_pic()
if __name__ == '__main__':
heyi = WeiBo('')
heyi.run()