备忘1：爬取热门微博评论

'''python3'''

'''2018/04/03'''

import requests
import os
import re
import csv
import time
import json#headers是请求加上头信息，伪装成浏览器访问，不然会被限制
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'}
Cookies = {"Cookie": "_T_WM=ac858e32012c2c4bfcf46782f5928d99; ALF=1525310818; SCF=AktDkPHfGtZ_G6P28yFN5QufvOsFbI5pFfURfdnppHMy3xq6GeZP_nVIBfgPk63bE8-IWesUjhW3dLT-uIR5Uwo.; SUB=_2A253xqp3DeRhGeVO41YZ8ijOwjyIHXVVSDY_rDV6PUJbktANLUHCkW1NTSfFYX53m_bQ56KWd8ygtVjCf2FsTQ5t; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WFEZ5vWGfwYo6aYTg5NIEeO5JpX5K-hUgL.Foe71hBReoqE1K52dJLoIpeLxKqL1-BLBKnLxKqL1KnL128j; SUHB=0cSXDvABOpSMLa; SSOLoginState=1522719271; WEIBOCN_FROM=1110006030; M_WEIBOCN_PARAMS=oid%3D4224051549979655%26luicode%3D10000011%26lfid%3D1076033084826290%26fid%3D1005053084826290%26uicode%3D10000011"}
#当出现一些解决不了的问题时候  试着更新一下Cookies#用户信息，同时也能获取到uid、fid、oid等关键参数
def get_user_info(usr_id):url = 'https://m.weibo.cn/api/container/getIndex?type=uid&value={usr_id}'.format(usr_id=usr_id)resp = requests.get(url, headers=headers, cookies=Cookies)jsondata = resp.json()#print(jsondata)nickname = jsondata.get('data').get('userInfo').get('screen_name')mblog_num = jsondata.get('data').get('userInfo').get('statuses_count')verified = jsondata.get('data').get('userInfo').get('verified')verified_reason = jsondata.get('data').get('userInfo').get('verified_reason')gender = jsondata.get('data').get('userInfo').get('gender')urank = jsondata.get('data').get('userInfo').get('urank')  #用户等级mbrank = jsondata.get('data').get('userInfo').get('mbrank')followers_count = jsondata.get('data').get('userInfo').get('followers_count')follow_count = jsondata.get('data').get('userInfo').get('follow_count')uid = jsondata.get('data').get('userInfo').get('toolbar_menus')[0].get('params').get('uid')try:fid = jsondata.get('data').get('userInfo').get('toolbar_menus')[1].get('actionlog').get('fid')oid = jsondata.get('data').get('userInfo').get('toolbar_menus')[2].get('params').get('menu_list')[1].get('actionlog').get('oid')#注意get('menu_list')[1] 是列表的第二个  不是第一个cardid = jsondata.get('data').get('userInfo').get('toolbar_menus')[1].get('actionlog').get('cardid')except:uid = ''fid = ''oid = ''cardid = ''containerid = jsondata.get('data').get('tabsInfo').get('tabs')[0].get('containerid')Info = {'nickname':nickname,'mblog_num':mblog_num,'verified':verified,'verified_reason':verified_reason,'gender':gender,'urank':urank,'mbrank':mbrank,'followers_count':followers_count,'follow_count':follow_count,'uid':uid,'fid':fid,'cardid':cardid,'containerid':containerid,'oid':oid}print(Info)return Info#获取所有热门微博信息（所发微博内容，每条微博的评论id,转发数，评论数...）
def mblog_list(uid,oid):Mblog_list = []base_url = 'https://m.weibo.cn/api/container/getIndex?containerid={oid}'#base_url='https://m.weibo.cn/api/container/getIndex?containerid={oid}&luicode=10000011&lfid=1005051282005885&featurecode=20000320'#base_url= 'https://m.weibo.cn/api/container/getIndex?containerid={uid}'page_url = 'https://m.weibo.cn/api/container/getIndex?containerid={oid}&type=uid&value={uid}&page={page}'#page_url ='https://m.weibo.cn/api/container/getIndex?containerid={uid}&page={page}'url = base_url.format(oid=oid)print(url)resp = requests.get(url, headers=headers, cookies=Cookies)resp.encoding = 'gbk'response = resp.json()#print(response)#热门微博数totaltotal = response['data']['cardlistInfo']['total']print(total)#热门微博网页数page_num = int(int(total)/10)+1for i in range(page_num+1,1):p_url = page_url.format(oid=oid, uid=uid, page=i)#print(p_url)page_resp = requests.get(p_url,headers=headers,cookies=Cookies)resp.encoding = 'gbk'page_data = page_resp.json()'''filename='22.json'with open(filename,'w') as f:json.dump(page_data,f)'''try:cards = page_data['data']['cards']#print(cards)for card in cards:#print(card)try:                    mblog = card['mblog']created_at = mblog['created_at']id = mblog['id']dirty_text = mblog['text']  #dirty_text中含有很多链接杂质cleaned1 = re.sub(r'<span .*?</span>', '', dirty_text)text = re.sub(r"<a .*?</a>", '', cleaned1)reposts_count = mblog['reposts_count']comments_count = mblog['comments_count']attitudes_count = mblog['attitudes_count']mblog_data = {'created_at': created_at, 'id': id, 'text': text, 'reposts_count': reposts_count,'comments_count': comments_count, 'attitudes_count': attitudes_count}Mblog_list.append(mblog_data)#print(' '*10,mblog_data)except:                 continueexcept:continuetime.sleep(1)return Mblog_list#获取某微博评论，保存到usr_id下的文件夹wb_id.csv文件中
def get_comments(usr_id, wb_id):url = 'https://m.weibo.cn/api/comments/show?id={id}'.format(id=wb_id)page_url = 'https://m.weibo.cn/api/comments/show?id={id}&page={page}'Resp = requests.get(url, headers=headers, cookies=Cookies)Resp.encoding = 'gbk'#print(url)page_max_num = Resp.json()['data']['max']path = os.getcwd()+'/{dirname}/'.format(dirname=usr_id)os.mkdir(path)path2 = os.getcwd() + '/%s/%s.csv'%(usr_id,wb_id)csvfile = open(path2, 'a+', encoding='gb18030', newline='')writer = csv.writer(csvfile)writer.writerow(('username','verified','verified_type','profile_url','source','review_id','like_counts','image','date','comment'))for i in range(1,page_max_num+1):if i==100:breakp_url = page_url.format(id=wb_id,page=i)resp = requests.get(p_url, cookies=Cookies, headers=headers)print(resp.status_code)resp_data = resp.json()try:data = resp_data.get('data').get('data')#print(data)for d in data:review_id = d['id']like_counts = d['like_counts']source = d['source']username = d['user']['screen_name']image = d['user']['profile_image_url']verified = d['user']['verified']verified_type = d['user']['verified_type']profile_url = d['user']['profile_url']dirty_text = d['text']cleaned1 = re.sub(r'<span .*?</span>', '', dirty_text)comment = re.sub(r"<a .*?</a>", '', cleaned1)date = d['created_at']print(comment)writer.writerow((username, verified, verified_type, profile_url, source, review_id, like_counts, image,date, comment))print('有%d页，已经爬了%d页   %s'%(page_max_num, i, comment))except:print(resp_data['msg'])continuetime.sleep(1)csvfile.close()def main():#user_id= '1655128924'#user_id='2736225585'#user_id = '2386831995'user_id= '1721030997'wb_id='4223783835437135'user_info = get_user_info(user_id)uid = user_info.get('uid')oid = user_info.get('oid')print(uid,oid)mblog_list(uid,oid) print('............')get_comments(user_id,wb_id)main()#备注1：微博的json内容 不断的在更新  如果出现数据为空的情况下，print（json数据）查看一个对应的字典内容有没有改变

#备注2：另外就目前而言只能爬取100页的微博评论

备忘1：爬取热门微博评论相关推荐

python+selenium爬虫，使用selenium爬取热门微博数据
python爬虫使用selenium爬取热门微博数据完整代码 from selenium.webdriver import Chrome import time import csvf = open ...
中国女足绝地大逆转，爬取了微博评论区，评论很精彩
昨晚,女足16年后重夺亚洲杯,决赛落后两球,依然能保持对比赛的观察和思考,下半场从容调度人手,最后完成逆转. 打开微博一看,WC,微博推给我的第一条就是一篇瓜文. 这几天正好有的瓜,于是就爬了一下微博 ...
爬取热门微博数据2018.3.27更新
import requests import os import re import csv import time import jsonheaders = {'User-Agent':'Mozil ...
Python爬取热门微博，并存储到MySQL中
目标网站:m.weibo.cn url的获取可以从浏览器的F12中的network的XHR中找到. weibo_demo.py: import requests import json from w3 ...
从入门到入土：Python实现爬取刷新微博推荐和最新好友微博|cookie调用|模拟登录
写在前面: 此博客仅用于记录个人学习进度,学识浅薄,若有错误观点欢迎评论区指出.欢迎各位前来交流.(部分材料来源网络,若有侵权,立即删除) Python实现爬取刷新微博推荐和最新好友微博|cookie ...
如何科学地蹭热点：用python爬虫获取热门微博评论并进行情感分析
前言:本文主要涉及知识点包括新浪微博爬虫.python对数据库的简单读写.简单的列表数据去重.简单的自然语言处理(snowNLP模块.机器学习).适合有一定编程基础,并对python有所了解的盆友阅读 ...
爬取热门网站的热榜，集中展示
爬取热门网站的热榜,集中展示:Integrate and display hot billboard or ranked topic from hot Chinese websites 抓取知乎热榜数 ...
(2020)使用Airtest来爬取某宝评论数据
本帖子背景:本帖子,是一个学习的过程.一个爬取某宝评论的小单子,促使我前来学习简单的.不用掉头发的.不用爆肝的(再说,我也没那能力去硬干它啊)教程昨天晚上,大学四年的舍友微信问我"说还搞爬 ...
Scrapy爬取猫眼电影评论
Scrapy爬取猫眼电影评论文章目录 Scrapy爬取猫眼电影评论 1.寻找评论接口 2.分析接口URL 接口URL规律构造URL接口分析JSON参数 3.Scrapy代码 spiders文件 ...

备忘1：爬取热门微博评论

备忘1：爬取热门微博评论相关推荐

最新文章

热门文章