采集百度百家号动态加载的数据信息

问题：采集百度百家号文章、动态、视频的标题，发布时间，播放量，视频文件
阅读数等

代码：

# coding = utf-8
import os
import re
import time
import json
from datetime import datetime, timedelta
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
import requests
from PIL import Image
from PIL import ImageChops
from pathlib import Path
from fake_useragent import UserAgent
# from config import no_access_screenshot_time
from multiprocessing.dummy import Pool  # 线程池
from lxml import etree# ua = UserAgent()
headers = {# "Accept": '*/*',# 'Accept-Encoding': 'gzip, deflate',# 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,ar;q=0.7',# 'Connection': 'keep-alive',# 'Content-Length': '62',# 'Content-Type': 'application/json;charset=UTF-8',# 'Host': 'cq.gov.cn',# 'Origin': 'http://www.cq.gov.cn',# 'Referer': 'http://www.cq.gov.cn/','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36',
}ctime = ''
import importlib,sys
importlib.reload(sys)api_url = 'https://mbd.baidu.com/webpage?'f = open('dynamic.txt', 'w', encoding='utf-8')
f.close()
f = open('article.txt', 'w', encoding='utf-8')
f.close()f = open('video.txt', 'w', encoding='utf-8')
f.close()def write_txt_d(content):with open('dynamic.txt', 'a+', encoding='utf-8') as f1:f1.write(content)f1.close()def write_txt_a(content):with open('article.txt', 'a+', encoding='utf-8') as f1:f1.write(content)f1.close()def write_txt_v(content):with open('video.txt', 'a+', encoding='utf-8') as f1:f1.write(content)f1.close()def dynamic():"""动态资源获取:return:"""ctime = ''while True:params = {'tab': 'dynamic','num': '10','uk': 'FqzL6WxL7taxiI2tDzr_9A','source': 'pc','ctime': ctime,# 'ctime': '16215848239999','type': 'newhome','action': 'dynamic','format': 'jsonp','otherext': 'h5_20210521143926','Tenger-Mhor': '3133358778',# 'callback': '__jsonp11622020821123',}# 自动获取cookiesession = requests.Session()session.get(url=api_url)response = session.get(url=api_url, params=params)page = response.text# print(page)a = re.findall(r'"https:\\/\\/mbd.baidu.com\\/newspage\\/data\\/.*?"', page)        # 获取动态全文页面的urla = [i.replace('\\', '').replace('"', '') for i in a]                                # 清洗成可访问的动态全文页面的urlctime  = re.findall(r'"query":{"ctime":"(.*?)"', page)                              # 获取动态全文页面的urlif not ctime:returnctime = ctime[0]# print(a,ctime)for url_i in a:page_z = session.get(url=url_i)txt_context = page_z.texttry:txt_context = re.findall(r'"date":.*?"follow', txt_context, re.S)[0]                 # 获取页面有用的数据except:continueprint('************************************************************************************')# print(txt_context)read_num = re.findall(r'readNum":{"count":(.*?)},', txt_context, re.S)[0]  # 获取页面有用的数据print("网页地址:", url_i)print("阅读量:", read_num)time_str = re.findall('"date":"(.*?)",', txt_context)[0] + " " + re.findall('"time":"(.*?)",', txt_context)[0]print("动态发布时间:", time_str)content_txt = re.findall('"title":"(.*?)"', txt_context)[0]content_txt = re.sub(r'\[.*?]', '', content_txt)             # 把表情图片剔除掉# content_txt = content_txt.rplprint("文章内容:", content_txt.encode('utf-8').decode('unicode-escape'))image_url = re.findall('"image_url":"(.*?)"', txt_context)image_url = [i.replace('\\', '').replace('"', '') for i in image_url]  # 清洗成可访问的动态全文页面的urlprint("文章图片地址:", image_url)# img = 'https://pics0.baidu.com/feed/060828381f30e9241b7c2e5e2fbae00e1f95f7c4.jpeg?token=5ce97d75300ceb50f1856941a7c1c10d'praise_num = re.findall(r'"praise_num":(\d+),', txt_context)        # 点赞数commentNum = re.findall(r'"commentNum":(\d+),', txt_context)        # 评论数print("点赞数:{}，评论数:{}.".format(praise_num[0], commentNum[0]))str_content = "网页地址:" + url_i + '\n' + "阅读量:" + read_num + '\n' + "动态发布时间:" + time_str + '\n' + \"文章内容:" + content_txt.encode('utf-8').decode('unicode-escape') + "\n" + "文章图片地址:" + \str(image_url) + '\n' + "点赞数:{}，评论数:{}.".format(praise_num[0], commentNum[0]) + '\n' +\"************************************************************************************" + "\n"write_txt_d(str_content)# time.sleep(1)dynamic()def article():"""文章资源获取:return:"""ctime = ''time_start = datetime.now()while True:params = {'tab': 'article','num': '10','uk': 'FqzL6WxL7taxiI2tDzr_9A','source': 'pc','ctime': ctime,'type': 'newhome','action': 'dynamic','format': 'jsonp','otherext': 'h5_20210521143926','Tenger-Mhor': '3133358778',# callback: __jsonp41622099188559}session = requests.Session()session.get(url=api_url)response = session.get(url=api_url, params=params)page = response.text# print(page)a = re.findall(r'"url":"(.*?)",', page)  # 获取动态全文页面的urla = [i.replace('\\', '').replace('"', '') for i in a]  # 清洗成可访问的动态全文页面的url# print(a)ctime = re.findall(r'"ctime":(\d+)}', page)  # 获取动态全文页面的url# print(ctime)if not ctime:stop_time = datetime.now()print("运行所用时间:{}".format(stop_time-time_start))return# 获取需要请求的参数asyncParams = re.findall(r'"asyncParams":{(.*?)},', page)# 定义阅读量的列表read_num_list = []# 获取阅读量for i in asyncParams:asyncParams_url = r'https://mbd.baidu.com/webpage?type=homepage&action=interact&format=jsonp&Tenger-Mhor=3133358778&params=[{%s}]&uk=FqzL6WxL7taxiI2tDzr_9A&' % iqwq = session.get(url=asyncParams_url).texttry:re_url = re.findall(r'"read_num":(.*?),', qwq)[0]read_num_list.append(re_url)except:print('qwqw')asyncParams_url = r'https://mbd.baidu.com/webpage?type=homepage&action=interact&format=jsonp&Tenger-Mhor=3133358778&params=[{%s}]&uk=FqzL6WxL7taxiI2tDzr_9A&' % iqwq = session.get(url=asyncParams_url).textre_url = re.findall(r'"read_num":(.*?),', qwq)[0]read_num_list.append(re_url)print("********未获取到重新获取一遍！*********")print(asyncParams_url)print(qwq)print(re_url)print(read_num_list)print('qwqw')# returnfor url_i in a:print("阅读量:", read_num_list[a.index(url_i)])print("网页地址:", url_i)page_z = session.get(url=url_i)# page_z = session.get(url=url_i)page_text = page_z.text# 文章发布时try:time_str = re.findall('"date":"(.*?)",', page_text)[0] +" "+ re.findall('"time":"(.*?)",', page_text)[0]except:continueprint("文章发布时间:", time_str)text_title = re.findall(r'leTitle_28fPT">(.*?)<', page_text)[0]print("文章标题:", text_title)txt_context = re.findall(r'诸暨市禾福健康管理有限公司官方帐号,优质健康领域创作者(.*?)相关文章', page_text)[0]text = re.sub(r'<.*?>','',txt_context)print("文章内容:", text)# 文章图片获取:image_url = re.findall('img src="(.*?)"', txt_context)print("文章图片地址:", image_url, len(image_url))str_content = "网页地址:" + url_i + '\n' + "阅读量:" + read_num_list[a.index(url_i)] + '\n' + "动态发布时间:" + time_str + '\n' + \"文章标题:" + text_title + '\n' + "文章内容:" + text + "\n" + "文章图片地址:" + \str(image_url) + '\n' + "************************************************************************************" + '\n'print('************************************************************************************')write_txt_a(str_content)article()video_api = 'https://mbd.baidu.com/webpage?'
def video():ctime = ''video_n = 0ctime_list = []while True:params = {'tab': 'video','num': '10','uk': 'FqzL6WxL7taxiI2tDzr_9A','source': 'pc','ctime': ctime,'type': 'newhome','action': 'dynamic','format': 'jsonp','otherext': 'h5_20210521143926','Tenger-Mhor': '2959007734',# callback: __jsonp41622099188559}session = requests.Session()session.get(url=video_api)response = session.get(url=api_url, params=params)page = response.text# print(page)a = re.findall(r'"share_url":"(.*?)"', page)  # 获取动态全文页面的urla = [i.replace('\\', '').replace('"', '') for i in a]  # 清洗成可访问的动态全文页面的urlctime = re.findall(r'"query":{"ctime":(.*?)}', page)  # 获取动态全文页面的urlctime_list.append(ctime)# print(a,ctime)if not ctime:returnif ctime_list.count(ctime) > 1:returnfor url_v in a:page_z = session.get(url=url_v)txt_context = page_z.text# print(txt_context)print('网页地址:'+url_v)video_title = re.findall(r'title">(.*?)<', txt_context)print('视频标题:'+video_title[0])time_num = re.findall(r'laynums">(.*?)<', txt_context)print('播放次数和时间:'+time_num[0])video_download = re.findall(r'"url":"(.*?)"', txt_context)video_download = [i.replace('\\', '').replace('"', '') for i in video_download]print(video_download[-1])video_n += 1print(video_n)bytes_v = session.get(url=video_download[-1]).contentwith open('./video/{}.mp4'.format(video_n), 'wb') as fv:fv.write(bytes_v)str_content = "网页地址:" + url_v + '\n' + '播放次数和时间:'+time_num[0] + '\n' + \"视频标题:" + video_title[0] + "\n" + "本地视频名称:" + '{}.mp4'.format(video_n) + "\n" + "视频下载地址:" + \video_download[-1] + '\n' + "************************************************************************************" + '\n'write_txt_v(str_content)print('************************************************************************************')video()
qtgw_url = 'https://author.baidu.com/home?from=bjh_article&app_id=1536910259501019'

采集后的txt文件:

采集百度百家号动态加载的数据信息相关推荐

企业如何借助百度百家号进行网络推广？
有些中小企业的网络推广需求很简单,就是想网上搜自己的品牌关键词及产品关键词能搜到自己企业的信息,这个最简单的方法就是用百度的百家号.百家号认证蓝V之后会增加一个百家名片的展示,且会给一些流量支持,相对 ...
百度百家号“漏洞”大曝光一篇文章十几万阅读量收入上万
越来越发现自媒体的好处,就是现在太多的平台都提供入口,基本是四大门户还有三大巨头互联网公司,百度.腾讯 .阿里都提供!那么接下来我要分享的是中国最大的搜索引擎公司-百度!,相信最近很多人都知道百度百家 ...
百度百家号作者昵称、ID、粉丝数量获取
最近在练习爬虫,试了试爬取百度百家号的作者粉丝数量,文中代码纯属练习专用. import requests import re import json from urllib import parse ...
【报告分享】2021内容行业研究报告-百度百家号新榜（附下载）
摘要:近年来,用户对内容消费的需求迅速增加,内容平台百家齐放,为创作者提供了多种选择的机会.为进一步盘点目前内容行业的生态,加深创作者对内容行业发展现状及趋势的认知,为创作者更好地实现个人价值和商业价 ...
Android使用webview获取百度百家号网站源码
Android使用webview获取网站源码最近在使用webview去获取网页的源码,但是对于百度百家号的文章,网页的源代码获取出来只要标题,内容不见了,想问问有没有大佬知道这个问题.去用pytho ...
李彦宏直播首秀：来百度百家号APP带货没问题
李彦宏直播首秀:来百度百家号APP带货没问题直播作为一个新的媒体形式,通过百度平台来表达信息,来表达知识,那是非常必要的,所以我自己也想亲自体会,我有可能一些问题是比较偏门的问题,在百度里头搜不一定 ...
如何查看百度百家号审核进度
百度百家号是一个比较新的自媒体,前段时间刚开放注册,但需要审核,有的朋友说秒过,有的反映说一直在等待审核中,如下图所示.百度占据了很大的流量入口,特别是pc端,然后又开放了作者广告收入模块,所以百家号 ...
vue动态跟新layui的select_vue+layui实现select动态加载后台数据的例子
vue+layui实现select动态加载后台数据的例子发布时间:2020-09-18 22:51:36 来源:脚本之家阅读:85 作者:qq_26814945 刚开始由于layui form渲染 ...
jquery table ajax,JQuery Ajax动态加载Table数据的实例讲解
我们在jsp定义一个select和一个table,要求实现根据select的选值,动态加载table数据. table第一次加载数据的function定义如下: function loadData() ...

采集百度百家号动态加载的数据信息

采集百度百家号动态加载的数据信息相关推荐

最新文章

热门文章