python爬去百度文库资料_Python在线百度文库爬虫(免下载券)

百度文库,文字类型的资源(如txt, doc, pdf),是可以在前端源码分析获取到的,如果能按照规则合理的提取这些文字资源,就可以实现免下载券获取资源.

作者完成了百度文库的在线版爬虫,只需提供文库资源页面的url,就可以获取对应的文库资源(免下载券)

在线测试地址:

http://zhaozhaoli.vicp.io/spider/bdwk

效果展示

初始界面.png

摄影课感想_百度文库_and_方圆小站.png

获取资源.png

展示文本.png

关于ppt的爬取,作者也写出来了,但是我的个人服务器带宽太小,导致前端等待时间过长,就把它注释了...有兴趣的可以看以前写过的这篇文章百度文库下载器,文章底部提供了编译好的可执行文件(支持ppt,txt,pdf,word格式的下载),可以下载到本地玩一玩...

核心源码:

import os

import re

import json

import requests

from lxml import etree

import sys

from fangyuanxiaozhan.settings import BASE_DIR

# import tkinter

# 创建文库基类

class BaiduWK(object):

def __init__(self, url):

self.title = None

self.url = url

self.docType = None

self.headers = {

'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36'}

self.get_response_content(self.url)

self.get_doc_type_and_title()

def get_response_content(self, url):

try:

response = requests.get(url, headers=self.headers)

return response.content

except Exception as e:

print(e)

pass

def get_doc_type_and_title(self):

# 获取源码

source_html = self.get_response_content(self.url)

# 解析源码

content = source_html.decode('gbk')

# 获取文档类型

self.docType = re.findall(r"docType.*?\:.*?\'(.*?)\'\,", content)[0]

# 获取文档标题

self.title = re.findall(r"title.*?\:.*?\'(.*?)\'\,", content)[0]

# 创建获取txt的类

class BDWKTXT(BaiduWK):

def __init__(self, url):

super().__init__(url)

self.docId = None

pass

def get_txt(self, url):

# 获取源码

source_html = self.get_response_content(url)

content = source_html.decode("gbk")

# 获取docId

self.docId = re.findall(r"docId.*?(\w{24}?)\'\,", content)[0]

# 拼接请求url

token_url = "https://wenku.baidu.com/api/doc/getdocinfo?callback=cb&doc_id=" + self.docId

# 再次请求

first_json = self.get_response_content(token_url).decode()

str_first_json = re.match(r'.*?$(\{.*?\})$.*', first_json).group(1)

# print(str_first_json)

the_first_json = json.loads(str_first_json)

md5sum = the_first_json["md5sum"]

rn = the_first_json["docInfo"]["totalPageNum"]

rsign = the_first_json["rsign"]

# print(md5sum,"-->",rn)

# 请求目标url

target_url = "https://wkretype.bdimg.com/retype/text/" + self.docId + "?" + md5sum + "&callback=cb" + "&pn=1&rn=" + rn + "&type=txt" + "&rsign=" + rsign

# https://wkretype.bdimg.com/retype/text/de53bafaf705cc1755270982?md5sum=b89f055e765e6f73db57525bcfc3c2d2&sign=7b385f9cf7&callback=cb&pn=1&rn=12&type=txt

sec_json = self.get_response_content(target_url).decode()

# print(type(sec_json),"-->")

str_sec_json = re.match(r'.*?$\[(.*)\]$$', sec_json).group(1)

str_sec_json += ","

str_json_list = str_sec_json.split('},')

result_txt = ""

# 截取尾部空格

str_json_list = str_json_list[:-1]

for str_json in str_json_list:

str_json += "}"

pure_dic = json.loads(str_json)

pure_txt = pure_dic["parags"][0]["c"].strip()

result_txt += pure_txt

# 创建文件目录

try:

# path = "." + os.sep + self.docType

path = BASE_DIR + "/static/spider_file/bdwk" + os.sep + self.docType

print(type(path),"path-->",path)

os.makedirs(path)

except Exception as e:

# print("文件夹%s已存在"%(path))

pass

# 创建文件,保存信息

try:

print(os.path)

file_name = BASE_DIR+"/static/spider_file/bdwk" + os.sep + self.docType + os.sep + self.title + ".txt"

with open(file_name, 'w', encoding='utf-8') as f:

f.write(result_txt)

print("已经保存为:", self.title + '.txt')

except Exception as e:

# print(e)

pass

# 创建获取word的类

class BDWKDOC(BaiduWK):

def __init__(self, url):

super().__init__(url)

# 保存数据来源url

self.pure_addr_list = list()

# 获取数据来源url

def get_pure_addr_list(self):

# 获取页面源码

source_html = self.get_response_content(self.url).decode('gbk')

# 从源码中批量提取数据url

all_addr = re.findall(r'wkbos\.bdimg\.com.*?json.*?expire.*?\}', source_html)

pure_addr_list = list()

# 获取文档标题

self.title = etree.HTML(source_html).xpath("//title/text()")[0].strip()

# 净化数据来源url列表

for addr in all_addr:

addr = "https://" + addr.replace("\\\\\\/", "/")

addr = addr[:-5]

pure_addr_list.append(addr)

# 将处理好的url列表保存为全局属性

self.pure_addr_list = pure_addr_list

return pure_addr_list

# 从数据来源的url列表中提取数据

def get_json_content(self, url_list):

content = ''

result = ''

sum = len(url_list)

i = 1

for pure_addr in url_list:

print("正在下载第%d条数据, 剩余%d条" % (i, sum - i))

i += 1

try:

# 获取json数据

content = self.get_response_content(pure_addr).decode()

# 处理json数据

content = re.match(r'.*?$(.*)$$', content).group(1)

# 将json数据中需要的内容提取出来

all_body_info = json.loads(content)["body"]

# 遍历获取所有信息,并将信息拼接到一起

for body_info in all_body_info:

try:

result = result + body_info["c"].strip()

# print(">>",result)

except Exception as e:

print(e)

pass

except Exception as e:

print(e)

pass

# 创建文件目录

try:

path = BASE_DIR+"/static/spider_file/bdwk" + os.sep + self.docType

# print(type(path),"path-->",path)

os.makedirs(path)

except Exception as e:

# print("文件夹%s已存在"%(path))

pass

# 创建文件,保存信息

try:

file_name = BASE_DIR+"/static/spider_file/bdwk" + os.sep + self.docType + os.sep + self.title + ".txt"

with open(file_name, 'w', encoding='utf-8') as f:

f.write(result)

print("已经保存为:", self.title + '.txt')

except Exception as e:

print(e)

# 创建获取ppt的类

class BDWKPPT(BaiduWK):

def __init__(self, url):

self.all_img_url = list()

super().__init__(url)

# 获取json数据保存文件

def get_ppt_json_info(self):

# 获取源文件

ppt_source_html = self.get_response_content(self.url)

# 解析源文件

content = ppt_source_html.decode('gbk')

# 测试

with open("test.html", "w") as f:

f.write(content)

# 获取文档Id

self.docId = re.findall(r"docId.*?(\w{24}?)\'\,", content)[0]

# 拼接请求json的接口

source_json_url = 'https://wenku.baidu.com/browse/getbcsurl?doc_id=%s&type=ppt&callback=zhaozhao' % self.docId

# 获取字符串类型的json数据

str_source_json = self.get_response_content(source_json_url).decode()

# 处理字符串类型的json数据,使其成为标准格式

pure_str_source_json = re.match(r'.*?$(.*?)$', str_source_json).group(1)

# 将字符串json转为可处理的正式json

source_json = json.loads(pure_str_source_json)

# 遍历字典中的数据类型list

for j in source_json['list']:

# 创建临时列表

temp_num_url = list()

# 将url和page拼接到列表中

temp_num_url.append(j["zoom"])

temp_num_url.append(j["page"])

# 将列表信息添加到全局变量中

self.all_img_url.append(temp_num_url)

# 建立文件夹

try:

os.makedirs(BASE_DIR+"/static/spider_file/bdwk/ppt/%s" % (self.title))

except Exception as e:

pass

for img_url in self.all_img_url:

print(BASE_DIR+"正在获取第%d页资源(剩余%d页)" % (img_url[1], len(self.all_img_url) - img_url[1]))

data = self.get_response_content(img_url[0])

path = BASE_DIR+"/static/spider_file/bdwk/ppt/%s/%s" % (self.title, str(img_url[1]) + '.jpg')

with open(path, 'wb') as f:

f.write(data)

print("写入完毕")

# 运行主程序

def main(url):

try:

# url = input("请输入资源所在的网址:")

docType = BaiduWK(url).docType

except:

print("您输入的url,有误请重新输入!")

os.exit()

print("类型为", "-->", docType)

if docType == "ppt2":

ppt = BDWKPPT(url)

print("您将要获取的演示文稿(ppt)名称为:", ppt.title)

ppt.get_ppt_json_info()

data = dict()

src = "/static/spider_file/bdwk/ppt/%s/" % (ppt.title)

title_name = ppt.title

data["src"] = src

data["title_name"] = title_name

return data

elif docType == "doc":

word = BDWKDOC(url)

print("您将要获取的文档(word)名称为", word.title)

pure_addr_list = word.get_pure_addr_list()

word.get_json_content(pure_addr_list)

data = dict()

src = "/static/spider_file/bdwk/doc/%s.txt" % (word.title)

title_name = word.title

data["src"] = src

data["title_name"] = title_name+".txt"

return data

elif docType == "pdf":

pdf = BDWKDOC(url)

print("您将要获取的PDF名称为:", pdf.title)

pure_addr_list = pdf.get_pure_addr_list()

pdf.get_json_content(pure_addr_list)

data = dict()

src = "/static/spider_file/bdwk/pdf/%s.txt" % (pdf.title)

title_name = pdf.title

data["src"] = src

data["title_name"] = title_name+".txt"

return data

elif docType == "txt":

txt = BDWKTXT(url)

print("您将要下载的文本文档(txt)名称为:", txt.title)

txt.get_txt(url)

data = dict()

src = "/static/spider_file/bdwk/txt/%s.txt" % (txt.title)

title_name = txt.title

data["src"] = src

data["title_name"] = title_name+".txt"

return data

else:

other = BDWKPPT(url)

print("暂不支持下载%s类型" % (other.docType))

data = dict()

data["src"] = ""

data["title_name"] = "暂不支持%s类型"%(other.docType)

return data

python爬去百度文库资料_Python在线百度文库爬虫(免下载券)相关推荐

python百度文库免费_Python在线百度文库爬虫(免下载券)
百度文库,文字类型的资源(如txt, doc, pdf),是可以在前端源码分析获取到的,如果能按照规则合理的提取这些文字资源,就可以实现免下载券获取资源. 作者完成了百度文库的在线版爬虫,只需提供文库 ...
《脚本》Python在线百度文库爬虫(免下载券)
百度文库,文字类型的资源(如txt, doc, pdf),是可以在前端源码分析获取到的,如果能按照规则合理的提取这些文字资源,就可以实现免下载券获取资源. 作者完成了百度文库的在线版爬虫,只需提供文库 ...
python爬去中国天气网_python爬取中国天气网并展示最低温度
import requests from bs4 import BeautifulSoup import lxml import json from echarts import Echart,Bar ...
python爬取小说项目概述_Python实战项目网络爬虫之爬取小说吧小说正文
本次实战项目适合,有一定Python语法知识的小白学员.本人也是根据一些网上的资料,自己摸索编写的内容.有不明白的童鞋,欢迎提问. 目的:爬取百度小说吧中的原创小说<猎奇师>部分小说内容 ...
python 爬取菜鸟教程python100题，百度贴吧图片反爬虫下载，批量下载
每天一点点,记录学习 python 爬取菜鸟教程python100题近期爬虫项目,看完请点赞哦: 1:python 爬取菜鸟教程python100题,百度贴吧图片反爬虫下载,批量下载 2:pytho ...
用Python爬取动态加载的诸如百度的图片
用Python爬取动态加载的诸如百度的图片使用原因代码如下模块介绍 selenium模块 lxml解析模块 requests模块去除重复开启多线程效果如下留言使用原因学习Python ...
python爬去淘宝西装数据
python爬去淘宝西装数据啥也不说代码先码上 #爬取数据 from selenium import webdriver from bs4 import BeautifulSoup import t ...
python爬微博数据合法吗_Python 免登录微博数据爬虫(仅研究使用)
微博数据是非常有价值的数据,这些数据可以用作我们进行一些系统开发时的数据源, 比如前段时间发过的:Python 短文本识别个体是否有自杀倾向,在此文中,我们使用微博绝望树洞的数据,利用SVM做了一个简 ...
Python爬取网易云音乐热歌榜（爬虫）
Python爬取网易云音乐热歌榜歌曲,并下载到本地找到要下载歌曲排行榜的链接,这里用的是: https://music.163.com/discover/toplist?id=3778678 然后更 ...

python爬去百度文库资料_Python在线百度文库爬虫(免下载券)

python爬去百度文库资料_Python在线百度文库爬虫(免下载券)相关推荐

最新文章

热门文章