def auto_page_search(url):print('开始自动查询网页')browser = webdriver.Chrome()# browser.get('https://wenku.baidu.com/view/dcfab8bff705cc175527096e.html')browser.get(url)print('等待5秒')time.sleep(5)# 下面这个语句并不是查找“继续阅读”按钮所在位置，而是更上面的元素，因为按照原本元素位置滑动窗口会遮挡住，大家可以试一试eles = browser.find_element_by_xpath('//*[@id="html-reader-go-more"]/div[1]/div[3]/div[1]')browser.execute_script('arguments[0].scrollIntoView();', eles)print('等待2秒')time.sleep(2)#点击“继续阅读”按钮browser.find_element_by_xpath('//*[@id="html-reader-go-more"]/div[2]/div[1]/span/span[2]').click()print('已显示文档所有内容')

def fetch_url(url):''':return: 网页源代码'''headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'}session = requests.session()sourse = session.get(url, headers=headers).content.decode('gbk')return sourse

def get_Id(sourse):Id=re.findall(r"docId.*?:.*?'(.*?)',",sourse)[0]return Iddef get_Type(sourse):Type=re.findall(r"docType.*?:.*?'(.*?)',",sourse)[0]return Typedef get_Title(sourse):Title=re.findall(r"title.*?:.*?'(.*?)',",sourse)[0]return Title

def parse_txt(Id):''':param Id: 之前爬取的 docId'''url1='https://wenku.baidu.com/api/doc/getdocinfo?callback=cb&doc_id={}'.format(Id)content1=fetch_url(url1)md5=re.findall('"md5sum":"(.*?)"',content1)[0]pn=re.findall('"totalPageNum":"(.*?)"',content1)[0]rsign=re.findall('"rsign":"(.*?)"',content1)[0]url2 = 'https://wkretype.bdimg.com/retype/text/' + Id + '?rn=' + pn + '&type=txt' + md5 + '&rsign=' + rsigncontent2=json.loads(fetch_url(url2))result=''for items in content2:for item in items:result=result+item['c'].replace('r','r').replace('n','n')return result#保存为 txt 文件
def save_file(filename, content):with open(filename, 'w', encoding='utf-8') as f:f.write(content)print('已保存为:' + filename)

def parse_doc(content):result = ''url_list = re.findall('(https.*?0.json.*?)x22}', content)url_list = [addr.replace("/", "/") for addr in url_list]for url in url_list[:-5]:content = fetch_url(url)y = 0txtlists = re.findall('"c":"(.*?)".*?"y":(.*?),', content)for item in txtlists:if not y == item[1]:y = item[1]n = 'n'else:n = ''result += nresult += item[0].encode('utf-8').decode('unicode_escape', 'ignore')return result

url_list = re.findall('(https.*?0.json.*?)x22}', content)

url_list = [addr.replace("/", "/") for addr in url_list]

def main():url1 = input('请输入需要爬取的文库地址:')auto_page_search(url1)sourse=fetch_url(url1)Id=get_Id(sourse)Type=get_Type(sourse)Title=get_Title(sourse)if Type=='txt':result=parse_txt(Id)save_file(Title+'.txt',result)elif Type=='doc':result=parse_doc(sourse)save_file(Title+'doc',result)