用python实现相声、评书自动下载（五）

写完这些后，觉得分两个脚本跑太麻烦，而且每次还要去改下载脚本的csv文件名，效率太低。于是把所有的代码整合到一起，实现伪·全自动下载，只要有第一页的链接，就可以下完所有的相声、评书。

发这篇博客的时候出现了一个问题，用

link = r.html.absolute_links爬源码，返回的结果集变成了se()，还不清楚是什么原因。

其实做成全自动的也有思路，就是打开任意一个人的作品集。抓取所有作品的“第一页”链接，然后用getAllLink()函数去打开每个作品的第一页，后面的步骤都一样。也许以后有空了会写一下。

from selenium import webdriver
from requests_html import HTMLSession
import requests
import time
import re
import pandas as pdfrom selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ECfrom selenium.webdriver.chrome.options import Options'''
try:element = WebDriverWait(browser, 10).until(EC.presence_of_element_located((By.ID, "myDynamicElement")))
finally:browser.quit()
''''''
chrome_options = Options()
chrome_options.add_experimental_option("debuggerAddress", "127.0.0.1:9222")
chrome_driver = "C:\Program Files\Google\Chrome\Application\chromedriver.exe"
driver = webdriver.Chrome(chrome_driver, chrome_options=chrome_options)
print(driver.title)
'''driver = webdriver.Chrome(r'E:\bbb\chromedriver.exe')targetUrl = 'https://www.pingshu8.com/MusicList/mmc_235_4615_1.Htm'
headUrl = 'https://www.pingshu8.com'
mp3name = ''session = HTMLSession()def openDriver(url):driver.get(url)   #打开浏览器.def getAllSession(url):    #当前页所有的链接print('It\'s comming getAllSession()!')downLink = []r = session.get(url)link = r.html.absolute_linksdownLink.append(link)  # 把所有的链接放入列表if downLink != None:return downLinkdef jump_link():    #获取第一页的跳转链接print('It\'s comming jump_link()!')for i in getAllSession(targetUrl):  # 这是页面跳转链接pattern = re.compile(r'\/Musiclist\/mmc_\w+\.htm')  # 编写正则表达式m = pattern.findall(str(i))  # 用正则表达式查出总共有多少页# print(m)'''通过split切割出最终的页码'''pagenum_list = []for n in m:pagename = re.split(r'\/Musiclist\/mmc_\d{2,3}_\d{0,}_', n, maxsplit=0, flags=0)  # 去掉页码前缀# print('pagename:',pagename)pagenum = pagename[1].split('.htm')  # 去掉页码后缀pagenum_list.append(pagenum)def end_num():  #最终的页码num1 = pagenum_list[0][0]num1 = int(num1)try:num2 = pagenum_list[1][0]num2 = int(num2)return num2except:num2 = 0return num2finally:        if num1 < num2:# print('num1:',num1,type(num1))return num2else:# print('num2:',num2,type(num2))return num1'''获取总页数之后，拼接每一页的跳转链接'''pagejump = []pages = 2# print(end_num())num = end_num()num = int(num)result1 = re.match('(\/Musiclist\/mmc_\d{2,3}_\d{0,}_)', n)t = result1.group(1)result2 = '.htm'while pages <= num:result = headUrl + t + str(pages) + result2pages += 1pagejump.append(result)# print('result:',result)if pagejump != None:return pagejumpdef down_link(url):    #每一页的下载链接print('It\'s comming down_link()!')save_down_link = []for i in getAllSession(url): #这是所有的链接pattern = re.compile(r'\/down_\d{1,6}\.html')    #编写正则表达式+#print('pattern:',pattern)re_link = pattern.findall(str(i)) #用正则表达式查出所有的down地址for j in re_link:links = f'{headUrl}'+j#print(links,type(links))#拼接完整的下载地址save_down_link.append(links)if save_down_link != None:return save_down_linkdef mp3Name(url):  #当前页所有的评书题目print('It\'s comming mp3Name()!')num = 5pstitle = []numlist = []while num < 33:numlist.append(num)num += 3maxfor = len(down_link(url))for nums in numlist[0:maxfor]:sel = driver.find_element_by_xpath(f'/html/body/div[2]/div[13]/div[1]/ul[2]/div[2]/ul/form/li[{nums}]/a').textpstitle.append(sel)        if pstitle != None:return pstitledef getAllLinks(url):  #打开每一页获取下载链接print('It\'s comming getAllLinks()!')ps_link_list = []ps_txt_list = []mylist = []ps_link_list.append(down_link(targetUrl))ps_txt_list.append(mp3Name(targetUrl))for i in jump_link():openDriver(i)  # 打开第二页result = driver.find_element_by_link_text('首页')if result != None:ps_link_list.append(down_link(i))ps_txt_list.append(mp3Name(i))else:time.sleep(50)ps_link_list.append(down_link(i))ps_txt_list.append(mp3Name(i))for links in ps_link_list:links.sort()print(ps_link_list)x = 10  #正常情况每页有10条y = len(ps_link_list)z = len(ps_link_list[-1])m = 0while m < y-1:    #正常遍历for n in range(x):mytext = ps_txt_list[m][n]mylink = ps_link_list[m][n]mylist.append((mytext, mylink))m += 1 if z < x:   #判断最后一页有没有10条，没有10条遍历10次会报溢出    l = 0while l < z:mytext = ps_txt_list[-1][l]mylink = ps_link_list[-1][l]mylist.append((mytext, mylink))l += 1if mylist != None:return mylist   def openXss(url,title):  #下载主方法'''计数 1/3 2/3global aa += 1print('\n',a,'/',lenxs)'''openDriver(url)time.sleep(1)handle = driver.current_window_handle  # 获取当前标签句柄'''获取文件名并赋值给filename'''
#     txt = driver.find_element_by_xpath('/html/body/div[6]/div[1]/div[3]/font').text
#     filename = txt.split('- 下载')[0]
#     print('filename:',filename)def name(title):name = titlereturn namebt = driver.find_element_by_id('clickina').click()handles = driver.window_handles  # 获取当前所有标签句柄for newHand in handles:  # 对标签进行遍历if newHand != handle:  # 筛选新打开的标签driver.switch_to.window(newHand)  # 切换到新打开的标签link = driver.current_url  # 获取当前页面地址print(link)driver.close()driver.switch_to.window(handles[0])myfile = requests.get(link)filename = name(title)open(f'e:\pydownload\\{filename}.mp3', 'wb').write(myfile.content)def Name():name = driver.find_element_by_xpath('/html/body/div[2]/div[12]/div/h1').textreturn namedef csv(list):print('It\'s comming csv()!')    global mp3namemp3name = Name()df = pd.DataFrame(list)df.columns = ['title','link']df.to_csv(f'{mp3name}.csv', encoding='gbk', index=False)def running(targetUrl):mylist = []ps_txt_list = []ps_link_list = []openDriver(targetUrl)def ex(): #判断有没有下一页try:sel = driver.find_element_by_xpath('/html/body/div[2]/div[13]/div[1]/ul[2]/div[3]/div/a[1]')return Trueexcept:return Falsedef z():try:            z = len(ps_link_list[-1])return zexcept:return x+1if ex():        jump_link()return csv(getAllLinks(targetUrl))else:x = 10  #正常情况每页有10条  m = 0ps_link_list.append(down_link(targetUrl))ps_txt_list.append(mp3Name(targetUrl))z = z()if z < x:   #判断最后一页有没有10条，没有10条遍历10次会报溢出    l = 0while l < z:mytext = ps_txt_list[-1][l]mylink = ps_link_list[-1][l]mylist.append((mytext, mylink))l += 1if mylist != None:return csv(mylist)running(targetUrl)
#mp3name = Name()data = pd.read_csv(f'{mp3name}.csv', encoding='GB18030')
content = data.values
lenxs = len(content)
a = 0for i in content:title = i[0]link = i[1]print(title,link)def jishu():'''计数 1/3 2/3'''global aa += 1print('\n',a,'/',lenxs)def down():try:for i in content:jishu()title = i[0]link = i[1]print(title, link)openXss(link, title)except:for i in content[a-1:]:jishu()title = i[0]link = i[1]print(title, link)openXss(link, title)finally:if a == lenxs:return 'OK!'else:for i in content[a-1:]:jishu()title = i[0]link = i[1]print(title, link)openXss(link, title)
down()

用python实现相声、评书自动下载（五）相关推荐

用python实现相声、评书自动下载（二）
接用python实现相声.评书自动下载(一). 之前的代码成功爬到了这个评书网址的郭德纲的相声,但是每次都要手动去切下载链接,挺麻烦的.而且这个网站评书资源比较少,而且时不时放不了,所以换了一个网站. ...
用python实现相声、评书自动下载（三）
在用python实现相声.评书自动下载(二)写了怎么爬所有的下载链接,那么在这篇写怎么下载. 其实代码和(一)差不多 from selenium import webdriver from reque ...
python 编写一段自动下载微信群的聊天记录中的图片并保存到本地的代码
以下是一段简单的 Python 代码,可以自动下载微信群聊天记录中的图片并保存到本地: import requestsdef download_image(url, file_name):respon ...
用python实现相声、评书自动下载（一）
由于最近开车听歌实在听腻了,就想听几段相声解解乏.但是各个音乐平台上相声实在是少,于是只能自己去网上找.这是最初版本的code,相当简陋,下载链接都是手动复制网页源代码,然后用正则切出来的. from ...
用python实现相声、评书自动下载（六）
用request_heml现在爬不到任何东西,应该是做了屏蔽.那么就要换个思路了,不用request请求网站,而且全部用selenium来操作.还是打开任意一个作品的第一页,然后获取网页源代码,通过源 ...
python自动下载安装软件_30行Python代码从百度自动下载图片(内附源码和exe程序)...
只需要30行代码就可以从百度自动下载图片大家好,我是行哥,一个专门教小学生撸Python的编程老师(小学生都能学会的编程) 这里行哥想问大家三个问题 : - 你还在为批量下载表情包发愁吗? - 你还 ...
Python实现在线版本检测自动下载更新
Python实现版本检测软件在线更新在线下载(Python simple implementation of software online update and download) 文章由生化 ...
用python实现相声、评书自动下载（四）
对到目前为止遇到的所有异常做处理. from selenium import webdriver from requests_html import HTMLSession import reques ...
python使用pip命令自动下载安装库
一.官网下载安装python.二.配置环境变量.path变量中加入pip.exe文件位置.E:\Python\Python38\Scripts三.在cmd窗口输入命令 pip install 库名 ...

用python实现相声、评书自动下载（五）

用python实现相声、评书自动下载（五）相关推荐

最新文章

热门文章