#!/usr/bin/env pythonimport os
import sys
import re
import time
import chardet
import urllib.request as ur
from urllib.parse import urljoin,urlparse
from bs4 import BeautifulSoup
from threading import Threadclass Download(Thread):                          #为每个章节分配多线程def __init__(self,filepath,info):Thread.__init__(self)self.filepath = filepath(self.link,self.chapter) = infodef run(self):print('开始下载： '+self.chapter)section(self.filepath,self.chapter,self.link)print('完成下载： '+self.chapter)def getData(url):                          #主要用于判断页面编码，但是发现BeautifulSoup自带判定能力，故废弃此函数charsets = 'utf8'response = ur.urlopen(url,timeout = 10)html = response.read()charinfo = chardet.detect(html)charsets = charinfo['encoding']data = html.decode(charsets)return datadef merge(tmpFiles,targetFile):             #将下载的章节合并for tmpFile in tmpFiles:with open(targetFile,'a+') as wfile:wfile.write(open(tmpFile,'r').read())os.remove(tmpFile)def content(link):                         #获取章节页面的小说内容。对于不同的站点，在此函数内修改获取章节内容的代码html = ur.urlopen(link,timeout = 10)soup =BeautifulSoup(html)contents = soup.find(id = 'readtext').p.span.text.replace('  ','\n')   #BeautifulSoup会自动将&nbsp;转换为空格，<br/>转换为特殊符号return contentsdef section(filepath,chapter,link):         #下载章节内容while True:                #反复请求页面try:with open(filepath,'w') as nfile:nfile.write(chapter+'\n'+content(link)+'\n')breakexcept:passdef index(url):                         #获取章节索引indexs = []while True:                   #反复请求页面try:html = ur.urlopen(url,timeout = 10)#html = html.read().decode('gb2312')#html = getData(url)soup = BeautifulSoup(html,from_encoding = 'gbk')#BeautifulSoup能自动识别编码，但是会将gbk页面识别为gb2312页面，可能导致页面内部分数据获取失败，故显式指定breakexcept:passtitle = soup.find(name = 'div',attrs = {'class':'booktext'}).textindexDiv = soup.find(name = 'div',attrs = {'class':'booktext'})indexUl = [ul for ul in indexDiv.find_all('ul') if ul][1:]for ul in indexUl:indexList = [li.a for li in ul.find_all('li') if li]index = [(urljoin(url,a.get('href')),a.text) for a in indexList if a]indexs +=indexreturn indexsdef novel(url):tmpFiles = []tasks = []try:indexs = index(url)tmpDir = os.path.join(os.getcwd(),'tmp')if not os.path.exists(tmpDir):             #创建章节片段存放的临时目录os.mkdir(tmpDir)for i,info in enumerate(indexs):tmpFile = os.path.join(tmpDir,str(i))tmpFiles.append(tmpFile)task = Download(tmpFile,info)            #开启新线程下载章节内容task.setDaemon(True)task.start()tasks.append(task)if len(tasks) >= 20:                  #将线程总数控制在20个以内，如果线程过多会导致程序崩溃while len([task for task in tasks if task.isAlive()]):print( '进度： {} / {}'.format(i+1-len([task for task in tasks if task.isAlive()]),len(indexs)))  #显示下载进度time.sleep(2)tasks = []if i == len(indexs) - 1:while len([task for task in tasks if task.isAlive()]):print( '进度： {} / {}'.format(len(indexs) - len([task for task in tasks if task.isAlive()]),len(indexs)))time.sleep(2)print( '进度： {} / {}'.format(len(indexs),len(indexs)))print('开始整合......')merge(tmpFiles,os.path.join(os.getcwd(),title+'.txt'))print('下载成功！')except Exception as ex:print(ex)print('下载失败！')sys.exit()
def main(argv):try:novel(argv[0])except KeyboardInterrupt as kbi:            #使用<C-c>中断下载后仍然能将已下载的章节合并tmpDir = os.path.join(os.getcwd(),'tmp')if os.path.exists(tmpDir):tmpFiles = [os.path.join(tmpDir,tfile) for tfile in os.listdir(tmpDir) if os.path.isfile(os.path.join(tmpDir,tfile))]print('开始整合不完整的下载......')try:merge(tmpFiles,os.path.join(os.getcwd(),'不完整文档.txt'))if os.path.exists(os.path.join(os.getcwd(),'不完整文档.txt')):print('部分章节下载成功！')else:print('下载失败！')except:print('下载失败！')sys.exit()os.rmdir(tmpDir)else:print('下载失败！')sys.exit()if os.path.exists(os.path.join(os.getcwd(),'tmp')):    os.rmdir(os.path.join(os.getcwd(),'tmp'))if __name__ == "__main__":if len(sys.argv) > 1:main(sys.argv[1:])#http://www.lueqiu.com/