#!/usr/bin/env python# -*- coding :utf-8 -*-
"""It is a multi-thread downloading tool
"""import sys
import os
import time
import urllib2
import urllib
from threading import Threadclass MyWorkThread(Thread, urllib.FancyURLopener):"""Multi-thread downloading class.run() is a vitual method of Thread"""def __init__(self, threadname, url, filename, ranges = 0):Thread.__init__(self, name = threadname)urllib.FancyURLopener.__init__(self)self.name = threadnameself.url = urlself.filename = filenameself.ranges = rangesself.downloaded = 0def run(self):"""virtual function in Thread"""try:self.downloaded = os.path.getsize(self.filename)except OSError:self.downloaded = 0#rebuild start pointself.startpoint = self.ranges[0] + self.downloaded#if this part is completedif self.startpoint >= self.ranges[1]:print 'Part %s has been downloaded over.' % self.filenamereturnself.oneTimeSize = 8 * 1024 #8K bytes / timeprint 'task %s will download from %d to %d' %(self.name, self.startpoint, self.ranges[1])self.addheader('Range', 'bytes=%d-%d' %(self.startpoint, self.ranges[1]))self.urlhandle = self.open(self.url)data = self.urlhandle.read(self.oneTimeSize)while data:filehandle = open(self.filename, 'ab+')filehandle.write(data)filehandle.close()self.downloaded += len(data)data = self.urlhandle.read(self.oneTimeSize)def GetUrlFileSize(url):urlHandler = urllib.urlopen(url)headers = urlHandler.info().headerslength = 0for header in headers:if header.find('Length') != -1:length = header.split(':')[-1].strip()length = int(length)return length
def SpliteBlocks(totalsize, blocknumber):blocksize = totalsize / blocknumberranges = []for i in range(0, blocknumber -1):ranges.append((i * blocksize, i * blocksize + blocksize -1))ranges.append((blocksize * (blocknumber -1), totalsize -1))return ranges
def isLive(tasks):for task in tasks:if task.isAlive():return Truereturn False
def downLoadFile(url, output, blocks = 6):sys.stdout.write('Begin to download from %s\n' %url )sys.stdout.flush()size = GetUrlFileSize(url)ranges = SpliteBlocks(size, blocks)threadname = ["thread_%d" %i for i in range(0, blocks)]filename = ["tmpfile_%d" %i for i in range(0, blocks)]tasks = []for i in range(0, blocks):task = MyWorkThread(threadname[i], url, filename[i], ranges[i])task.setDaemon(True)task.start()tasks.append(task)time.sleep(2)while isLive(tasks):downloaded = sum([task.downloaded for task in tasks])process = downloaded / float(size) * 100show = u'\rFilesize: %d Downloaded:%d Completed: %.2f%%' %(size, downloaded, process)sys.stdout.write(show)sys.stdout.flushtime.sleep(1)output = formatFileName(output)filehandle = open(output, 'wb+')for i in filename:f = open(i, 'rb')filehandle.write(f.read())f.close()os.remove(i)filehandle.close()sys.stdout.write("Completed!\n")sys.stdout.flush()def formatFileName(filename):if isinstance(filename, str):header, tail = os.path.split(filename)if tail != '':tuple = ('\\','/',':', '*', '?', '"', '<', '>', '|')for char in tuple:if tail.find(char) != -1:tail = tail.replace(char, '')filename = os.path.join(header, tail)#print filenamereturn filenameelse:return 'None'if __name__ == '__main__':url = r'http://www.51voa.com/path.asp?url=/201008/hennessy_africa_wildlife_18aug10-32b.mp3'output = r"D:\Voa\Study:'Shoot to Kill' Policy in Africa's Parks Abuses Human Rights.mp3"downLoadFile(url, output, blocks = 4)

#!/usr/bin/env python# -*- coding:utf-8 -*-import urllib2import chardetimport osimport timeimport stringimport refrom HTMLParser import HTMLParserimport sysfrom BeautifulSoup import BeautifulSoupimport multiThreadDownloadTool

VOA_URL = r'http://www.51voa.com'DOWNLOAD_DIR = r'D:/Voa'

"""File downloading from the web."""

def getURLContent(url):    """    get url content of the url, begin with html and ignor the doctype declarations    """    file = urllib2.urlopen(url)    #print file.info()    data = file.read()    file.close()    #return data.decode('utf-8')    index = data.find('html')    data = data[index - 1 :]    return datadef getVOAURLs(content):    """    find the voa script urls in the content    """    urls = {}    soup = BeautifulSoup(content)    divs = soup.findAll('div', {'id':'rightContainer'})    #print divs    neededDiv = None    if len(divs) >= 1:        neededDiv = divs[0]    if neededDiv != None:        #pass the div        #print neededDiv        neededSpan = neededDiv.find('span', {'id' : 'list'})        #print neededSpan        lis = neededSpan.findAll('li')        #print lis        for li in lis:            needAs = li.findAll('a')            #got it            #print needAs[1]            #print needAs[1]['href']            #print needAs[-1].string            urls[needAs[-1].string] = VOA_URL + needAs[-1]['href']    print "getVOAURLs() urls count is " , len(urls)    return urlsdef filterbyDate(urls ,date):    """    filter the urls by date    """    neededURLs = {}    currentDate = time.localtime(time.time());    #currentDateStr = time.strftime('%Y-%m-%d', currentDate)    #currentDateStr = currentDate.tm_year + '-' + currentDate.tm_mon + ' ' + currentDate.tm_mday    currentDateStr = "%s-%s-%s"  %(currentDate.tm_year, currentDate.tm_mon, currentDate.tm_mday)    if(date != None):        currentDateStr = date    for url in urls.keys():        name = url.lstrip().rstrip()        length = len(name)        publishDate = name[- len(currentDateStr) - 1 : -1]        #print publishDate        if publishDate == currentDateStr:            neededURLs[name] = urls[url]            print 'find ', name

    print 'After filter, the count is ' , len(neededURLs)    return neededURLs

def findMP3FileInURL(url):    """    find MP3 files in a url    """    print 'parse the content of ', url    urls = []    #define a MP3 re string    p = re.compile(r'/path.asp\?url=[-\w/]*\.mp3')    #p = re.compile(r'/[-\w/]*\.mp3')    content = getURLContent(url)    matchLinks = p.findall(content)    #print matchLinks    for link in matchLinks:        tmp = VOA_URL + link        if tmp in urls: # check if exist already            pass        else:            urls.append(tmp)    print 'Current count of mp3 files is ', len(urls)    return urlsdef getHTMLFile(url, file_name):    ifile = urllib2.urlopen(url)    content = ifile.read()    local_file = open(file_name, 'w')    local_file.write(content)    local_file.close()

def downloadFile(url, fileName2Store):    """    download file from url, and store it to local system using fileName2Store parameter    """    try:        full_path = os.path.join(DOWNLOAD_DIR, fileName2Store)        print 'begin to download url to ', full_path        if os.path.isfile(full_path):            #already exist            print 'the file ', full_path, 'is alreasy exist, so just skip it!'        else:            print '\tDownloading the mp3 file...',              data=urllib2.urlopen(url).read()              print 'Done'              print '\tWriting data info file...',              f=file(full_path, 'wb')              f.write(data)              print 'Done'              f.close()    except Exception, ex:        print 'some exceptions occur when downloading ', exif __name__ == "__main__":    try:        #getHTMLFile(VOA_URL, r'.\Voa.html')        context = getURLContent(VOA_URL)        #file_read = open(r'.\Voa.html', 'r')        #context = file_read.read()        #print context        #print '\n' * 5#print chardet.detect(context)print 'Begin to get download information, it may cost some minuts, please wait...'files2download = getVOAURLs(context)neededDownload = filterbyDate(files2download, None)neededDownloadMp3s = {}for name in neededDownload.keys():fullURL = neededDownload[name]formatedName = name[: -11].lstrip().rstrip()#formatedName = formatedName.replace(' ', '-')#print formatedName, ' ' * 5, fullURL#print fullURLmp3Names = findMP3FileInURL(fullURL)if len(mp3Names) == 1:#there is only on mp3 file in this file ,so we will use the formatednameneededDownloadMp3s[formatedName] = mp3Names[0]else:for name in mp3Names:print nameindex_begin = name.rfind('/')index_end = name.rfind('.')tmpName = name[index_begin + 1 : index_end]neededDownloadMp3s[tmpName] = nameprint 'Now , the mp3 files  are :'print neededDownloadMp3s#findMP3FileInURL(r'http://www.51voa.com/VOA_Special_English/Phoning-Fertilizer-Philippine-Rice-Farmers--38545.html')#findMP3FileInURL(r'http://www.51voa.com/Voa_English_Learning/Learn_A_Word_38412.html')#down load filefor filename in neededDownloadMp3s.keys():try:full_path = os.path.join(DOWNLOAD_DIR, filename)full_path = full_path + r'.mp3'if full_path == r'D:\Voa\hennessy_africa_wildlife_18aug10-32b.mp3':multiThreadDownloadTool.downLoadFile(neededDownloadMp3s[filename], full_path)except Exception, ex:print 'Some exceptions occur, when downloading file from %s, exception messages are %s' %(neededDownloadMp3s[filename], ex)#downloadFile(r'http://www.51voa.com/path.asp?url=/201008/mercer_australia_election_16aug10-32b.mp3', 'test.mp3')except Exception, ex:print 'Exception caught, tracebacks are :',sys.exc_info(), exprint 'download all completed!'raw_input("Press any key to continue...")