Python3 多线程下载代码

根据http://www.oschina.net/code/snippet_70229_2407修改而来的增强版。貌似原版源自Axel这个多线程下载工具。

'''
Created on 2014-10-24

@author: Maple
'''

import sys
import os
import time
import getopt
import urllib.request
import urllib.parse
from threading import Thread

#===============================================================================
# def download(url, output=os.getcwd(), blocks=6, proxies=local_proxies)
# output:输出文件路径，默认为当前路径
# blocks:线程数
# proxies:代理地址
#===============================================================================

local_proxies = {}#代理地址

class Maple(Thread):
    version = "Mozilla/5.0"

    def __init__(self, threadname, url, filename, ranges=0, proxies={}):
        Thread.__init__(self, name=threadname)
        self.name = threadname
        self.url = url
        self.proxies = proxies
        self.filename = filename
        self.ranges = ranges
        self.downloaded = 0

    def run(self):

        try:
            self.downloaded = os.path.getsize( self.filename )        #获取已下载的文件字节块块，支持断点续传
        except OSError:
            #print 'never downloaded'
            self.downloaded = 0
        opener=GetUrlOpener(self.proxies)                                    #根据代理参数生成相应的url opener
        if self.ranges:                                                                         #ranges为线程需要下载的文件块的字节范围
            # rebuild start poind
            self.startpoint = self.ranges[0] + self.downloaded        #从已下载字节块后的位置开始下载

            # This part is completed
            if self.startpoint >= self.ranges[1]:
                self.downloaded = self.ranges[1] - self.ranges[0]
                print ('Part %s has been downloaded over.' % self.filename)
                return
            opener.addheaders=[('Range','bytes={}-{}'.format(self.startpoint, self.ranges[1])),('User-agent','Mozilla/5.0')]     #添加请求头部内容，仅下载指定范围的字节，伪装成浏览器请求
            print ('task %s will download from %d to %d' % (self.name, self.startpoint+1, self.ranges[1]+1))
        else:                                       #ranges未指定（文件大小未知，无法切割），从已下载字节块后的位置开始下载剩余全部字节
            self.startpoint = self.downloaded
            opener.addheaders=[('Range','bytes={}-'.format(self.startpoint)),('User-agent','Mozilla/5.0')]
        self.fetchsize = 16384                                                     #每次读取的字节数
        self.urlhandle = opener.open(self.url)                             #打开文件地址
        data = self.urlhandle.read( self.fetchsize )
        while data:                                                          #循环读取数据写入临时文件，并更新已下载字节数
            filehandle = open( self.filename, 'ab+' )
            filehandle.write( data )
            filehandle.close()
            self.downloaded += len( data )
            data = self.urlhandle.read( self.fetchsize )

def Sec2Time(second):                        #将秒数转换为标准时间格式。以为有现成的函数，结果愣是没找到
    day=second//(3600*24)
    second-=day*3600*24
    hour=second//3600
    second -=hour*3600
    minute=second//60
    second-=minute*60
    if day == 0:
        if hour == 0:
            if minute == 0:
                return '{:0.2f}S.'.format(second)
            else:
                return '{:02}M:{:0.2f}S'.format(minute,second)
        else:
            return '{:02}H:{:02}M:{:0.2f}S'.format(hour,minute,second)
    else:
        return '{:03}D:{:02}H:{:02}M:{:0.2f}S'.format(day,hour,minute,second)

def GetUrlOpener(proxies={}):         #分析代理参数，返回url opener。完整代理格式：user/passwd@http://127.0.0.1:8087。如格式不同，需要修改此分析函数
    if proxies:
        try:
            ap=proxies.split('@')
            if len(ap) > 1:
                auth=ap[0]
                addr=ap[1]
            else:
                addr=ap[0]
                auth=''
            if '://' in addr:
                ptype=addr[:addr.find('://')]
                phost=addr[addr.find('://')+3:]
            else:
                ptype='http'
                phost=addr
            proxy={ptype:ptype+'://'+phost}
            proxy_handler = urllib.request.ProxyHandler(proxy)
        except Exception as ex:
            print(ex)
            return urllib.request.build_opener()
        try:
            authlist=auth.split('/')
            if len(authlist) > 1:
                user=authlist[0]
                passwd=authlist[1]
                proxy_auth_handler = urllib.request.ProxyBasicAuthHandler()
                proxy_auth_handler.add_password('realm',phost,user,passwd)
                opener = urllib.request.build_opener(proxy_handler,proxy_auth_handler)
            else:
                opener = urllib.request.build_opener(proxy_handler)
            return opener
        except Exception as ex:
            print(ex)
            return urllib.request.build_opener(proxy_handler)
    else:
       # urlHandler=urllib.request.urlopen(url)
        return urllib.request.build_opener()

def GetUrlFileInfo(url,proxies={}):            #获取要下载的文件的信息，包括文件名，文件类型和文件大小
    scheme, netloc, path, query, fragment = urllib.parse.urlsplit(url)     #分析url
    filename=urllib.parse.unquote(path)       #如果url中的文件名部分存在中文，将其正确解码出来
    filename=filename.split('/')[-1]
    opener=GetUrlOpener(proxies)          #通过网络请求读取响应头部，根据头部获取文件信息。文件名以服务器返回的文件名信息为准
    urlHandler=opener.open(url)
    headers=urlHandler.info()
    if 'Content-Disposition' in headers:       #Content-Disposition字段有可能获取到文件名，不过可能是乱码，没找到解决办法</span>
        disposition=headers.get('Content-Disposition')
        if 'filename=' in disposition:
            filename = disposition.split('filename=')[1]
            if filename[0] == '"' or filename[0] == "'":
                filename = filename[1:-1]
    filename=urllib.parse.unquote(filename)
    if filename:
        (name,ext)=os.path.splitext(filename)
    else:
        (name,ext)=('Unknown','')
    if 'Content-Length' in headers:          #获取文件长度，如果获取失败，则只能使用单线程下载
        length=int(headers.get('Content-Length'))
    else:
        length=-1
    (type, kind)=headers.get('Content-Type').split('/') #获取文件类型，备用
    infos=[(name,ext),(type,kind),length]
    return infos

def SpliteBlocks(totalsize, blocknumber):   #根据指定的线程数参数和获取到的文件长度划分各线程的下载范围
    blocksize = totalsize//blocknumber
    ranges = []
    for i in range(0, blocknumber-1):
        ranges.append((i*blocksize, i*blocksize +blocksize - 1))
    ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))

    return ranges
def islive(tasks):                        #检查各线程是否全部下载完成
    for task in tasks:
        if task.isAlive():
            return True
    return False

def download(url, target=os.getcwd(), blocks=6, proxies=local_proxies):               

    flag=True
    print('Retrieving resource information...')

    url=urllib.parse.quote(url,safe='/%&@=+?$;,:')        #将提供的url编码，非英文字符将被编码为标准格式
    try:
        infos=GetUrlFileInfo(url,proxies)                            #获取文件信息
    except Exception as ex:
        print(ex)
        flag=False
    if flag:
        if not os.path.exists(target):
            os.makedirs(target)
        size=infos[2]                                                              #获取到的文件大小
        output=os.path.join(target,''.join(infos[0]))         #根据获取到的文件名和指定的保存目录生成完整路径
        type=infos[1][0]

        starttime=time.time()                                              #开始计时
        print('Infomation:')
        print('FileName:{0}     FileType:{1}    FileLength:{2}'.format(''.join(infos[0]),'/'.join(infos[1]),infos[2] if int(infos[2]) > 0 else 'Unknown'))   #打印获取到的文件信息
        if size > 0:                                                                               #size大于0表示成功获取文件长度，可以进行多线程下载
            print('Starting multithread download...')
            ranges = SpliteBlocks( size, blocks )
        else:                                                                                         #只能单线程下载，线程数置1，ranges置空，
            print('Starting single thread download...')
            ranges=()
            blocks=1
        threadname = [ infos[0][0]+"_thread_%d" % i for i in range(0, blocks) ]         #生成线程名
        filename = [ infos[0][0]+ "_tmpfile_%d" % i for i in range(0, blocks) ]             #生成各线程的临时文件名
        tasks = []
        for i in range(0,blocks):                                                                                        #生成下载线程，设置为后台线程后启动，将线程加入到线程列表中
            task = Maple( threadname[i], url, filename[i], ranges[i] if ranges else ranges,proxies)
            task.setDaemon( True )
            task.start()
            tasks.append( task )

        time.sleep( 1 )
        downloaded = 0
        while islive(tasks):                                                                                                 #统计线程列表中各线程的状态，输出下载进度
            downloaded = sum( [task.downloaded for task in tasks] )
            if size > 0:
                process = downloaded/float(size)*100
                show = '\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
            else:
                show = '\rDownloaded:%d ' % downloaded
            sys.stdout.write(show)
            sys.stdout.flush()
            time.sleep( 0.2 )
        endtime=time.time()                                                                                           #下载完成后停止计时
        consuming=Sec2Time(endtime-starttime)
        if size > 0:                                                                                                             #多线程下载的后续处理
            downloadsize = 0
            for i in filename:
                downloadsize += os.path.getsize(i)
            if downloadsize == size:
                show = '\rFilesize:%d Downloaded:%d Completed:%.2f%%\n' % (size, downloadsize,100)
            else:
                show = '\nSize is not mathed!\n'
                flag=False
        else:                                                                                                                   #单线程下载的后续处理
            show = '\nTotal Size: %d\n'% downloaded
        sys.stdout.write(show)
        sys.stdout.flush()
        if flag:                                                                                                                #确认下载的临时文件没问题后将各文件整合为最终的目标文件
            print('Integrating files...')
            num=1
            while os.path.exists(output):                                                                      #防止与本地已存在文件重名
                fname,fext=os.path.splitext(output)
                if '('+str(num-1)+')'+fext in output:
                    output = output.replace('('+str(num-1)+')'+fext,'('+str(num)+')'+fext)
                else:
                    fname += '('+str(num)+')'
                    output = fname+fext
                num +=1
            if len(filename) ==1 :                                                                                  #单线程下载的话，直接将下载的文件重命名为目标文件即可
                os.rename(filename[0], output)
            else:                                                                                                             #多线程临时文件整合
                filehandle = open( output, 'wb+' )
                for i in filename:
                    try:
                        f = open( i, 'rb' )
                        filehandle.write( f.read() )
                        f.close()
                        os.remove(i)
                    except Exception as ex:
                        print(ex)
                filehandle.close()
            if os.path.exists(output):
                print('Download Complete!')
            else:
                print('Failed to generate target file!')
                try:
                    #os.remove(output)
                    pass
                except:
                    pass
        else:
            for i in filename:
                try:
                    os.remove(i)
                    pass
                except:
                    pass
            print('Download Failed!')
            pass
        print('Consuming: {}\n'.format(consuming))                        #输出耗时
    else:
        print('Failed to retrieve resource information!')
        sys.exit()
def main(argv):                                                                                #处理传入参数，使用了getopt模块，另外有一个更强大的处理传入参数的模块optparse
    try:
        options,args=getopt.getopt(argv,'hu:f:n:p:',['help','url=','target=','num=','proxy='])
    except Exception as ex:
        print(ex)
        sys.exit()
    num = 2
    url,target,proxies= '','',''

    url = 'http://www.pygtk.org/dist/pygtk2-tut.pdf'
    target = '/home/maple/Desktop'
    #proxies = 'http://127.0.0.1:8087'
    #proxies={}
    for name, value in options:
        if name in ('-h','--help'):
            print('No Help ^^')
            sys.exit()
        if name in ('-u','--url'):
            url = value
        if name in ('-t','--target'):
            target = value
        if name in ('-n','--num'):
            num = int(value)
        if name in ('-p','--proxy'):
            proxies=value
    #check args
    download(url,target,num,proxies)

if __name__ == '__main__':
    main(sys.argv[1:])

这段代码在异常处理方面写得有些乱，没怎么关心异常处理，需要时再改吧

另外多线程下载时，如果使用了代理，会导致下载到的文件与服务器提供的文件大小不符。从而下载失败。我使用的是GoAgent，代理服务器会自动对目标文件进行多线程下载，无视程序指定的下载字节范围。第一个线程就会下载到完整的文件，其他线程会下载冗余内容。没有找到规范的处理办法。变通的处理办法有2种：

1、将使用了代理的多线程下载强制指定为单线程下载

2、不进行文件大小的校验，将多线程下载的0号临时文件重命名为目标文件，其他临时文件删除。

两种方法实现都很简单，但是破坏代码的整体逻辑。没有加入代码中。运行截图：