Python 批量文件下载

python文件 批量下载 、图片批量下载 ，批量请求、爬虫

#====================================================================================

downloadFile.py

#====================================================================================

#!/usr/bin/env python
# -*- coding:utf-8 -*-
 
from gevent import monkey
monkey.patch_all()
from gevent.pool import Pool
import requests
import sys
import os

def download(url):
    chrome = 'Mozilla/5.0 (X11; Linux i86_64) AppleWebKit/537.36 ' +'(KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36'
    headers = {'User-Agent': chrome}
    # filename = url.split('/')[-1].strip()
    filename = "respose.log"
    r = requests.get(url.strip(), headers=headers, stream=True)
    with open(filename, 'a+') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:
                f.write(chunk)
        f.flush()
        print filename,"is ok"

def removeLine(key, filename):
    os.system('sed -i /%s/d %s' % (key, filename))

if __name__ =="__main__":
    if len(sys.argv) == 2:
        filename = sys.argv[1]
        f = open(filename,"r")
        p = Pool(4)
        for line in f.readlines():
            if line:
                p.spawn(download, line.strip())
                key = line.split('/')[-1].strip()
                #removeLine(key, filename)
                f.close()
                p.join()
    else:
        print 'Usage: python %s urls.txt' % sys.argv[0]

#====================================================================================

#====================================================================================

测试文件 url.txt

#===========================

http://download2.boulder.ibm.com/sar/CMA/RAA/075lj/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075ln/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075lt/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075m7/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075m9/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075mb/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075mf/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075mn/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075ms/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075mv/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075nd/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075nk/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075no/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075nr/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075ns/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075nu/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075ny/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075o0/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075o1/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075p8/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075px/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075py/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075pz/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075q1/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075q3/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075q5/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/075zm/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/0758i/0/
http://download2.boulder.ibm.com/sar/CMA/RAA/0759s/0/

#=================================================

运行结果

G:\test\appscan>python downloadFile.py url.txt

respose.log is ok
respose.log is ok
respose.log is ok
respose.log is ok
respose.log is ok
respose.log is ok
respose.log is ok
respose.log is ok
respose.log is ok

#================================

结果文件