python下载网页视频

因网站不同需要修改。

下载 mp4 连接

from bs4 import BeautifulSoup
import requests
import urllib
import re
import json
encodestyle = 'gbk'
homepage='http://www.**.html'
htmlhead='http://www.**'  # GetwVideoHtml() 函数用

#GetNPage_html(homepage,n)
#HtmlList2Mp4List(sumhtml)
#Writelist2json(listname,lists)

def GetwVideoHtml(furl):
    retlist=[]
    res = requests.get(furl)
    res.encoding= encodestyle
    soup = BeautifulSoup(res.text,'html.parser')
    for Tag_contentpage in soup.select('.video_box'):   #<div class= video_box>
        for tag_a in Tag_contentpage.select('a'):       #<a href = 'http-html' target='_blank'>
            httphtml=tag_a['href']
            retlist.append(htmlhead+httphtml)  # use htmlhead
            #print(imgsrc)
    return retlist
def GetNPage_html(homepage,n):
    rethtml=[]
    for num in range(1,n+1):
        if num == 1:
            homewebpage=homepage
        else:
            homewebpage= homepage.rsplit('.',1)[0] + '_'+ str(num) + '.html'
        print(homewebpage)
        htmllinks = GetwVideoHtml(homewebpage)
        rethtml = rethtml + htmllinks
    return rethtml

def GetMp4SrcFromHtml(url):
    headers = ("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36")
    opener = urllib.request.build_opener()
    opener.addheaders = [headers]
    urllib.request.install_opener(opener)
    file = urllib.request.urlopen(url).read()
    file = file.decode('gbk')
    pattern = re.compile(r'(https?://.*.mp4)', re.I)
    videolinks = pattern.findall(file)
    videolinks = list(set(videolinks))
    return videolinks
def HtmlList2Mp4List(sumhtml):
    retmp4s = []
    for html in sumhtml:
        mp4s = GetMp4SrcFromHtml(html)
        for mp4 in mp4s:
            retmp4s.append(mp4)
    return retmp4s
def Writelist2json(listname,lists):
    length = str(len(lists))
    with open('D:/ipynb/commfile/'+ listname + '_len_'+length +'.json', 'w') as fw:
        json.dump(lists, fw)

sumhtml = GetNPage_html(homepage,3)
mp4list = HtmlList2Mp4List(sumhtml)
Writelist2json("mp4list",mp4list)

下载部分

from bs4 import BeautifulSoup
import requests
import urllib
import json
import threading
import datetime
import os

def mkdir(path):
    folder = os.path.exists(path)
    if not folder:                   #判断是否存在文件夹如果不存在则创建为文件夹
        os.makedirs(path)            #makedirs 创建文件时如果路径不存在会创建这个路径
        print ("---  new folder...  ---")
        print ("---  OK  ---")
    else:
        print ("---  There is this folder!  ---")

def Schedule(a,b,c):
    '''
    回调函数：用于显示下载进度
    a:已经下载的数据块
    b:数据块的大小
    c:远程文件的大小
   '''
    per = 100.0 * a * b / c
    if (per > 100) :
        per = 100
    print ('%.2f%%' % per)
def createdownloadlink(name,url):
    urllib.request.urlretrieve(url,name)

class myThread (threading.Thread):
    def __init__(self, name, url):
        threading.Thread.__init__(self) # 线程初始化
        self.name = name  # 赋值成员变量
        self.url = url
    def run(self):
        print ("开始下载：" + self.name)
        urllib.request.urlretrieve(self.url,self.name)
        #createdownloadlink(self.name, self.url)  # 在线程中运行的函数
        print ("完成下载：" + self.name)
def DownMp4file(lists):
    dateASfolder=datetime.datetime.now().strftime('%m-%d')
    foldername = 'D:/videos/'+dateASfolder
    mkdir( foldername)
    threadlist = [];#存放线程的数组，相当于线程池
    filenum=0
    for url in lists:
        filename = foldername + '/'+ str(filenum)+ '.mp4'
        filenum=filenum+1
        thread = myThread(filename, url)  # 创建线程对象
        threadlist.append(thread)        #这个线程放到线程threads
    return threadlist
# 执行部分

with open('D:/ipynb/commfile/srcmp4s_len_66.json', 'r') as fr:
    srcmp4s = json.load(fr)

print(len(srcmp4s))
srcmp4s[0]

threads= DownMp4file(srcmp4s)

for t in threads[:10]:#让线程池中的所有数组开始
    t.start();
for t in threads[:10]:
    t.join();#等待所有线程运行完毕才执行一下的代码