python爬虫脚本下载YouTube视频

python爬虫脚本下载YouTube视频

爬虫

python

YouTube视频

工作环境：

• python 2.7.13

• pip

• lxml, 安装 pip install lxml,主要用xpath查找节点，可以使用re模块代替

• pytube, 安装 pip install pytube

• ***工具

参考:

• pytube

• 利用Python爬取YouTube上的视频播放地址

• python实现读取命令行参数的方法

• Python函数中的局部变量和全局变量

• Python中网络页面抓取和页面分析

源码：

1. # coding: utf-8 
   

   ---

2. __author__ = "zwzhou" 
   

   ---

3. __date__ = "2017-03-19" 
   

   ---

4. ---

5. import urllib2 
   

   ---

6. from pytube import YouTube 
   

   ---

7. from pprint import pprint 
   

   ---

8. from lxml import etree 
   

   ---

9. import sys,getopt 
   

   ---

10. ---

11. def getHtml(url): 
    

    ---

12. user_agent='Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1284.0 Safari/537.13' 
    

    ---

13. headers={'User-Agent':user_agent} 
    

    ---

14. request=urllib2.Request(url,headers=headers) 
    

    ---

15. response=urllib2.urlopen(request) 
    

    ---

16. html=response.read() 
    

    ---

17. return html 
    

    ---

18. ---

19. def getUrl(html): 
    

    ---

20. global savepath 
    

    ---

21. global maxNumber 
    

    ---

22. global timeThreshold 
    

    ---

23. global cur_count 
    

    ---

24. global videoLists 
    

    ---

25. tree=etree.HTML(html) 
    

    ---

26. urllist=tree.xpath(u'//div[@class="thumb-wrapper"]/a/@href') 
    

    ---

27. #print urllist 
    

    ---

28. urllist_time=tree.xpath(u'//div[@class="thumb-wrapper"]/a/span/span/text()') 
    

    ---

29. ---

30. baseurl=r'https://www.youtube.com' 
    

    ---

31. for (item_name,item_length) in zip(urllist,urllist_time): 
    

    ---

32. #print item_name 
    

    ---

33. #print item_length 
    

    ---

34. try: 
    

    ---

35. yt = YouTube(baseurl+item_name) 
    

    ---

36. except: 
    

    ---

37. print "Some thing wrong about the authority" 
    

    ---

38. ---

39. print("video name:"+yt.filename) 
    

    ---

40. print("video time:"+item_length) 
    

    ---

41. if yt.filename in videoLists: # 文件已经存在 
    

    ---

42. print "This video has been downloaded!" 
    

    ---

43. else: 
    

    ---

44. if checktime(item_length): 
    

    ---

45. video = yt.filter('mp4')[-1] 
    

    ---

46. print("Now is loading %s------------>"%yt.filename) 
    

    ---

47. video.download(savepath) 
    

    ---

48. print("--------------->%sVideo is loaded!"%yt.filename) 
    

    ---

49. cur_count+=1 
    

    ---

50. videoLists.append(yt.filename) 
    

    ---

51. if cur_count >= maxNumber:# 达到要求 
    

    ---

52. print('There are %d videos downloaded!This task is completed!'%maxNumber) 
    

    ---

53. # TODO: if necessary, the videoLists can be logged 
    

    ---

54. sys.exit()  
    

    ---

55. else: 
    

    ---

56. print 'This video is too long and it will not be downloaded, just be ignored!' 
    

    ---

57. if urllist: 
    

    ---

58. getUrl(baseurl+urllist[0]) #下一个页面 
    

    ---

59. ---

60. ---

61. def checktime(timelength): 
    

    ---

62. global timeThreshold 
    

    ---

63. strs=timelength.split(':') 
    

    ---

64. time =int(strs[0])*60+int(strs[1]) 
    

    ---

65. if time< timeThreshold: 
    

    ---

66. return True 
    

    ---

67. else: 
    

    ---

68. return False 
    

    ---

69. ---

70. def usage(): 
    

    ---

71. print ''' 
    

    ---

72. usage: python dl_youtube [option] [arg] 
    

    ---

73. options and args: 
    

    ---

74. -s : download path 
    

    ---

75. -t : time threshold of the video to be loaded, in seconds 
    

    ---

76. -u : start url which to be crawled, it can be set more than one time 
    

    ---

77. -n : when downloading is stop, i.e. how many videos will be downloaded, default is 10000. 
    

    ---

78. -h : print this help message 
    

    ---

79. ''' 
    

    ---

80. ---

81. if __name__ == "__main__": 
    

    ---

82. start_urls=['https://www.youtube.com/watch?v=TThzH_sJo6o'] 
    

    ---

83. videoLists=[] # 保存文件名，防止重复下载 
    

    ---

84. # 初始值 
    

    ---

85. savepath=r"D://MyDownloads" 
    

    ---

86. maxNumber=10000 
    

    ---

87. timeThreshold=240 
    

    ---

88. cur_count=0 
    

    ---

89. ---

90. opts,args=getopt.getopt(sys.argv[1:],'hs:t:n:u:') 
    

    ---

91. for op,value in opts: 
    

    ---

92. if op == "-s": # 下载路径，如默认 D://MyDownloads 
    

    ---

93. savepath=value 
    

    ---

94. elif op == '-t': # 时常限制，默认240s 
    

    ---

95. timeThreshold =int(value) 
    

    ---

96. elif op == "-h": # help 
    

    ---

97. usage() 
    

    ---

98. sys.exit() 
    

    ---

99. elif op == '-n': 
    

    ---

100. maxNumber=int(value) 
     

     ---

101. elif op == '-u': # 初始的搜索链接 
     

     ---

102. start_urls.append(value) 
     

     ---

103. ---

104. for item in start_urls: 
     

     ---

105. html = getHtml(item) 
     

     ---

106. getUrl(html) 
     

     ---

107. ---

使用

1. python dl_youtube.py -n 10 -s D://MyDownloads -t 600 -u https://www.youtube.com/watch?v=TThzH_sJo6o 
   

   ---

将从页面 https://www.youtube.com/watch?v=TThzH_sJo6o 开始搜索下载10段时长小于6分钟的video保存到D://MyDownloads文件夹中。