利用Python爬取电影网站

#!/usr/bin/env python
#coding = utf-8
'''
本爬虫是用来爬取6V电影网站上的电影资源的一个小脚本程序，爬取到的电影链接会通过网页的形式显示出来
'''
import requests
import re
from bs4 import BeautifulSoup as bs
from queue import Queue
from other import getUser_Agent
import threading
import sys
import time

headers = getUser_Agent.getUser_Agent()

class Movielinks(threading.Thread):

    def __init__(self,que,filepath,totalcount,starttime):
        threading.Thread.__init__(self)
        self._que = que
        self._filepath = filepath
        self._totalcount = totalcount
        self._starttime = starttime

    def run(self):
        try:
            while not self._que.empty():
                url = self._que.get()
                threading.Thread(target=self.showdetail).start()
                self.spider(url)
        except:
            print('error--->def run(self):')

    def spider(self,url):
        try:
            r = requests.get(url,headers)
            file = open(self._filepath,'a+')
            if r.status_code == 200:
                soup = bs(r.content.decode('gbk'),'html.parser')
                links = soup.find_all('td')
                title = re.search(r'《(.*?)》',soup.title.string)

                # print(title.group())

                for link in links:
                    if '下载帮助' not in str(link):
                        hrefs = link.find_all('a')
                        # print(link)
                        # print('**' * 100)
                        # mima = re.search(r'</a>(.*?)</td>',str(link))
                        # print(mima.group())
                        for href in hrefs:
                            if 'ed2k' in href['href'] or '.torrent' in href['href'] or 'thunder' in href['href'] or 'ftp' in href['href']:
                                # sys.stdout.write('\r'+'\t\t迅雷下载链接：%s'%href['href'])
                                file.write('<li style="list-style-type: decimal;">迅雷下载链接：<a href="%s" target="%s">%s</a></li>' % (href['href'],'_blank',title.group()))
                                file.write('\n<br>')
                            elif 'baidu' in href['href'] and re.search(r'密码：\w{4}',str(link)):
                                wangpan_password = re.search(r'密码：\w{4}',str(link))
                                # sys.stdout.write('\r'+'\t\t百度网盘下载链接：%s    网盘%s'%(href['href'],wangpan_password.group()))
                                file.write('<li style="list-style-type: decimal;">百度网盘链接：<a href="%s" target="%s">%s</a><b>%s</b></li>' % (href['href'], '_blank', title.group(),wangpan_password.group()))
                                file.write('\n<br>')
                            elif '正版观看' in str(href):
                                # print('\t\t %s  该影片需要正版观看！！' % title.group())
                                file.write('<b><font color="#00ff00">%s需要正版版权才能观看</font></b>' % title.group())
                                file.write('\n<br>')
                            else:
                                file.write('<b><font color="#ff0000">%s 的该条链接无法正常爬取，尽情谅解</font></b>' % title.group())
                                file.write('\n<br>')
                                # sys.stdout.write('\r'+'\t\terror--->def spider(self,url)-in:%s'%title.group())
                # print()
                file.write('<br>')
                file.close()
            else:
                print('%s 该磁力链接已坏！！' % url)
        except:
            # print('error--->def spider(self,url)-out:%s' % title.group())
            file.write('<b><font color="#ff0000">%s 无法正常爬取，尽情谅解</font></b>' % title.group())

    def showdetail(self):
        usetime = time.time() - self._starttime
        per = 100 - (float(self._que.qsize())/float(self._totalcount)) * 100
        sys.stdout.write('\r'+'下载链接进度：%.2f %s  用时：%.3f 秒' % (float(per),'%',float(usetime)))

def getMovieCount(searchid):
    url = 'http://www.6vhao.tv/e/search/result/?searchid=' + str(searchid)
    r = requests.get(url,headers)
    soup = bs(r.content,'html.parser')
    divs = soup.find(name='div',attrs='channellist')
    pages = re.search(r'\d+',divs.h2.string)

    print('%s 一共有%2.f页'%(divs.h2.string,float(pages.group())/20))

def getWantPagesUrls(url,startpage,endpage,searchid):
    urls = []
    for i in range(startpage,endpage+1):
        link = 'http://www.6vhao.tv/e/search/result/index.php?page='+str(i)+'&searchid='+str(searchid)
        urls.append(link)

    return urls

def getpagesLinks(urls): # 返回一个电影名字和链接对应的字典
    pageslink_dic = {}
    moviename_list = []
    moviehref_list = []

    for url in urls:
        r = requests.get(url,headers)
        soup = bs(r.content,'html.parser')
        divs = soup.find_all('div',class_='listimg')
        for div in divs:
            moviehref = div.find('a')['href']
            moviename = div.find('a').img['alt']
            moviehref_list.append(moviehref)
            # moviename_list.append(moviename)
    # for i,name in zip(range(len(moviename_list)),moviename_list):
    #     pageslink_dic[name] = moviehref_list[i]
    # print(moviehref_list)

    return moviehref_list

def getqueue(urls):
    que = Queue()

    for url in urls:
        que.put(url)

    return que

def main():
    tishi = '''
           *6v电影网站电影链接爬虫*
       #要搜索的常用关键字代码#
           1.韩国--->185773
           2.日本--->185691
           3.国产--->186504
           4.美国--->187181
           5.英国--->188161
           6.香港--->188461
           7.喜剧--->185441
           8.恐怖--->187193
           9.悬疑--->190226
           10.记录片--->187963
           11.科幻--->189866
           12.战争--->187830
           13.动画--->187978

       '''
    print(tishi)

    keywords = {1:185773,2:185691,3:186504,4:187181,5:188161,6:188461,7:185441,8:187193,9:190226,10:187963,11:189866,12:187830,13:187978}

    threads = []
    url = 'http://www.6vhao.tv/e/search/result/index.php?page='

    searchid = input('请输入搜索关键字序号>>>')
    print('正在计算电影总量... ...')
    getMovieCount(keywords[int(searchid)])
    startpage = int(input('请输入起始页码>>>'))
    endpage = int(input('请输入终止页码>>>'))
    filepath = input('请输入电影链接要保存的文件路径（扩展名是：.html）')

    print('The program is running,Please waiting... ...')

    urls = getpagesLinks(getWantPagesUrls(url,startpage,endpage,keywords[int(searchid)]))

    que = getqueue(urls)

    print('本次下载行为：从%d页到%d页，有%d部电影正在下载...'%(startpage,endpage,que.qsize()))

    thread_count = que.qsize()
    starttime = time.time()

    for i in range(thread_count):
        threads.append(Movielinks(que,filepath,thread_count,starttime))

    for t in threads:
        t.start()

    for t in threads:
        t.join()

if __name__ == '__main__':
    # ch = int(input('请输入选项（负数退出）>>>'))
    # while ch >0:
    #     main()
    #     ch = int(input('请输入选项（负数退出）>>>'))
    main()

在爬取网页时，需要模拟浏览器的操作，所以在爬取数据时需要模拟一个user-agent ,相关代码如下：

from random import randint

def getUser_Agent():
    headers = [
        'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',   #safari 5.1 – MAC
        'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',  #safari 5.1 – Windows
        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',   #IE 9.0
        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',   #IE 8.0
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',   #IE 7.0
        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',   # IE6.0
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',   #Firefox 4.0.1 – MAC
        'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',     #Firefox 4.0.1 – Windows
        'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',     #Opera 11.11 – MAC
        'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',  #Opera 11.11 – Windows
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',    #Chrome 17.0 – MAC
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',  #傲游（Maxthon）
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)',  #腾讯TT
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',  #世界之窗（The World） 2.x
        'ozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)',  #世界之窗（The World） 3.x
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)',  #搜狗浏览器 1.x
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)',  #360浏览器
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)',     #Avant
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',     #Green Browser
        ##移动端用户代理
        'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5',     #safari iOS 4.33 – iPhone
        'Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5',     #safari iOS 4.33 – iPod Touch
        'Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5',     #safari iOS 4.33 – iPad
        'Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1',  #Android N1
        'MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1',  #Android QQ浏览器 For android
        'Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10',  #Android Opera Mobile
        'Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+',  #Android Pad Moto Xoom
        'Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0',  #WebOS HP Touchpad
        'Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124',  #Nokia N97
        'Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)',  #Windows Phone Mango
        'UCWEB7.0.2.37/28/999',  #UC无
        'Openwave/ UCWEB7.0.2.37/28/999',  #UCOpenwave
        'Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999',  #UC Opera
    ]
    return {'User-Agent':headers[randint(0,len(headers)-1)]}