我的第一个python爬虫

我的第一个爬虫，哈哈，纯面向过程

实现目标:

　　1.抓取本地conf文件，其中的URL地址，然后抓取视频名称以及对应的下载URL

　　2.抓取URL会单独写在本地路径下，以便复制粘贴下载

废话补多少，代码实现效果如下:

代码如下:

 #!/usr/local/python/bin/python3

 import requests
 import re
 import chardet
 import random
 import signal
 import time
 import os
 import sys

 def DealwithURL(url):
     r = requests.get(url)
     pattern = re.compile('<meta http-equiv="refresh" content="0.1;url=')
     findurl=re.findall(pattern,r.text)

     if findurl:
         pattern = re.compile('<meta http-equiv="refresh" content="0.1;url=(.*)"')

         transferurl = re.findall(pattern,r.text)[0]

         return transferurl
     else :
         return True

 def GetNewURL(url):
     r = requests.get(url)

     r.encoding='utf-8'
     pattern = re.compile('alert(.*)">')
     findurl=re.findall(pattern,r.text)
     findurl_str = (" ".join(findurl))
     return (findurl_str.split(' ',1)[0][2:])

 def gettrueurl(url):
     if DealwithURL(url)==True:
         return url
     else :
         return GetNewURL(DealwithURL(url))

 def SaveLocalUrl(untreatedurl,treatedurl):
     if untreatedurl == treatedurl :
         pass
     else :
         try:
             fileconf = open(r'main.conf','r')
             rewritestr = ""

             for readline in fileconf:
                 if re.search(untreatedurl,readline):
                     readline = re.sub(untreatedurl,treatedurl,readline)
                     rewritestr = rewritestr + readline
                 else :
                     rewritestr = rewritestr + readline
             fileconf.close()

             fileconf = open(r'main.conf','w')
             fileconf.write(rewritestr)
             fileconf.close()

         except:
             print ("get new url but open files ng write to logs")

 def handler(signum,frame):
     raise AssertionError

 def WriteLocalDownloadURL(downfile,downurled2k):
     urlfile = open(downfile,'a+')

     urlfile.write(downurled2k+'\n')

 def GetDownloadURL(sourceurl,titleurl,titlename,update_file,headers):
     downurlstr = (" ".join(titleurl))
     downnamestr = (" ".join(titlename))

     r = requests.get((sourceurl+downurlstr),headers)
     pattern = re.compile('autocomplete="on">(.*)/</textarea></div>')

     downurled2k = re.findall(pattern,r.text)
     downurled2kstr = (" ".join(downurled2k))

     WriteLocalDownloadURL(update_file , downurled2kstr)

     print (downnamestr , downurled2kstr)

 def ReadLocalFiles() :
     returndict={}
     localfiles = open(r'main.conf')

     readline = localfiles.readline().rstrip()

     while readline :
         if readline.startswith('#'):
             pass
         else:
             try:
                 readline = readline.rstrip()
                 returndict[readline.split('=')[0]] = readline.split('=')[1]
             except:
                 print ("Please Check your conf %s" %(readline))
                 sys.exit(1)
         readline = localfiles.readline().rstrip()

     localfiles.close()
     return returndict

 def GetListURLinfo(sourceurl , title , getpagenumber , total,update_file,headers):

     if total >= 100:
         total = 100

     if total <= 1:
         total = 2

     getpagenumber = total

     for number in range(0,total) :
         try:
             signal.signal(signal.SIGALRM,handler)
             signal.alarm(3)

             url = sourceurl + title + '-' + str(random.randint(1,getpagenumber)) + '.html'

             r = requests.get(url,headers)

             pattern = re.compile('<div class="info"><h2>(.*)</a><em></em></h2>')
             r.encoding = chardet.detect(r.content)['encoding']
             allurl = re.findall(pattern,r.text)

             for lineurl in allurl:
                 try:
                     signal.signal(signal.SIGALRM,handler)
                     signal.alarm(3)

                     pattern = re.compile('<a href="(.*)" title')
                     titleurl = re.findall(pattern,lineurl)

                     pattern = re.compile('title="(.*)" target=')
                     titlename = re.findall(pattern,lineurl)

                     GetDownloadURL(sourceurl,titleurl,titlename,update_file,headers)
                     signal.alarm(0)
                 except AssertionError:
                     print (lineurl,titlename , "Timeout Error: the cmd 10s have not finished")
                     continue

 #            title = '/list/'+str(random.randint(1,8))
 #            print (title)
 #            print (title_header)

         except AssertionError:
             print ("GetlistURL Infor Error")
             continue

 def GetTitleInfo(url,down_page,update_file,headers):

     title = '/list/'+str(random.randint(1,8))

     titleurl = url + title + '.html'

     r = requests.get(titleurl,headers)
     r.encoding = chardet.detect(r.content)['encoding']
     pattern = re.compile('&nbsp;当前:.*/(.*)页&nbsp;')
     getpagenumber = re.findall(pattern,r.text)

     getpagenumber = (" ".join(getpagenumber))

     GetListURLinfo(url , title , int(getpagenumber) , int(down_page),update_file,headers)

 def write_logs(time,logs):
     loginfo = str(time)+logs
     try:
         logfile = open(r'logs','a+')
         logfile.write(loginfo)
         logfile.close()
     except:
         print ("Write logs error,code:154")

 def DeleteHisFiles(update_file):
     if os.path.isfile(update_file):
         try:
             download_files = open(update_file,'r+')
             download_files.truncate()
             download_files.close()
         except:
             print ("Delete " + update_file + "Error --code:166")
     else :
         print ("Build New downfiles")

 def main():
     headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit 537.36 (KHTML, like Gecko) Chrome","Accept": "text/html,application/xhtml+xml,application/xml; q=0.9,image/webp,*/*;q=0.8"}

     readconf = ReadLocalFiles()

     try:
         file_url = readconf['url']
         down_page = readconf['download_page']
         update_file = readconf['download_local_files']
     except:
         print ("Get local conf error,please check it")
         sys.exit(-1)

     DeleteHisFiles(update_file)

     untreatedurl = file_url

     treatedurl = gettrueurl(untreatedurl)
     SaveLocalUrl(untreatedurl,treatedurl)

     url = treatedurl

     GetTitleInfo(url,int(down_page),update_file,headers)

 if __name__=="__main__":
     main()

对应的main.conf如下：

本着对爬虫的好奇来写下这些代码，如有对代码感兴趣的，可以私聊提供完整的conf信息，毕竟，我也是做运维的，要脸，从不用服务器下片。