python核心编程中网络爬虫的例子

 #!/usr/bin/env python

 import cStringIO                    #
 import formatter                    #
 from htmllib import HTMLParser         # We use various classes in these modules for parsing HTML.
 import httplib                        # We only need an exception from this module
 import os                            # This provides various file system functions
 import sys                            # We are just using argv for command-line arguments
 import urllib                        # We only need the urlretrieve()function for downloading Web pages
 import urlparse                        # We use the urlparse()and urljoin()functions for URL manipulation

 class Retriever(object):
     __slots__ = ('url','file')

     def __init__(self,url):
         self.url, self.file = self.get_file(url)

     def get_file(self, url, default='index.html'):
         'Create usable local filename from URL'
         parsed = urlparse.urlparse(url)                     # ParseResult(scheme='http', netloc='www.baidu.com', path='', params='', query='', fragment='')
         host = parsed.netloc.split('@')[-1].split(':')[0]    # 'www.baidu.com'
         filepath = '%s%s' % (host,parsed.path)                # 'www.baidu.com'
         if not os.path.splitext(parsed.path)[1]:            # ''
             filepath = os.path.join(filepath, default)        # 'www.baidu.com\\index.html'
         linkdir = os.path.dirname(filepath)                    # 'www.baidu.com'
         if not os.path.isdir(linkdir):                        # False
             if os.path.exists(linkdir):                        # False
                 os.unlink(linkdir)
             os.makedirs(linkdir)                            # make a directory named by link directory on the hard disc
         return url, filepath

     def download(self):
         'Download URL to specific name file'
         try:
             retval = urllib.urlretrieve(self.url, self.file)
         except (IOError, httplib.InvalidURL) as e:
             retval = (('*** ERROR:bad URL "%s": %s' % (self.url,e)),)
         return retval

     def parse_links(self):
         'Parse out the links found in downloaded HTML file'
         f = open(self.file, 'r')
         data = f.read()
         f.close()
         parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(cStringIO.StringIO())))
         parser.feed(data)
         parser.close()
         return parser.anchorlist

 class Crawler(object):
     count = 0                                                # the number of objects downloaded from the internet

     def __init__(self, url):
         self.q = [url]                                        # a queue of links to download
         self.seen = set()                                    # a set containing all the links that we have seen(downloaded) already
         parsed = urlparse.urlparse(url)
         host = parsed.netloc.split('@')[-1].split(':')[0]
         self.dom = '.'.join(host.split('.')[-2:])            # 'b.a.i.d.u'

     def get_page(self, url, media=False):
         'Download page & parse links, add to queue if nec'
         r = Retriever(url)
         fname = r.download()[0]                                # 'www.baidu.com\\index.html'
         if fname[0] == '*':                                    # 'w'
             print fname, '... skipping parse'
             return
         Crawler.count += 1                                    #
         print '\n(', Crawler.count, ')'                        # (1)
         print 'URL:', url                                    # URL: http://www.baidu.com
         print 'FILE:', fname                                # FILE: www.baidu.com\\index.html
         self.seen.add(url)                                    # set(['http://www.baidu.com'])
         ftype = os.path.splitext(fname)[1]                    # '.html'
         if ftype not in ('.htm', '.html'):                    # False
             return

         for link in r.parse_links():
             if link.startswith('mailto:'):                    # False
                 print '... discarded, mailto link'
                 continue
             if not media:                                    # False
                 ftype = os.path.splitext(link)[1]
                 if ftype in ('.mp3','.mp4','.m4v','.wav'):
                     print '... discarded, media file'
                     continue
             if not link.startswith('http://'):                # False
                 link = urlparse.urljoin(url, link)
             print '*', link,
             if link not in self.seen:                        # True
                 if self.dom not in link:                    # False
                     print '... discarded, not in domain'
                 else:
                     if link not in self.q:
                         self.q.append(link)
                         print '... new, added to Q'
                     else:
                         print '... discarded, already in Q'
             else:
                 print '... discarded, already processed'

     def go(self, media=False):
         'Process next page in queue (if any)'
         while self.q:
             url = self.q.pop()
             self.get_page(url, media)

 def main():
         if len(sys.argv) > 1:
             url = sys.argv[1]
         else:
             try:
                 url = raw_input('Enter starting URL:')
             except(KeyboardInterrupt, EOFError):
                 url = ''
         if not url:
             return
         if not url.startswith('http://') and not url.startswith('ftp://'):
             url = 'http://%s/' % url
         robot = Crawler(url)
         robot.go()

 if __name__ == '__main__':
         main()