用python做爬虫的例子

主要就是用了两个库，urllib和BeautifulSoup.

作用是从HTML中解析出解梦的查询词和具体的解释。

 # -*- coding: utf-8 -*-
 import urllib, urllib2
 import time, random
 from BeautifulSoup import BeautifulSoup

 def fetchURL(str_url):

     user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) \
                   AppleWebKit/537.36 (KHTML, like Gecko)'
     values     = {}
     headers    = {'User-Agent': user_agent}
     data       = urllib.urlencode(values)

     content = ''

     try:
         request = urllib2.Request(str_url)
         response = urllib2.urlopen(request)
         html = response.read().decode('gb2312')
         content = parse_content_page(html)
     except:
         content = None

     return content

 def parse_content_page(html):
     parsed_html = BeautifulSoup(html)
     try:
         title   = parsed_html.body.find('h1', attrs={'class':'art_title'}).text
         content = parsed_html.body.find('div', attrs={'class':'dream_detail'}).text
     except:
         return None

     return [title, content]

 if __name__ == '__main__':

     foutput = 'jiemeng.txt'
     with open(foutput, 'w') as fout:
         for i in xrange(1, 10):
             reques_url = 'http://tools.2345.com/zhgjm/%s.htm' % str(i)
             x = fetchURL(reques_url)
             if x != None:
                 print >>fout, x[0].encode('utf8')[3:-3]
                 print >>fout, x[1].encode('utf8')

             # sleep for a while between two http requests
             seconds = random.random()*10 + 2
             time.sleep(seconds)