Python爬取糗事百科示例代码

参考链接：http://python.jobbole.com/81351/#comment-93968

主要参考自伯乐在线的内容，但是该链接博客下的源码部分的正则表达式部分应该是有问题，试了好几次，没试成功。后来在下面的评论中看到有个使用BeautifulSoup的童鞋，试了试，感觉BeautifulSoup用起来确实很便捷。

 # -*- coding:utf-8 -*-

 '''
 Author:LeonWen
 '''

 import urllib
 import urllib2
 # import re
 from bs4 import BeautifulSoup

 page = 1
 url = 'http://www.qiushibaike.com/hot/page/' + str(page)
 # set the headers
 user_agent = 'Mozilla/4.0(compatible;MSIE 5.5;Windows NT)'
 headers = {'User-Agent':user_agent}
 try:
     request = urllib2.Request(url,headers=headers)
     response = urllib2.urlopen(request)
     object_bs = BeautifulSoup(response.read())
     # print object_bs.prettify()
     # items 是一个list保存着返回结果
     items = object_bs.body.find_all("div",{"class":"article block untagged mb15"})
     # print items
     floor = 1
     tag = 0
     for item in items:
         if item.find("div",{"class":"thumb"}) == None:
             # class=thumb为带有图片的标签
             author = item.find("h2")
             upNum = item.find("i",{"class":"number"})
             content = item.find("div",{"class":"content"})
             # print content.prettify()
             # print content.text
             print u"===============",floor,u" 楼 ======================="
             print u"作者:",author.text
             print u"赞同数:",upNum.text
             print u"内容:",content.get_text()
             floor += 1
         else:
             tag += 1
     print u"图片个数:",tag
 except urllib2.URLError,e:
     if hasattr(e,"code"):
         print e.code
     if hasattr(e,"reason"):
         print e.reason

原文地址：http://www.cnblogs.com/leonwen/p/5721843.html