Python 爬58同城 城市租房信息

爬取完会自动生成csv电子表格文件，含有房价、押付、链接等信息

环境

py2.7

pip install lxml

pip install cssselect

 

 #coding:utf-8
 import csv
 import urllib2
 import lxml.html
 import time
 import sys
 from lxml.cssselect import CSSSelector
 import threading
 reload(sys)
 sys.setdefaultencoding('utf8')

 print "请输入要爬取得城市简称例如bj（北京）："
 CITY=str(raw_input(">>>"))
 def download(url, user_agent='Google', num_retries=2):

     headers = {'User-agent': user_agent}
     request = urllib2.Request(url, headers=headers)
     try:
         html = urllib2.urlopen(request).read()
     except urllib2.URLError as e:
         html = None
         if num_retries > 0:
             if hasattr(e, 'code') and 500 <= e.code < 600:
                 return download(url, num_retries-1)
     return html

 def get_data(url):
     html_text_detail = download(url)
     try:
         tree = lxml.html.fromstring(html_text_detail)
         house_ext = CSSSelector('div.house-pay-way > span:nth-child(3)')
         house_title = CSSSelector('div.main-wrap > div.house-title > h1')
         house_pay_way1 = CSSSelector('div.house-pay-way > span:nth-child(1)')
         house_pay_way2 = CSSSelector('div.house-pay-way > span:nth-child(2)')
         print house_title(tree)[0].text_content()
         print '%s|%s' % (house_pay_way1(tree)[0].text_content(), house_pay_way2(tree)[0].text_content())

         for i in range(7):
             for j in range(2):
                 css = 'div.house-desc-item > ul.f14 > li:nth-child(%s) > span:nth-child(%s)' % (i+1, j+1)
                 house_info = CSSSelector(css)
                 data = [
                 ('标题 ： ',house_title(tree)[0].text_content(), '#',url),
                 ('价格： ',house_pay_way1(tree)[0].text_content(), '#'),
                 ('压付： ',house_pay_way2(tree)[0].text_content(), '#'),
                 ('详情： ',house_info(tree)[0].text_content().replace(' ', ''), '#')]
                 with open('%s_houses.csv'%CITY,'ab+') as csvfile:
                     writer = csv.writer(csvfile,lineterminator='\n')
                     writer.writerows(data)

     except TypeError as e:
         pass
     except IndexError as e:
         pass

 def get_url(html):
     tree = lxml.html.fromstring(html)
     sel = CSSSelector('div.mainbox > div.main > div.content > div.listBox > ul.listUl > li > div.des > h2 > a')
     url_list = []
     for i in sel(tree):
         if i.get('href') not in url_list:
             url_list.append(i.get('href'))
     return url_list

 if __name__ == '__main__':
     url_index = 'http://%s.58.com/chuzu/'%CITY
     html_text_list = download(url_index)
     url_list = get_url(html_text_list)

     for url_detail in url_list:
         thr = threading.Thread(target=get_data, args=(url_detail,))
         thr.start()

         time.sleep(0.001)

py58.py