用Python写爬虫爬取58同城二手交易数据

爬了14W数据,存入Mongodb,用Charts库展示统计结果,这里展示一个示意

模块1 获取分类url列表

from bs4 import BeautifulSoup
import requests,pymongo

main_url = 'http://bj.58.com/sale.shtml'
client = pymongo.MongoClient('localhost',27017)
tc_58 = client['58tc']
tab_link_list = tc_58['link_list']

web_data = requests.get(main_url)
soup = BeautifulSoup(web_data.text,'lxml')
sub_menu_link = soup.select('ul.ym-submnu > li > b > a')

link_list = []
count = 0
for link in sub_menu_link:
    link = 'http://bj.58.com' + link.get('href')
    #print(link)
    if link == 'http://bj.58.com/shoujihao/':
        pass
    elif link == 'http://bj.58.com/tongxunyw/':
        pass
    elif link == 'http://bj.58.com/tiaozao/':
        count += 1
        if count == 1:
            data = {'link':link}
            link_list.append(data)
    else:
        data = {'link': link}
        link_list.append(data)

for i in link_list:
    tab_link_list.insert(i)

模块2 获取每个商品详情信息

from bs4 import BeautifulSoup
import requests,re,pymongo,sys
from multiprocessing import Pool

client = pymongo.MongoClient('localhost',27017)
tc_58 = client['58tc']
# detail_link = tc_58['detail_link']
tab_link_list = tc_58['link_list']
# tc_58_data = client['58tcData']

def getDetailUrl(page_url,tab):
    url_list = []
    web_data = requests.get(page_url)
    soup = BeautifulSoup(web_data.text,'lxml')
    detail_url = soup.select('div.infocon > table > tbody > tr > td.t > a[onclick]')

    #获取详细页面url
    for url in detail_url:
        url_list.append(url.get('href').split('?')[0])

    #插入mongodb
    count = 0
    client = pymongo.MongoClient('localhost', 27017)
    tc_58 = client['58tc']
    tab_list = tc_58[tab+'_list']
    for i in url_list:
        count += 1
        tab_list.insert({'link':i})
    return count

original_price_patt = re.compile('原价：(.+)')
def getInfo(detail_url):
    try:
        web_data = requests.get(detail_url)
        soup = BeautifulSoup(web_data.text,'lxml')
        title = soup.title.text.strip()
        view_count = soup.select('body > div.content > div > div.box_left > div.info_lubotu.clearfix > div.box_left_top > p > span.look_time')[0].text
        want_count = soup.select('body > div.content > div > div.box_left > div.info_lubotu.clearfix > div.box_left_top > p > span.want_person')[0].text
        current_price = soup.select('body > div.content > div > div.box_left > div.info_lubotu.clearfix > div.info_massege.left > div.price_li > span > i')
        current_price = current_price[0].text if current_price else None
original_price = soup.select('body > div.content > div > div.box_left > div.info_lubotu.clearfix > div.info_massege.left > div.price_li > span > b')
        original_price = original_price[0].text if original_price else None
original_price = re.findall(original_price_patt,original_price) if original_price else None
location = soup.select('body > div.content > div > div.box_left > div.info_lubotu.clearfix > div.info_massege.left > div.palce_li > span > i')[0].text
        tag = soup.select('body > div.content > div > div.box_left > div.info_lubotu.clearfix > div.info_massege.left > div.biaoqian_li')
        tag = list(tag[0].stripped_strings) if tag else None
seller_name = soup.select('body > div.content > div > div.box_right > div.personal.jieshao_div > div.personal_jieshao > p.personal_name')[0].text
        # level = soup.select('body > div.content > div > div.box_right > div.personal.jieshao_div > div.personal_jieshao > span')
        # level = str(level[0]).split('\n')
        #
        # full_count = 0
        # half_count = 0
        # for j in level:
        #     if '<span class="icon_png "></span>' == j:
        #         full_count += 1
        #     elif '<span class="icon_png smallScore"></span>' == j:
        #         half_count += 1
        full_count = len(soup.find_all('span', class_='icon_png '))
        half_count = len(soup.find_all('span', class_='icon_png smallScore'))

        level_count = {'full':full_count,'half':half_count}
        desc = soup.select('body > div.content > div > div.box_left > div:nth-of-type(3) > div > div > p')
        desc = desc[0].text if desc else None
data = {
            'title':title,
            'view_count':view_count,
            'want_count':want_count,
            'current_price':current_price,
            'original_price':original_price,
            'location':location,
            'tag':tag,
            'seller_name':seller_name,
            #'level':level,
            'level_count':level_count,
            'desc':desc,
            'link':detail_url
}
        return data
    except:
        print(sys.exc_info()[0], sys.exc_info()[1])
        return None
# for i in tab_link_list.find({},{'link':1,'_id':0}):
#     print(i['link'])
#     getDetailUrl(i['link'])

#规律每个页面最多70页
def insertDetailLin(sub_menu_list):
    patt = re.compile('.+?com/([a-z]+)/')
    tab_list = []
    for i in sub_menu_list.find({},{'link':1,'_id':0}):
    #for i in [{'link':'http://bj.58.com/shouji/'}]:
        i = i['link']
        sub_menu_name = re.findall(patt,i)[0]
        print(sub_menu_name+': ',end='')
        url_list = []
        for j in range(1,71):
            link = i + 'pn' + str(j)
            url_list.append(link)

        cnt = 0
        for k in url_list:
            cnt = cnt + getDetailUrl(k, sub_menu_name)
        print(str(cnt) + ' lines inserted')
        if cnt != 0:
            tab_list.append(sub_menu_name+'_list')
    return tab_list

# for i in tab_link_list.find({},{'link':1,'_id':0}):
#     print(i)

#insertDetailLin(tab_link_list)

allMenCollectionName = tc_58.collection_names()
#allMenCollectionName.remove('detail_link')
allMenCollectionName.remove('link_list')
def insertData(tab_name):
    client = pymongo.MongoClient('localhost', 27017)
    tc_58 = client['58tc']
    tc_58_data = client['58tcDataNew']
    fenLei = tab_name[:-5]
    fenLei = tc_58_data[fenLei+'_data']
    tab_name = tc_58[tab_name]
    #print(tab_name)
    for i in tab_name.find({},{'link':1,'_id':0}):
        data = getInfo(i['link'])
        fenLei.insert(data)

def getContinuingly(fenlei):
    client = pymongo.MongoClient('localhost',27017)
    tc_58_data = client['58tcDataNew']
    tc_58 = client['58tc']
    fenlei_data = tc_58_data[fenlei+'_data']
    fenlei_list = tc_58[fenlei+'_list']
    db_urls = [item['link'] for item in fenlei_data.find()]
    index_url = [item['link'] for item in fenlei_list.find()]
    x=set(db_urls)
    y=set(index_url)
    rest_of_urls = y-x
    return list(rest_of_urls)

def startgetContinuingly(fenlei):
    client = pymongo.MongoClient('localhost', 27017)
    tc_58_data = client['58tcDataNew']
    fenLei = tc_58_data[fenlei+'_data']
    #rest_of_urls = getContinuingly('chuang')
    rest_of_urls = getContinuingly(fenlei)
    #print(rest_of_urls)
    for i in rest_of_urls:
        data = getInfo(i)
        fenLei.insert(data)

# startgetContinuingly('bijiben')
pool = Pool()
pool.map(insertData,allMenCollectionName)
#pool.map(insertData,['chuang_list'])
#insertData(allMenCollectionName)

模块3 分析

from collections import Counter
import pymongo,charts

def getTotalCount(database,host=None,port=None):
    client = pymongo.MongoClient(host,port)
    db = client[database]
    tab_list = db.collection_names()
    #print(tab_list)
    count = 0
    for i in tab_list:
        count = count + db[i].find({}).count()
    print(count)
    return count

#getTotalCount('58tcDataNew')
#14700

def getAreaByClassify(classify,database='58tcDataNew',host=None,port=None):
    client = pymongo.MongoClient(host, port)
    db = client[database]
    classify = classify + '_data'
    #location_list = [ i['location'][3:] if i['location'] != '' and i['location'][:2] == '北京' else None for i in db['bijiben_data'].find(filter={},projection={'location':1,'_id':0})]
    location_list = [i['location'][3:] for i in db['yueqi_data'].find(filter={}, projection={'location': 1, '_id': 0})
                     if i['location'] != '' and i['location'][:2] == '北京' and i['location'][3:] != '']
    loc_name = list(set(location_list))
    dic_count = {}
    for i in loc_name:
        dic_count[i] = location_list.count(i)
    return dic_count

# bijiben_area_count = getAreaByClassify(classify='yueqi')
# print(bijiben_area_count)
# danche_area_count = getAreaByClassify(classify='danche')
# sum_area_count = Counter(bijiben_area_count) + Counter(danche_area_count)
# print(sum_area_count)

def myCounter(L,database='58tcDataNew',host=None,port=None):
    client = pymongo.MongoClient(host, port)
    db = client[database]
    tab_list = db.collection_names()
    dic_0 = {}
    for i in tab_list:
        loc = i[:-5] + '_area_count'
        dic_0[loc] = 0

    if not L:
        return Counter(dic_0)
    else:
        return Counter(L[0]) + myCounter(L[1:])

def getAllCount(database='58tcDataNew',host=None,port=None):
    client = pymongo.MongoClient(host, port)
    db = client[database]
    tab_list = db.collection_names()
    dic_all_count = {}
    for i in tab_list:
        dic = getAreaByClassify(i[:-5])
        loc = i[:-5] + '_area_count'
        dic_all_count[loc] = dic

    dic_val = [dic_all_count[x] for x in dic_all_count]
    my = myCounter(dic_val)

    dic_all_count['total_area_count'] = dict(my)
    return dic_all_count

dic_all_count = getAllCount()
# print(dic_all_count['bijiben_area_count'])
# print(dic_all_count['total_area_count'])
#
#

tmp_list = []
for i in dic_all_count['total_area_count']:
    data = {
        'name':i,
        'data':[dic_all_count['total_area_count'][i]],
        'type':'column'
    }
    tmp_list.append(data)

options = {
    'chart'   : {'zoomType':'xy'},
    'title'   : {'text': '北京58同城二手交易信息发布区域分布图'},
    'subtitle': {'text': '数据来源: 58.com'},
    'xAxis'   : {'categories': ['']},
    'yAxis'   : {'title':{'text':'数量'}},
    'plotOptions': {'column': {'dataLabels': {'enabled': True}}}
    }
charts.plot(tmp_list,show='inline',options=options)