使用requests、BeautifulSoup、线程池爬取艺龙酒店信息并保存到Excel中

import requests
import time, random, csv
from fake_useragent import UserAgent
from bs4 import BeautifulSoup
from threadpool import ThreadPool, makeRequests

def request_url(city_code, city_name, city_letter):
    """
    请求主页
    """
    with open('has_elong.json', 'a+', encoding='utf-8') as hs:
        hs.write(city_code + '\n')
    hs.close()
    if city_code and int(city_code) < 1000:
        city_code = '' + str(city_code)
    else:
        city_code = str(city_code)
    with open('艺龙/%s.csv' % city_name, 'w+', encoding='utf-8-sig') as f:
        cs = csv.writer(f, dialect='excel')
        # [酒店名称，价格，地址，星级，主题，可供服务，酒店信息]
        cs.writerow(['酒店名称', '价格', '地址', '星级', '主题', '可供服务', '酒店信息'])
        # 循环1-89页
        for n in range(1, 89):
            url = 'http://hotel.elong.com/%s/' % city_letter
            data = {
                "code": "",
                "listRequest.areaID": "",
                "listRequest.bookingChannel": "",
                "listRequest.cardNo": "",
                "listRequest.checkInDate": "2019-03-02 00:00:00",  # 入住时间
                "listRequest.checkOutDate": "2019-03-03 00:00:00",  # 离开时间
                "listRequest.cityID": city_code,
                "listRequest.cityName": city_name,  # 北京等地区
                "listRequest.customLevel": "",
                "listRequest.distance": "",
                "listRequest.endLat": "",
                "listRequest.endLng": "",
                "listRequest.facilityIds": "",
                "listRequest.highPrice": "",
                "listRequest.hotelBrandIDs": "",
                "listRequest.isAdvanceSave": "false",
                "listRequest.isAfterCouponPrice": "true",
                "listRequest.isCoupon": "false",
                "listRequest.isDebug": "false",
                "listRequest.isLimitTime": "false",
                "listRequest.isLogin": "false",
                "listRequest.isMobileOnly": "true",
                "listRequest.isNeed5Discount": "true",
                "listRequest.isNeedNotContractedHotel": "false",
                "listRequest.isNeedSimilarPrice": "false",
                "listRequest.isReturnNoRoomHotel": "true",
                "listRequest.isStaySave": "false",
                "listRequest.isTrace": "false",
                "listRequest.isUnionSite": "false",
                "listRequest.keywords": "",
                "listRequest.keywordsType": "",
                "listRequest.language": "cn",
                "listRequest.listType": "",
                "listRequest.lowPrice": "",
                "listRequest.orderFromID": "",
                "listRequest.pageIndex": n,  # 翻页
                "listRequest.pageSize": "",
                "listRequest.payMethod": "",
                "listRequest.personOfRoom": "",
                "listRequest.poiId": "",
                "listRequest.promotionChannelCode": "",
                "listRequest.proxyID": "ZD",
                "listRequest.rankType": "",
                "listRequest.returnFilterItem": "true",
                "listRequest.sellChannel": "",
                "listRequest.seoHotelStar": "",
                "listRequest.sortDirection": "",
                "listRequest.sortMethod": "",
                "listRequest.starLevels": "",
                "listRequest.startLat": "",
                "listRequest.startLng": "",
                "listRequest.taRecommend": "false",
                "listRequest.themeIds": "",
                "listRequest.ctripToken": "1c06a555-04ce-4884-aa05-e6f92ad0e84e",
                "listRequest.elongToken": "jc94shhj-d5a1-4092-8060-828b168dbb61"
            }
            headers = {
                'Accept': 'application/json, text/javascript, */*; q=0.01',
                'Accept-Encoding': 'gzip, deflate',
                'Accept-Language': 'zh-CN,zh;q=0.8',
                'Cache-Control': 'no-cache',
                'Content-Length': '',
                'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
                # 'Cookie':'……61b8-48a1-b398-8b9ec1903f05……',
                'Host': 'hotel.elong.com',
                'Origin': 'http://hotel.elong.com',
                'Pragma': 'no-cache',
                'Proxy-Connection': 'keep-alive',
                'Referer': 'http://hotel.elong.com/%s/' % city_letter,
                'User-Agent': UserAgent(verify_ssl=False).random,
                'X-Requested-With': 'XMLHttpRequest'
            }
            try:
                time.sleep(random.randint(1, 4))
                res = requests.get(url, data=data, headers=headers)
                dete_list = get_info_and_req_details(res.text)
                for data in dete_list:
                    cs.writerow(data)
            except Exception:
                continue
    f.close()

def get_info_and_req_details(html):
    """
    清洗该页列表数据并向请求各个酒店的详情页
    page_list = [酒店名称，价格，地址，星级，主题，可供服务，酒店信息]
    """
    bs = BeautifulSoup(html, "lxml")
    h_list = bs.find_all('div', attrs={'class': 'h_item'})
    page_list = []
    i = 0
    for hotel in h_list:
        if i < 25:
            try:
                hotel_name = hotel.find('div', attrs={'class': 'h_info_pic'}).find('img').get('alt')
                hotel_price = str(hotel.find('span', attrs={'class': 'h_pri_num'}).get_text()) + '元起'
                hotel_add = hotel.find('p', attrs={'class': 'h_info_b2'}).find('a').get_text().replace('[', '').replace(']', '')
                hotel_ress = hotel.find('span', attrs={'class': 'l1'}).get('data-hoteladdress')
                try:
                    hotel_grade = hotel.find('b', attrs={'class': 'icon_stars'}).get('title')
                except Exception:
                    hotel_grade = '经济型'
                try:
                    hotel_theme = hotel.find('div', attrs={'class': 'tagList'}).get_text().replace('\n', ',')
                except Exception:
                    hotel_theme = ''
                try:
                    hotel_link = hotel.find('div', attrs={'class': 'h_info_pic'}).find('a').get('href')
                    time.sleep(random.randint(1, 3))
                    detail_html = requests.get('http://hotel.elong.com%s#hotelContent' % hotel_link)
                    server, hotel_info = get_details(detail_html.text)
                except Exception:
                    server = ''
                    hotel_info = ''
            except Exception:
                continue
            page_list.append([hotel_name, hotel_price, str(hotel_add)+str(hotel_ress), hotel_grade, hotel_theme, server, hotel_info])
        i += 1
    return page_list

def get_details(detail_html):
    """
    清洗详情页数据
    """
    detail = BeautifulSoup(detail_html, 'lxml')
    server = ''
    hotel_info = ''
    try:
        server = detail.find('ul', attrs={'class': 'dview_icon_list'}).get_text().replace('\n', ',')
        hotel_info = detail.find('div', attrs={'class': 'dview_info'}).get_text().replace('\n', ',').replace('\t', ',')
    except Exception:
        return server, hotel_info
    return server, hotel_info

if __name__ == '__main__':
    has_num = []
    req_list = []
　　// 地址爬取请借鉴爬取携程酒店信息
    for line in open('elong.json', encoding='utf-8'):
        line_list = line.replace("\n", "").split(',')
        for has in open("has_elong.json", encoding='utf-8'):
            has_num.append(int(has.replace('\n', '')))
        if int(line_list[0]) in has_num:
            continue
        # request_url(line_list[0], line_list[1], line_list[2])
        line_tuple = (line_list, None)
        req_list.append(line_tuple)
    pool = ThreadPool(3)
    requests_list = makeRequests(request_url, req_list)
    [pool.putRequest(req) for req in requests_list]
    pool.wait()