Python爬虫--淘宝“泸州老窖”

爬虫淘宝--“泸州老窖”

爬去淘宝“泸州老窖” 相关信息：

import requests
import re
import json
import pandas as pd

## cookie
usercookie = 'miid.......'

def cookie_get(usercookie):
    """
    # 把字符串转换为字典
    :param usercookie:
    :return:
    """
    cookies = {}
    for a in usercookie.split(";"):
        name, value = a.strip().split("=", 1)
        cookies[name] = value
    return cookies

def open_url(keyword, page):
    keyload = {'q': keyword ,'s':str((page-1) * 44), 'sort':'sale-desc'}
    url = "https://s.taobao.com/search"
    cookies = cookie_get(usercookie)

    res = requests.get(url, params=keyload, cookies = cookies)
    return res

def get_page_items():
    with open("items.txt", "r", encoding="utf-8") as file1:
        g_page_config = re.search(r"g_page_config = (.*?);\n", file1.read())
        page_config_json = json.loads(g_page_config.group(1))
        page_items = page_config_json['mods']['itemlist']['data']['auctions']
        return page_items

def get_reslut(page_items, results):
    # results = []
    # page_items = get_page_items()
    for each_items in page_items:
        dict1 = dict.fromkeys(('nid', 'title', 'detail_url', 'view_price', 'view_sales', 'nick'))
        dict1['nid'] = each_items['nid']
        dict1['title'] = each_items['title']
        dict1['detail_url'] = each_items['detail_url']
        dict1['view_price'] = each_items['view_price']
        dict1['view_sales'] = each_items['view_sales']
        dict1['nick'] = each_items['nick']
        results.append(dict1)
    return results

def results_to_excel(results):
    pf = pd.DataFrame(list(results))
    order = ['nid', 'title', 'detail_url', 'view_price', 'view_sales', 'nick']
    pf = pf[order]
    file_name = pd.ExcelWriter('泸州老窖.xlsx')
    pf.fillna(' ', inplace=True) # 替换空格
    pf.to_excel(file_name, encoding='utf-8', index = False)
    file_name.save()

def main():
    keyword =  '泸州老窖' # input("请输入搜索关键词")
    length = 4 # 获取前3页商品( 44 * 3 )，每页44个商品，
    results = []
    for each in list(range(1, length)):
        res = open_url(keyword, each)
        with open("items.txt", 'w', encoding="utf-8") as file:
            file.write(res.text)
        page_items = get_page_items()
        results = get_reslut(page_items,results)

    results_to_excel(results)

if __name__ == "__main__":
    main()