python 丰田经销商

 import requests
 import json
 from dbutil.pgsql import PgsqlPipeline
 from datetime import date

 headers={
         'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36',
         'Accept-Encoding': 'gzip, deflate, br',
         'Accept-Language': 'zh-CN,zh;q=0.9',

     }

 def get_Provinces():
     url = 'https://www.ftms.com.cn/website/Maintenance/getProvince'
     response =requests.get(url,headers=headers,verify=False)
     provinces_list= json.loads(response.text)['data']
     return provinces_list

 def get_Citys(provinces_cid):

     city_url =f'https://www.ftms.com.cn/website/Maintenance/getCity?cid={provinces_cid}'

     res=requests.get(city_url,verify=False,headers=headers)
     cities=json.loads(res.text)['data']
     return cities
     # print(cities)

     # city_list=json.loads(reponse.text)['data']
     # for city_dict in city_list:
     #     city_cid=city_dict.get('cid')
     #     city_name=city_dict.get('name')
     #     print(city_cid,city_name)
     #

 def get_Dealers(city_id,provinces_cid):

     dealers_url ='https://www.ftms.com.cn/website/Dealer/getDealer'
     data={'cityName': "" ,'cityid': city_id,'dealerName': "",'provinceName': "",'provinceid': provinces_cid}
     # data={"provinceid":"420000","cityid":"420700","dealerName":"","cityName":"","provinceName":""}
     print(data)

     dealer_res=requests.post(dealers_url,headers=headers,verify=False,data=json.dumps(data))
     dealers=json.loads(dealer_res.text)['data']['list']
     return dealers

 def get_item(dealer, province_name, city_name):
     item={
         "dealer_name":dealer["fullname"],
         "brand_id": None,
         "address": dealer["address"],
         "brand": "丰田",
         "province": province_name,
         "city": city_name,
         "sale_call": dealer["phone_seal"],
         "customer_service_call": dealer["phone_service"],
         "update_time": date.today(),
         "longitude": dealer['lng'],
         "latitude": dealer['lat'],
         "dealer_type": None,
         "manufacturer_id": None,
         "manufacturer": "丰田汽车",
         "state": None,
         "opening_date": None,
         "close_date": None,
         "dealer_id_web": None,
         "controlling_shareholder": None,
         "other_shareholders": None,
         "status": None,
         "remarks": None,

     }

     return item

 def get_items():
     items=[]
     provinces_list =get_Provinces()
     for provinces_dict in provinces_list:
         provinces_cid =provinces_dict.get('cid')
         provinces_name=provinces_dict.get('name')
         # print(provinces_cid,provinces_name)

         cities =get_Citys(provinces_cid)
         for city in cities:
             city_id =city.get('cid')
             city_name=city.get('name')
             # print(city_id,city_name)
             dealers=get_Dealers(city_id,provinces_cid)
             for dealer in dealers:

                 print(dealer)
                 item=get_item(dealer,provinces_name,city_name)
                 items.append(item)

     return items

 def main():
     print('爬虫开始.....')
     items=get_items()

     print('等待数据存储')
     pg =PgsqlPipeline()
     for item in items:
         pg.process_item(item)
     pg.close()
     print('数据存储完成！！！')

 if __name__ == '__main__':
     main()