Python爬虫（四）

爬取雪球网上的房产信息

源码：

 import requests
 import json
 import pymysql

 # 建立数据库连接
 db = pymysql.connect(host='127.0.0.1', user='root', password='', port=3306, database='xueqiu')
 # 创建游标对象
 cursor = db.cursor()

 # 定义请求头信息
 headers = {
     "Accept": "*/*",
     # "Accept-Encoding": "gzip, deflate, br",
     "Accept-Language": "zh-CN,zh;q=0.9",
     "Connection": "keep-alive",
     "Cookie": "aliyungf_tc=AQAAAO+yOl0mxQEAUhVFeV0ZK5j5OLZs; xq_a_token=584d0cf8d5a5a9809761f2244d8d272bac729ed4; xq_a_token.sig=x0gT9jm6qnwd-ddLu66T3A8KiVA; xq_r_token=98f278457fc4e1e5eb0846e36a7296e642b8138a; xq_r_token.sig=2Uxv_DgYTcCjz7qx4j570JpNHIs; _ga=GA1.2.857846928.1534331621; _gid=GA1.2.1996927600.1534331621; Hm_lvt_1db88642e346389874251b5a1eded6e3=1534331622; Hm_lpvt_1db88642e346389874251b5a1eded6e3=1534331622; u=831534331622164; device_id=6715ed8e4eba695ab8a41bd752dbd204",
     "Host": "xueqiu.com",
     "Referer": "https://xueqiu.com/",
     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
     "X-Requested-With": "XMLHttpRequest",
 }

 max_id = -1
 # 循环三次，爬取3页信息
 for i in range(3):
     # 生成url
     url = 'https://xueqiu.com/v4/statuses/public_timeline_by_category.json?since_id=-1&max_id={}&count=15&category=111'.format(max_id)
     # 发送get请求
     response = requests.get(url, headers=headers)
     # print(response.json())
     # 响应字典格式数据
     res = response.json()
     # 重新赋值下一次的max_id
     max_id = res['next_max_id']
     # print(res['list'])
     for dict_ in res['list']:
         # print(dict_)
         # 将json数据转成字典
         dic = json.loads(dict_['data'])
         # print(type(dic),dic)
         id = str(dic['id'])
         title = dic['title']
         description = dic['description']
         target = dic['target']
         # print(id)
         # print(title)
         # print(description)
         # print(target)
         # 拼接sql语句
         sql = "insert into news(id,title,description,target) values('"+id+"','"+title+"','"+description+"','"+target+"');"
         print('正在插入数据：\n'+sql)
         # 执行sql
         cursor.execute(sql)
         # 提交
         db.commit()
 # 关闭游标对象
 cursor.close()
 # 关闭数据库连接
 db.close()