04爬取拉勾网Python岗位分析报告

# 导入需要的包
import requests
import time,random
from openpyxl import Workbook
import pymysql.cursors
#@ 连接数据库；
# 这个是我本地上边运行的程序，用来获取代理服务器。
def get_proxy():
    try:
        PROXY_POOL_URL = 'http://localhost:5555/random'
        response = requests.get(PROXY_POOL_URL)
        print(response.text)
        if response.status_code == 200:
            return response.text
    except ConnectionError:
        return None
# 用来连接本地mysql，可以不连接，直接写入Excel中
def get_conn():
    """连接本地数据库"""
    # 定义要连接的主机IP，账号名称和密码，连接的数据库，编码等等
    conn = pymysql.connect(host = 'localhost',
                           user = 'root',
                           password = '123456',
                           db = 'python',
                           charset = 'utf8mb4',
                           cursorclass = pymysql.cursors.DictCursor)
    return conn
# 将数据写入到数据库中
def insert(conn,info):
    """数据写入数据库"""
    with conn.cursor() as cursor:
        sql = "INSERT INTO `python` (`companyShortName`, `companyFullName`, `industryField`, `companySize`, `salary`, `city`, `education`) VALUES (%s, %s, %s, %s, %s, %s, %s)"
        cursor.execute(sql, info)
    conn.commit()
# 获取当前网址的信息
def get_json(url,page,lang_name):
    """返回当前页面的信息列表"""
    data = {'first':'false','pn':page,'kd':lang_name}
    proxies = get_proxy()
    proxies = {
        "http": "http://" + proxies
    }
    json = ses.post(url,data,proxies = proxies).json()
    list_con = json['content']['positionResult']['result']
    info_list = []
    for i in list_con:
        info = []
        info.append(i.get('companyShortName','无')) # 公司名称
        info.append(i.get('companyFullName','无'))
        info.append(i.get('industryField','无'))
        info.append(i.get('companySize','无'))
        info.append(i.get('salary','无'))
        info.append(i.get('city','无'))
        info.append(i.get('education','无'))
        info_list.append(info)
    return info_list

def main():
    lang_name = 'python'
    wb = Workbook() # 打开Excel工作薄
    conn = get_conn()  # 建立数据库连接 不存放数据，注释此行
    for i in ['北京','上海','广州','深圳','杭州']: #五个城市
        page = 1
        wsl = wb.active
        wsl.title = lang_name
        url = 'https://www.lagou.com/jobs/positionAjax.json?city={}&needAddtionalResult=false'.format(i)
        while page < 2: # 每个城市30页信息
            info = get_json(url,page,lang_name)
            page += 1
            # time.sleep(random.randint(10,20))
            for row in info:
                # 插入数据库，若不想存入 注释此行
                insert(conn,tuple(row))
                wsl.append(row)
    # 关闭数据库连接，不存放数据，注释此行
    conn.close()
    wb.save('{}职位信息.xlsx'.format(lang_name))

if __name__ == "__main__":
    my_headers = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36",
    "Referer": "https://www.lagou.com/jobs/list_Python?city=%E5%85%A8%E5%9B%BD&cl=false&fromSearch=true&labelWords=&suginput=",
    "Content-Type": "application/x-www-form-urlencoded;charset = UTF-8"
    }
    # time.sleep(5)
    ses = requests.session()  # 获取 session
    ses.headers.update(my_headers)  # 更新
    ses.get(
        "https://www.lagou.com/jobs/list_python?city=%E5%85%A8%E5%9B%BD&cl=false&fromSearch=true&labelWords=&suginput=")
    main()