Python 爬基金数据

爬科学基金共享服务网中基金数据

#coding=utf-8
import json
import requests
from lxml import etree
from HTMLParser import HTMLParser
from pymongo import MongoClient

data = {'pageSize':10,'currentPage':1,'fundingProject.projectNo':'','fundingProject.name':'','fundingProject.person':'','fundingProject.org':'',
'fundingProject.applyCode':'','fundingProject.grantCode':'','fundingProject.subGrantCode':'','fundingProject.helpGrantCode':'','fundingProject.keyword':'',
'fundingProject.statYear':'','checkCode':'%E8%AF%B7%E8%BE%93%E5%85%A5%E9%AA%8C%E8%AF%81%E7%A0%81'}
url = 'http://npd.nsfc.gov.cn/fundingProjectSearchAction!search.action'
headers = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding':'gzip, deflate',
'Accept-Language':'zh-CN,zh;q=0.9',
'Cache-Control':'max-age=0',
'Connection':'keep-alive',
'Content-Length':'',
'Content-Type':'application/x-www-form-urlencoded',
'Cookie':'JSESSIONID=8BD27CE37366ED8022B42BFC68FF82D4',
'Host':'npd.nsfc.gov.cn',
'Origin':'http://npd.nsfc.gov.cn',
'Referer':'http://npd.nsfc.gov.cn/fundingProjectSearchAction!search.action',
'Upgrade-Insecure-Requests':'',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}

def main():
    client = MongoClient('localhost', 27017)
    db = client.ScienceFund
    db.authenticate("","")
    collection=db.science_fund
    for i in range(1, 43184):
        print i
        data['currentPage'] = i
        result = requests.post(url, data = data, headers = headers)
        html = result.text
        tree = etree.HTML(html)
        table = tree.xpath("//dl[@class='time_dl']")
        for item in table:
            content = etree.tostring(item, method='html')
            content =  HTMLParser().unescape(content)
            # print content
            bson = jiexi(content)
            collection.insert(bson)

def jiexi(content):
    # 标题
    title1 = content.find('">', 20)
    title2 = content.find('</')
    title = content[title1+2:title2]
    # print title
    # 批准号
    standard_no1 = content.find(u'批准号', title2)
    standard_no2 = content.find('</dd>', standard_no1)
    standard_no = content[standard_no1+4:standard_no2].strip()
    # print standard_no
    # 项目类别
    standard_type1 = content.find(u'项目类别', standard_no2)
    standard_type2 = content.find('</dd>', standard_type1)
    standard_type = content[standard_type1+5:standard_type2].strip()
    # print standard_type
    # 依托单位
    supporting_institution1 = content.find(u'依托单位', standard_type2)
    supporting_institution2= content.find('</dd>', supporting_institution1)
    supporting_institution = content[supporting_institution1+5:supporting_institution2].strip()
    # print supporting_institution
    # 项目负责人
    project_principal1 = content.find(u'项目负责人', supporting_institution2)
    project_principal2 = content.find('</dd>', project_principal1)
    project_principal = content[project_principal1+6:project_principal2].strip()
    # print project_principal
    # 资助经费
    funds1 = content.find(u'资助经费', project_principal2)
    funds2 = content.find('</dd>', funds1)
    funds = content[funds1+5:funds2].strip()
    # print funds
    # 批准年度
    year1 = content.find(u'批准年度', funds2)
    year2 = content.find('</dd>', year1)
    year = content[year1+5:year2].strip()
    # print year
    # 关键词
    keywords1 = content.find(u'关键词', year2)
    keywords2 = content.find('</dd>', keywords1)
    keywords = content[keywords1+4:keywords2].strip()
    # print keywords
    dc = {}
    dc['title'] = title
    dc['standard_no'] = standard_no
    dc['standard_type'] = standard_type
    dc['supporting_institution'] = supporting_institution
    dc['project_principal'] = project_principal
    dc['funds'] = funds
    dc['year'] = year
    dc['keywords'] = keywords
    return dc

if __name__ == '__main__':
    main()