python之中国大学爬虫

 #!/usr/bin/env python3
 #-*- coding:utf-8 -*-
 ############################
 #File Name: zuihaodaxuepaiming.py
 #Author: frank
 #Mail: frank0903@aliyun.com
 #Created Time:2018-05-15 21:45:55
 ############################

 import requests
 from bs4 import BeautifulSoup

 #1. 获取html页面的内容
 def getHTMLText(url):
     try:
         r = requests.get(url)
         r.raise_for_status()
         #r.encoding = r.apparent_encoding
         r.encoding = 'utf-8'
         return r.text

     except:
         print("爬取异常")
         return ""

 #2. 从html页面内容中爬取大学排名信息, (排名，学校名称，总分)
 def getUnivInfo(uLst, html):
     soup = BeautifulSoup(html, "lxml")
     #获取排名中的大学的数量
     print(len(soup.tbody.find_all("tr", recursive=False)))
     #列出tbody的子节点
     #for tag in soup.tbody.find_all('tr', recursive=False):
     #    #print("name:{},type:{}".format(tag.name, type(tag)))
     #    print(tag.td.string)
     for tr in soup.tbody.find_all('tr', recursive=False,limit=1):
         tds = tr('td')
         #print(tr.prettify())
         #print(tds)
         #print(tds[0].string,tds[1].string, tds[3].string)
         uLst.append([tds[0].string,tds[1].string, tds[3].string])

 #3. 显示排名信息
 #def printUnivInfo(uLst, num):
 #    print("{0:^10}\t{1:{3}^10}\t{2:^10}".format("排名","学校名称","总分",chr(12288)))
 #    for uni in uLst[0:num]:
 #        print("{0:^10}\t{1:{3}^10}\t{2:^10}".format(uni[0],uni[1],uni[2],chr(12288)))

 def printUnivInfo(uLst, num):
     ptstr = "{0:^10}\t{1:{3}^10}\t{2:^10}"
     print(ptstr.format("排名","学校名称","总分",chr(12288)))
     for uni in uLst[0:num]:
         print(ptstr.format(uni[0],uni[1],uni[2],chr(12288))) #中文字符的空格填充 chr(12288)

 def main():
     url = "http://www.zuihaodaxue.cn/zuihaodaxuepaiming2016.html"
     uLst = []
     html = getHTMLText(url)
     #print(html)
     print("getHTMLText")
     getUnivInfo(uLst, html)
     print("getUnivInfo")
     printUnivInfo(uLst, 10)

 main()

该实例主要是针对 requests 和 BeautifulSoup的应用。