lxml库介绍及实例

 XPath常用规则

表达式	描述
nodename	选取此节点的所有子节点
/	从当前节点选取直接子节点
//	从当前节点选取子孙节点
.	选取当前节点
..	选取当前节点的父节点
@	选取属性

html = lxml.etree.HTML(text)
#使用text构造一个XPath解析对象,etree模块可以自动修正HTML文本

html = lxml.etree.parse('./ex.html',etree.HTMLParser())
#直接读取文本进行解析

from lxml import etree
result = html.xpath('//*')
#选取所有节点

result = html.xpath('//li')
#获取所有li节点

result = html.xpath('//li/a')
#获取所有li节点的直接a子节点

result = html.xpath('//li//a')
#获取所有li节点的所有a子孙节点

result = html.xpath('//a[@href="link.html"]/../@class')
#获取所有href属性为link.html的a节点的父节点的class属性

result = html.xpath('//li[@class="ni"]')
#获取所有class属性为ni的li节点

result = html.xpath('//li/text()')
#获取所有li节点的文本

result = html.xpath('//li/a/@href')
#获取所有li节点的a节点的href属性

result = html.xpath('//li[contains(@class,"li")]/a/text())
#当li的class属性有多个值时，需用contains函数完成匹配

result = html.xpath('//li[contains(@class,"li") and @name="item"]/a/text()')
#多属性匹配

result = html.xpath('//li[1]/a/text()')
result = html.xpath('//li[last()]/a/text()')
result = html.xpath('//li[position()<3]/a/text()')
result = html.xpath('//li[last()-2]/a/text()')
#按序选择，中括号内为XPath提供的函数

result = html.xpath('//li[1]/ancestor::*')
#获取祖先节点
result = html.xpath('//li[1]/ancestor::div')
result = html.xpath('//li[1]/attribute::*')
#获取属性值
result = html.xpath('//li[1]/child::a[@href="link1.html"]')
#获取直接子节点
result = html.xpath('//li[1]/descendant::span')
#获取所有子孙节点
result = html.xpath('//li[1]/following::*[2]')
#获取当前节点之后的所有节点的第二个
result = html.xpath('//li[1]/following-sibling::*')
#获取后续所有同级节点

爬取bangumi动画排行榜并写入文件示例：

 import json
 import requests
 from requests.exceptions import RequestException
 import re
 import time
 from lxml import etree

 def get_one_page(url):
     try:
         headers = {
             'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36'
         }
         response = requests.get(url, headers=headers)
         if response.status_code == 200:
             return response.content.decode('utf-8')
         return None
     except RequestException:
         return None

 def parse_one_page(html):
     newhtml = etree.HTML(html)
     ranks = newhtml.xpath('//div/span[@class="rank"]/text()')
     names = newhtml.xpath('//div/h3/a/text()')
     others = newhtml.xpath('//div/p[@class="info tip"]/text()')
     scores = newhtml.xpath('//div/p[@class="rateInfo"]/small/text()')
     peoples = newhtml.xpath('//div/p[@class="rateInfo"]/span/text()')
     ret = []
     for i in range(len(ranks)):
         tmp = {
             'rank':ranks[i],
             'name':names[i],
             'other':others[i],
             'score':scores[i],
             'people':peoples[i]
             }
         ret.append(tmp)
         i += 1
     return ret

 def main(offset):
     url = 'http://bangumi.tv/anime/browser?sort=rank&page=' + str(offset)
     html = get_one_page(url)
     texts = parse_one_page(html)
     for text in texts:
         print(text)
         write_to_file(text)

 def write_to_file(content):
     with open('bangumi.txt', 'a', encoding='utf-8') as f:
         f.write(json.dumps(content, ensure_ascii=False) + '\n')

 if __name__ == '__main__':
     for i in range(1,20):
         main(offset=i)
         time.sleep(1)