BeautifulSoup的基本用法

 # -*- coding:UTF-8 -*-
 from urllib import request
 from bs4 import BeautifulSoup
 import re
 import sys

 if __name__ == "__main__":
     #创建txt文件
     file = open('一念永恒.txt', 'w', encoding='utf-8')
     #一念永恒小说目录地址
     target_url = 'http://www.biqukan.com/1_1094/'
     #User-Agent
     head = {}
     head['User-Agent'] = 'Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166  Safari/535.19'
     target_req = request.Request(url = target_url, headers = head)
     target_response = request.urlopen(target_req)
     target_html = target_response.read().decode('gbk','ignore')
     #创建BeautifulSoup对象
     listmain_soup = BeautifulSoup(target_html,'lxml')

     #搜索文档树,找出div标签中class为listmain的所有子标签
     chapters = listmain_soup.find_all('div',class_ = 'listmain')
     #使用查询结果再创建一个BeautifulSoup对象,对其继续进行解析
     download_soup = BeautifulSoup(str(chapters), 'lxml')
     #计算章节个数
     numbers = (len(download_soup.dl.contents) - 1) / 2 - 8
     index = 1
     #开始记录内容标志位,只要正文卷下面的链接,最新章节列表链接剔除
     begin_flag = False
     #遍历dl标签下所有子节点
     for child in download_soup.dl.children:
         #滤除回车
         if child != '\n':
             #找到《一念永恒》正文卷,使能标志位
             if child.string == u"《一念永恒》正文卷":
                 begin_flag = True
             #爬取链接并下载链接内容
             if begin_flag == True and child.a != None:
                 download_url = "http://www.biqukan.com" + child.a.get('href')
                 download_req = request.Request(url = download_url, headers = head)
                 download_response = request.urlopen(download_req)
                 download_html = download_response.read().decode('gbk','ignore')
                 download_name = child.string
                 soup_texts = BeautifulSoup(download_html, 'lxml')
                 texts = soup_texts.find_all(id = 'content', class_ = 'showtxt')
                 soup_text = BeautifulSoup(str(texts), 'lxml')
                 write_flag = True
                 file.write(download_name + '\n\n')
                 #将爬取内容写入文件
                 for each in soup_text.div.text.replace('\xa0',''):
                     if each == 'h':
                         write_flag = False
                     if write_flag == True and each != ' ':
                         file.write(each)
                     if write_flag == True and each == '\r':
                         file.write('\n')
                 file.write('\n\n')
                 #打印爬取进度
                 sys.stdout.write("已下载:%.3f%%" % float(index/numbers) + '\r')
                 sys.stdout.flush()
                 index += 1
     file.close()

>>> for link in soup.find_all('a'):
... print(link.get('href'))
#用于爬取a标签的链接

　　

Beautiful Soup 4.4.0 文档链接：http://beautifulsoup.readthedocs.io/zh_CN/v4.4.0/

已下文章来自博客园大佬：http://www.cnblogs.com/sakura3/p/8460224.html（为了复习方便，搬一下，谢谢）

爬小说：

 #!/usr/bin/python
 # -*- coding: UTF-8 -*-
 import requests
 from bs4 import BeautifulSoup
 # get_url_list 获取所有章节的URL，在一个list里
 def get_url_list(url):
     content = requests.get(url).content           #获取页面内容
     soup = BeautifulSoup(content,'lxml')          #Beautifulsoup 实例化对象
     url_list = []                                #空的url_list 数组
     # urls = soup.find('div',{'id':'list'}).find('dl').find_all('dd')
     urls = soup.select('#list > dl > dd > a')    # 根据页面选择到URL ，还可以urls = soup.find('div',{'id':'list'}).find('dl').find_all('dd')
     for i in urls:          #遍历里面的每一章的URL
         i = i.get('href')   #获取URL
         # print(i)
         i = 'http://www.biquge.com.tw' + i   #分析文章组成，形成最终的URL
         url_list.append(i)    #添加到url_list 里面去
     # print (url_list)
     return url_list
 # 获取这一章的内容
 def get_data(url):
     content = requests.get(url).content
     soup = BeautifulSoup(content, 'lxml')
     f = open(r'C:\Users\HBX\Documents\staudy\HMXX.txt','a+',encoding='utf-8')  #不加utf-8 会有编码报错
     text_name = soup.find('div',{'class':'bookname'}).find('h1').text     #获得章节名字
     # text_content = soup.select('#content')
     text_content = soup.find('div',{'id':'content'}).get_text()    #获得章节内容  ，还有一种select css 选择的获取章节内容的方式
     book =text_name+ '\r\n' + text_content    #整体的一章
     # print(book)
     f.write((book)+'\r\n')   #换行写入
     f.close()   #关闭文件
     # for x in text_content:
     #     a = x.text.replace('readx();', '')
     #     print(a)

 if __name__ =='__main__':
     url = 'http://www.biquge.com.tw/18_18049/'   #笔趣阁的小说目录页面
     url_list = get_url_list(url)   #获取了所有的url
     for i in url_list:    # 循环一章url
         get_data(i)      #获取文章内容