如何分页爬取数据--beautisoup

'''本次爬取讲历史网站'''#!usr/bin/env python#-*- coding:utf-8 _*-"""@author:Hurrican@file: 分页爬取数据.py@time: 2018/11/03 9:30

"""from bs4 import BeautifulSoupimport requests

def get_urls():    urls = ['http://www.jianglishi.cn/jiemi/page_{}.html'.format(str(i)) for i in range(2, 21)]    urls.insert(0, 'http://www.jianglishi.cn/jiemi/')    return urls

def get_title():    for a1 in get_urls():        web_data = requests.get(a1)

        web_data.encoding = 'utf-8'        web = web_data.text        soup = BeautifulSoup(web, 'html5lib')        soup1 = soup.findAll(name='div', attrs={'class': 'title'})        for piece in soup1:            title = piece.a.string            print(title)

if __name__ == '__main__':    get_title()运行结果：

方法2：

#!usr/bin/env python#-*- coding:utf-8 _*-"""@author:lenovo@file: spider_urllib.py@time: 2018/11/07 14:31

"""import urllib.requestfrom bs4 import  BeautifulSoup'''python3中unicode 与 bytes 相互转化   str类型转化为bytes类型，使用encode()内置函数；反过来，使用decode()函数'''def get_content():

    urls = ['http://www.jianglishi.cn/jiemi/page_{}.html'.format(str(i)) for i in range(2, 21)]    urls.insert(0, 'http://www.jianglishi.cn/jiemi/')    for url in urls:        html = urllib.request.urlopen(url)        content = html.read()        content = content.decode()        html.close()

        osoup = BeautifulSoup(content,'html5lib')        all_title  = osoup.find_all('div',class_="title")       # print(all_images)        for title  in all_title:            print(title.a.string)

get_content()

衍生阅读：

我们来继续爬取图片 

#!usr/bin/env python# -*- coding: utf-8 -*-"""@author:lenovo@file: spider_urllib.py@time: 2018/11/07 14:31

"""import urllib.requestfrom bs4 import  BeautifulSoup

'''python3中unicode 与 bytes 相互转化   str类型转化为bytes类型，使用encode()内置函数；反过来，使用decode()函数'''def get_urls():    urls = ['http://www.jianglishi.cn/jiemi/page_{}.html'.format(str(i)) for i in range(2, 21)]    urls.insert(0, 'http://www.jianglishi.cn/jiemi/')    return urls

def get_content():

    # urls = ['http://www.jianglishi.cn/jiemi/page_{}.html'.format(str(i)) for i in range(2, 21)]    # urls.insert(0, 'http://www.jianglishi.cn/jiemi/')    x= 1    for url in get_urls():        html = urllib.request.urlopen(url)        content = html.read()        content = content.decode('utf-8')        html.close()

        osoup = BeautifulSoup(content,'html5lib')        all_images  = osoup.find_all('img',onerror="this.src='/statics/statics/img/nopic.gif';this.onerror=null;")        print(all_images)

        for img in all_images:            #print(img['src'])            dow_img = img['src'].encode('utf-8').decode('utf-8')            g = dow_img.replace('。','.')    #此处我们发现有一个链接出现中文的句号，需要将其处理成标准的url           # print(g)            s = urllib.request.urlretrieve(g,r'H:\py\image\\%s.jpg'%x)            print("正在下载%s"%dow_img)            x+=1    print("下载完成")

get_content()'''<img src="http://cimg2。163.com/cnews/2006/9/25/20060925163612ab80e.jpg" alt="三峡蓄水奉节老城全淹　“刘备疑冢”永沉长江底" onerror="this.src='/statics/statics/img/nopic.gif';this.onerror=null;">'''

上面这种情况直接是要用try catch  就能避免