爬取漫画DB上的《浪客行》

漫画链接：https://www.manhuadb.com/manhua/324

建议：早上爬，速度较快。

天下无双宫本武藏

代码

# https://www.manhuadb.com/manhua/324

import os
import re
import time
import requests
from requests import codes
from bs4 import BeautifulSoup
from requests import RequestException

def get_page(url):
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36'
                   + '(KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36'}
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return 'status_code = ' + str(status_code)
    except RequestException:
        return 'RequestException'

def get_pagesNumber(text):
    soup = BeautifulSoup(text, 'lxml')
    pagesNumber = soup.find(name='div', class_="d-none vg-r-data")
    return pagesNumber.attrs['data-total'] 

def parse_page(text):
    soup = BeautifulSoup(text, 'lxml')
    url = soup.find(name='img', class_="img-fluid show-pic")
    chapter = soup.find(name='h2', class_="h4 text-center")
    page = soup.find(name='span', class_="c_nav_page")
    yield {
        'url': url['src'],
        'chapter': chapter.get_text(),
        'page': page.get_text()
    }  

def save_image(item):
    img_path = '浪客行' + os.path.sep + item.get('chapter') #os.path.sep是路径分隔符\
    if not os.path.exists(img_path):
        os.makedirs(img_path)
    try:
        resp = requests.get(item.get('url'))
        if codes.ok == resp.status_code:
            file_path = img_path + os.path.sep + '{file_name}.{file_suffix}'.format(
                file_name=item.get('chapter')[-2:]+'-'+item.get('page'), file_suffix='jpg')
            if not os.path.exists(file_path):
                with open(file_path, 'wb') as f:
                    f.write(resp.content)
                print('Downloaded image path is %s' % file_path)
            else:
                print('Already Downloaded', file_path)
    except Exception as e:
        print(e)

if __name__ == '__main__':
    for chapter in range(3678, 3715): #共37章节，3678到3714
        base_url = 'https://www.manhuadb.com/manhua/324/320_'+str(chapter)
        text = get_page(base_url+'.html')
        pagesNumber = get_pagesNumber(text) #获取当前章节总页数
        for page in range(1, int(pagesNumber)+1):
            url = base_url+'_'+str(page)+'.html'
            text = get_page(url)
            for item in parse_page(text):
                save_image(item)