python应用-爬取猫眼电影top100

使用requests库和正则表达式爬取猫眼电影前100

import requests
import re
import json
import time
from requests.exceptions import RequestException

def get_one_page(url):
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36'
        }
        response = requests.get(url,headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        return None

def parse_one_page(html):
    pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name"><a'
                         + '.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>'
                         + '.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>', re.S)
    items = re.findall(pattern,html)
    for item in items:
        yield {
            'index':item[0],
            'image':item[1],
            'title':item[2].strip(),
            'actor':item[3].strip()[3:] if len(item[3]) > 3 else '',
            'time':item[4].strip()[5:] if len(item[4]) > 5 else '',
            'score':item[5].strip() + item[6].strip()
        }

def write_to_file(content):
    with open('result.txt','a',encoding='utf-8') as f:
        f.write(json.dumps(content,ensure_ascii=False)+'\n')
def main(offset):
    url = 'https://maoyan.com/board/4?offset=' + str(offset)
    html = get_one_page(url)
    # print(html)
    for item in parse_one_page(html):
        print(item)
        write_to_file(item)
if __name__ == '__main__':
    for i in range(10):
        main(offset=i*10)
        time.sleep(1)

使用requests库和Beautifulsoup库爬去猫眼电影前100

import requests
from bs4 import BeautifulSoup

def gethtmlpage(url):
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36"
    }
    response = requests.get(url, headers=headers)
    if response.status_code == 200:
        return response.text
    return None

def parsehtmlpage(html):
    soup = BeautifulSoup(html, 'lxml')
    a = soup.select('.movie-item-info a')
    return a

def write_to_file(content):
    with open('result.txt', 'a', encoding="utf-8") as f:
        f.writelines(content + '\n')

def main(url):
    html = gethtmlpage(url)
    title = parsehtmlpage(html)
    for i in range(0, len(title)):
        write_to_file(title[i].string)

if __name__ == '__main__':
    for i in range(10,100,10):
        url = "https://maoyan.com/board/4?offset=%d" % i
        main(url)

使用Beautiful库和requests库爬去：

import requests
from bs4 import BeautifulSoup
import bs4

def gethtmlpage(url):
    try:
        r = requests.get(url)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except ConnectionError:
        return "网络链接出错"
    except:
        return "未知错误"

def parsehtmlpage(html):
    soup = BeautifulSoup(html, 'lxml')
    ol = soup.select("ol.grid_view")
    li = ol[0].select('li')
    movie=[]
    for i in range(0, len(li)):
        index = li[i].select(".pic em")[0].string
        title = li[i].find("span", attrs={'class', 'title'}).string
        rating_num = li[i].find("span", attrs={'class', 'rating_num'}).string
        lianjie = li[i].select(".hd a")[0].get('href')
        if isinstance(li[i].find("span", attrs={'class', 'inq'}), bs4.element.Tag):
            inq = li[i].find("span", attrs={'class', 'inq'}).string
        else:
            inq = "没有简介"
        movie.append([index, title, rating_num, lianjie, inq])
    return movie

def writetofile(content):
    with open('result.csv', 'a', encoding='utf-8') as f:
        f.write(content)

def main(url):
    html = gethtmlpage(url)
    movie = parsehtmlpage(html)
    for i in range(0, len(movie)):
        writetofile("{0:^5}\t{1:{5}^10}\t{2:^10}\t{3:^40}\t{4:<10}\n".format(movie[i][0], movie[i][1], movie[i][2], movie[i][3], movie[i][4], chr(12288)))

if __name__ == '__main__':
    writetofile("{0:^5}\t{1:{5}^10}\t{2:^8}\t{3:^40}\t{4:<10}\n".format("排名", "电影名", "评分", "链接", "一句话介绍电影", chr(12288)))
    for i in range(0, 10):
        url = "https://movie.douban.com/top250?start={}".format(i*25)
        main(url)