python爬虫https://www.imdb.com/chart/top的电影

目标：爬取https://www.imdb.com/chart/top网页上面的电影top20

直接上main.py代码：

 #!/usr/bin/python35
 # -*- coding:utf-8 -*-
 # author: "Keekuun"

 import requests
 from lxml import html
 from download import download_url #download.py

 # 传入网址
 url = 'https://www.imdb.com/chart/top'
 headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36'}

 # 下载网页
 req = download_url(url, headers)
 tree = html.fromstring(req)
 xpath_x = '//*[@id="main"]/div/span/div/div/div[3]/table/tbody/tr'

 def info(x):
     # 下载排行  #strip() 方法用于移除字符串头尾指定的字符（默认为空格或换行符）。后面加上“text()”获取内容
     rank = tree.xpath(xpath_x + '[{}]/td[2]/text()'.format(x))[0].strip().strip('.')
     # print(rank)

     # 下载电影名
     name = tree.xpath(xpath_x + '[{}]/td[2]/a/text()'.format(x))[0]
     # print(name)

     # 下载电影评分
     score = tree.xpath(xpath_x + '[{}]/ td[3]/strong/text()'.format(x))[0]

     # print(score)

     # 下载电影海报链接。后面加上“@src”获取内容
     img_url = tree.xpath(xpath_x + '[{}]/td[1]/a/img/@src'.format(x))[0]
     # print(img_url)

     info = {
         'movie_rank':rank,
         'movie_name':name,
         'movie_score':score,
         'movie_img_url':img_url
     }
     return info

 with open('top_movie.txt','a',encoding='utf-8') as f:
     for x in range(1,21):
         movie = info(str(x))
         print(movie)
         movie_str ='Rank:{}\t Name:{}\t Score:{}\t ImgUrl:{}'.format(
             movie['movie_rank'],
             movie['movie_name'],
             movie['movie_score'],
             movie['movie_img_url']
         )
         f.write(movie_str + '\n')# 不可直接写入字典，必须为str

download.py代码部分：

#!/usr/bin/python35
# -*- coding:utf-8 -*-
# author: "Keekuun"

import requests

def download_url(url,headers):
    req = requests.get(url,headers)
    return req.content

def download_img(url,path):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36'}
    img = download_url(url,headers)
    with open(path,'wb') as f:  # “wb”方式打开文件
        f.write(img)

img.py保存图片：

 #!/usr/bin/python35
 # -*- coding:utf-8 -*-
 # author: "Keekuun"
 from download import download_img
 import os

 path = 'Movie_img'
 if not os.path.isdir(path):
     os.mkdir(path)

 # 打开main.py保存的top_movie.txt,从中获取图片现在地址
 with open('top_movie.txt','r',encoding='utf-8') as f:
     for x in f.readlines():
         rank = x.split(' ')[0].strip('Rank:').strip('\t')
         print(rank)
         img = x.split(' ')[-1].strip('ImgUrl:').strip()
         path = os.path.join('Movie_img/','{}.jpg'.format(rank))
         download_img(img, path)

结果：

top_movie.txt：

Rank:1 Name:The Shawshank Redemption Score:9.2 ImgUrl:https://ia.media-imdb.com/images/M/MV5BMDFkYTc0MGEtZmNhMC00ZDIzLWFmNTEtODM1ZmRlYWMwMWFmXkEyXkFqcGdeQXVyMTMxODk2OTU@._V1_UY67_CR0,0,45,67_AL_.jpg
Rank:2 Name:The Godfather Score:9.2 ImgUrl:https://ia.media-imdb.com/images/M/MV5BM2MyNjYxNmUtYTAwNi00MTYxLWJmNWYtYzZlODY3ZTk3OTFlXkEyXkFqcGdeQXVyNzkwMjQ5NzM@._V1_UY67_CR1,0,45,67_AL_.jpg
Rank:3 Name:The Godfather: Part II Score:9.0 ImgUrl:https://ia.media-imdb.com/images/M/MV5BMWMwMGQzZTItY2JlNC00OWZiLWIyMDctNDk2ZDQ2YjRjMWQ0XkEyXkFqcGdeQXVyNzkwMjQ5NzM@._V1_UY67_CR1,0,45,67_AL_.jpg
Rank:4 Name:The Dark Knight Score:9.0 ImgUrl:https://ia.media-imdb.com/images/M/MV5BMTMxNTMwODM0NF5BMl5BanBnXkFtZTcwODAyMTk2Mw@@._V1_UY67_CR0,0,45,67_AL_.jpg
Rank:5 Name:12 Angry Men Score:8.9 ImgUrl:https://ia.media-imdb.com/images/M/MV5BMWU4N2FjNzYtNTVkNC00NzQ0LTg0MjAtYTJlMjFhNGUxZDFmXkEyXkFqcGdeQXVyNjc1NTYyMjg@._V1_UX45_CR0,0,45,67_AL_.jpg
Rank:6 Name:Schindler's List Score:8.9 ImgUrl:https://ia.media-imdb.com/images/M/MV5BNDE4OTMxMTctNmRhYy00NWE2LTg3YzItYTk3M2UwOTU5Njg4XkEyXkFqcGdeQXVyNjU0OTQ0OTY@._V1_UX45_CR0,0,45,67_AL_.jpg
Rank:7 Name:The Lord of the Rings: The Return of the King Score:8.9 ImgUrl:https://ia.media-imdb.com/images/M/MV5BNzA5ZDNlZWMtM2NhNS00NDJjLTk4NDItYTRmY2EwMWZlMTY3XkEyXkFqcGdeQXVyNzkwMjQ5NzM@._V1_UY67_CR0,0,45,67_AL_.jpg
........

Movie_img：电影海报
1.jpg
2.jpg
3.jpg
.....
结果如下：