python 爬取豆瓣的美剧

pc版大概有500条记录,mobile大概是50部,只有热门的,所以少一点

url构造很简单,主要参数就是page_limit与page_start,每翻一页,start+=20即可,tag是"美剧"编码后的结果,直接带着也可以,用unquote解码也可以,注意headers中一定要带上refer

 import json
 import requests
 import math
 import os
 import shutil
 from pprint import pprint
 from urllib import parse

 class DoubanSpliderPC:
     def __init__(self):
         self.url = parse.unquote(
             "https://movie.douban.com/j/search_subjects?type=tv&tag=%E7%BE%8E%E5%89%A7&sort=recommend&page_limit=20&page_start={}")

         self.headers = {
             "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
             "Referer": "https://movie.douban.com/tv/"
         }
         self.file_dir = "./douban_american_pc.txt"

     def parse_url(self):
             number = 0
             while True:
                 url = self.url.format(number)
                 print(url)
                 response = requests.get(url, headers=self.headers)
                 response_dict = json.loads(response.content.decode())
                 subjects_list = response_dict["subjects"]
                 with open(self.file_dir, "a", encoding="utf-8") as file:
                     for subject in subjects_list:
                        file.write(json.dumps(subject, ensure_ascii=False))
                        file.write("\r\n")
                 if len(subjects_list) < 20:
                     break
                 number += 20

     def run(self):
         # 删除之前保存的数据
         if os.path.exists(self.file_dir):
             os.remove(self.file_dir)
             print("文件已清空")
         self.parse_url()

 def main():
     splider = DoubanSpliderPC()
     splider.run()

 if __name__ == '__main__':
     main()

moblie类似,不过抓包的时候找那个Item就可以了

 import json
 import requests
 import math
 import os
 import shutil
 from pprint import pprint

 # 爬取豆瓣的美剧页面(手机版只有50条)
 class DouBanSpliderMobile:
     pageCount = 18
     total = None

     def __init__(self):
         self.first_url = "https://m.douban.com/rexxar/api/v2/subject_collection/tv_american/items?os=ios&for_mobile=1&start={}&count=18&loc_id=108288&_=1552995446961"
         self.headers = {
             "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
             "Referer": "https://m.douban.com/tv/american"
         }
         self.file_dir = "./douban_american_mobile.txt"

     def get_url_list(self):
         url_list = []
         for i in range(math.ceil(DouBanSpliderMobile.total / DouBanSpliderMobile.pageCount)):
             url = self.first_url.format(i * 18)
             url_list.append(url)
         return url_list

     def parse_url(self, url):
         response = requests.get(url, headers=self.headers)
         response_dict = json.loads(response.content.decode())
         DouBanSpliderMobile.total = int(response_dict["total"])
         with open(self.file_dir, "a", encoding="utf-8") as file:
             json.dump(response_dict["subject_collection_items"], file, ensure_ascii=False, indent=2)

     def run(self):
         # 解析第一个url,获取total
         self.parse_url(self.first_url.format(0))
         url_list = self.get_url_list()

         # 删除之前保存的文件
         if os.path.exists(self.file_dir):
             os.remove(self.file_dir)

         for url in url_list:
             self.parse_url(url)

 def main():
     douban_splider = DouBanSpliderMobile()
     douban_splider.run()

 if __name__ == '__main__':
     main()