python 爬取bilibili 视频信息

抓包时发现子菜单请求数据时一般需要rid,但的确存在一些如游戏->游戏赛事不使用rid,对于这种未进行处理,此外rid一般在主菜单的响应中,但有的如番剧这种,rid在子菜单的url中,此外返回的data中含有页数相关信息,可以据此定义爬取的页面数量

 # -*- coding: utf-8 -*-
 # @author: Tele
 # @Time  : 2019/04/08 下午 1:01
 import requests
 import json
 import os
 import re
 import shutil
 from lxml import etree

 # 爬取每个菜单的前5页内容
 class BiliSplider:
     def __init__(self, save_dir, menu_list):
         self.target = menu_list
         self.url_temp = "https://www.bilibili.com/"
         self.headers = {
             "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
          #  "Cookie": "LIVE_BUVID=AUTO6715546997211617; buvid3=07192BD6-2288-4BA5-9259-8E0BF6381C9347193infoc; stardustvideo=1; CURRENT_FNVAL=16; sid=l0fnfa5e; rpdid=bfAHHkDF:cq6flbmZ:Ohzhw:1Hdog8",
         }
         self.proxies = {
             "http": "http://61.190.102.50:15845"
         }
         self.father_dir = save_dir

     def get_menu_list(self):
         regex = re.compile("//")
         response = requests.get(self.url_temp, headers=self.headers)
         html_element = etree.HTML(response.content)
         nav_menu_list = html_element.xpath("//div[@id='primary_menu']/ul[@class='nav-menu']/li/a")

         menu_list = list()
         for item in nav_menu_list:
             menu = dict()
             title = item.xpath("./*/text()")
             menu["title"] = title[0] if len(title) > 0 else None
             href = item.xpath("./@href")
             menu["href"] = "https://" + regex.sub("", href[0]) if len(href) > 0 else None

             # 子菜单
             submenu_list = list()
             sub_nav_list = item.xpath("./../ul[@class='sub-nav']/li")
             if len(sub_nav_list) > 0:
                 for sub in sub_nav_list:
                     submenu = dict()
                     sub_title = sub.xpath("./a/span/text()")
                     submenu["title"] = sub_title[0] if len(sub_title) > 0 else None
                     sub_href = sub.xpath("./a/@href")
                     submenu["href"] = "https://" + regex.sub("", sub_href[0]) if len(sub_href) > 0 else None
                     submenu_list.append(submenu)
             menu["submenu_list"] = submenu_list if len(submenu_list) > 0 else None
             menu_list.append(menu)
         return menu_list

     # rid=tid
     def parse_index_url(self, url):
         result_list = list()
         # 正则匹配
         regex = re.compile("<script>window.__INITIAL_STATE__=(.*)</script>")
         response = requests.get(url, headers=self.headers)
         result = regex.findall(response.content.decode())
         temp = re.compile("(.*);\(function").findall(result[0]) if len(result) > 0 else None
         sub_list = json.loads(temp[0])["config"]["sub"] if temp else list()
         if len(sub_list) > 0:
             for sub in sub_list:
                 # 一些子菜单没有rid,需要请求不同的url,暂不处理
                 if "tid" in sub:
                     if sub["tid"]:
                         sub_menu = dict()
                         sub_menu["rid"] = sub["tid"] if sub["tid"] else None
                         sub_menu["title"] = sub["name"] if sub["name"] else None
                         result_list.append(sub_menu)
                 else:
                     pass

         return result_list

     # 最新动态  region?callback
     # 数据 newlist?callback
     def parse_sub_url(self, item):
         self.headers["Referer"] = item["referer"]
         url_pattern = "https://api.bilibili.com/x/web-interface/newlist?rid={}&type=0&pn={}&ps=20"

         # 每个菜单爬取前5页
         for i in range(1, 6):
             data = dict()
             url = url_pattern.format(item["rid"], i)
             print(url)
             try:
                 response = requests.get(url, headers=self.headers, proxies=self.proxies, timeout=10)
             except:
                 return
             if response.status_code == 200:
                 data["content"] = json.loads(response.content.decode())["data"]
                 data["title"] = item["title"]
                 data["index"] = i
                 data["menu"] = item["menu"]
                 # 保存数据
                 self.save_data(data)
             else:
                 print("请求超时")  # 一般是403,被封IP了

     def save_data(self, data):
         if len(data["content"]) == 0:
             return
         parent_path = self.father_dir + "/" + data["menu"] + "/" + data["title"]
         if not os.path.exists(parent_path):
             os.makedirs(parent_path)
         file_dir = parent_path + "/" + "第" + str(data["index"]) + "页.txt"

         # 保存
         with open(file_dir, "w", encoding="utf-8") as file:
             file.write(json.dumps(data["content"], ensure_ascii=False, indent=2))

     def run(self):
         # 清除之前保存的数据
         if os.path.exists(self.father_dir):
             shutil.rmtree(self.father_dir)

         menu_list = self.get_menu_list()
         menu_info = list()
         # 获得目标菜单信息
         # 特殊列表,一些菜单的rid必须从子菜单的url中获得
         special_list = list()
         for menu in menu_list:
             for t in self.target:
                 if menu["title"] == t:
                     if menu["title"] == "番剧" or menu["title"] == "国创" or menu["title"] == "影视":
                         special_list.append(menu)
                     menu_info.append(menu)
                     break

         # 目标菜单的主页
         if len(menu_info) > 0:
             for info in menu_info:
                 menu_index_url = info["href"]
                 # 处理特殊列表
                 if info in special_list:
                     menu_index_url = info["submenu_list"][0]["href"]
                 # 获得rid
                 result_list = self.parse_index_url(menu_index_url)
                 print(result_list)
                 if len(result_list) > 0:
                     for item in result_list:
                         # 大菜单
                         item["menu"] = info["title"]
                         item["referer"] = menu_index_url
                         # 爬取子菜单
                         self.parse_sub_url(item)

 def main():
     target = ["动画", "番剧", "国创", "音乐", "舞蹈", "游戏", "科技", "数码", "生活", "鬼畜", "时尚", "广告", "娱乐", "影视"]
     splider = BiliSplider("f:/bili_splider", target)
     splider.run()

 if __name__ == '__main__':
     main()

可以看到番剧少了新番时间表与番剧索引,因为这两个请求不遵循https://api.bilibili.com/x/web-interface/newlist?rid={}&type=0&pn={}&ps=20的格式,类似的不再赘述