Python——selenium爬取斗鱼房间信息

 from selenium import webdriver
 import os
 import json
 import time

 class Douyu:
     def __init__(self):
         # 1.发送首页的请求
         self.driver = webdriver.Chrome('../chromedriver.exe')
         self.driver.get('https://www.douyu.com/g_yz')

     # 获取页面内容
     def get_content(self):
         time.sleep(3)
         li_list = self.driver.find_elements_by_xpath("//*[@class='layout-Cover-list']/li[@class='layout-Cover-item']")
         length = len(li_list)
         contents = []

         # 遍历房间列表
         for i in range(length):
             item = {}
             item['主播'] = self.driver.find_elements_by_xpath('//h2[@class="DyListCover-user"]')[i].text
             item['房间名'] = self.driver.find_elements_by_xpath('//h3[@class="DyListCover-intro"]')[i].get_attribute(
                 'title')
             item['热度'] = self.driver.find_elements_by_xpath('//span[@class="DyListCover-hot"]')[i].text
             item['封面'] = self.driver.find_elements_by_class_name('DyImg-content')[i].get_attribute(
                 'src')
             contents.append(item)
         return contents

     # 保存数据到本地
     def save_content(self, contents):
         with open('douyu.json', 'a',encoding='utf-8') as f:
             for content in contents:
                 json.dump(content, f, ensure_ascii=False, indent=2)
                 f.write(os.linesep)

     def run(self):
         # 1.发送首页的请求：初始化时已经发送请求
         # 2.获取第一页的数据
         contents = self.get_content()
         self.save_content(contents)

         # 3.循环:点击下一页按钮,只要没有下一页的按钮
         while self.driver.find_elements_by_class_name('dy-Pagination-item-custom')[1]:
             # 点击下一页的按钮
             self.driver.find_elements_by_class_name('dy-Pagination-item-custom')[1].click()
             # 4.继续获取下一页的内容
             contents = self.get_content()
             # 4.保存下一页保存内容
             self.save_content(contents)

 if __name__ == '__main__':
     douyu = Douyu()
     douyu.run()