爬虫实例之使用requests和Beautifusoup爬取糗百热门用户信息

这次主要用requests库和Beautifusoup库来实现对糗百的热门帖子的用户信息的收集，由于糗百的反爬虫不是很严格，也不需要先登录才能获取数据，所以较简单。

思路，先请求首页的热门帖子获得用户详情链接，然后请求用户详情页，用Beautifusoup解析得到基本的用户信息

代码：

 #!/usr/bin/env python
 # -*- coding:utf-8 -*-
 import requests
 import json
 import urllib3
 import pymongo
 from bs4 import BeautifulSoup

 urllib3.disable_warnings()

 class Qsbk():
     def __init__(self,url):
         self.url = url # 糗事百科首页热门帖子链接
         self.base_url = 'https://www.qiushibaike.com' # 用于用户链接拼接
         self.user_info = {} # 保存用户信息
         self.headers = {
             'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36'
         } # 请求头信息
         self.proxies = {
             'http':'192.168.105.71:80',
             'https':'192.168.105.71:80'
         } # 使用代理
         self.file = open('qsbk.json','a',encoding='utf-8') # 把用户信息保存到json文本中，以追加的形式

     def get_data(self):
         try:
             response = requests.get(url=self.url,headers=self.headers,proxies=self.proxies,verify=False).text  # 请求糗百热门页面
             soup = BeautifulSoup(response,'lxml')
             node_list = soup.find_all('div',class_='author clearfix')
             for node in node_list:
                 item = node.find('a')
                 if item != None:  # 某些匿名帖子没有用户链接，需要排除掉
                     link = item.get('href') # 得到用户详情链接
                     user_link = self.base_url + link
                     # print(user_link)
                     self.parse_data(user_link) # 请求详情页并解析得到用户信息
         except Exception as e:
             print(e)

     def parse_data(self,user_link):
         try:
             result = requests.get(user_link,headers=self.headers,proxies=self.proxies,verify=False).content.decode()
             soup = BeautifulSoup(result,'lxml')
             status = soup.find('h3').get_text()
             if '当前用户已关闭糗百个人动态' in status:  # 排除关闭糗百的用户
                 pass
             else:
                 username = soup.find('h2').get_text()
                 self.user_info['用户名'] = username
                 node_list = soup.find_all('div',class_="user-statis user-block")
                 fans = node_list[0].select('ul > li')[0].get_text().split(':',1)[-1] #得到是“粉丝数：111”类似的数据，需要先用":"分割然后得到具体的数字，后续信息类似
                 concerns = node_list[0].select('ul > li')[1].get_text().split(':',1)[-1]
                 comments = node_list[0].select('ul > li')[3].get_text().split(':',1)[-1]
                 coins = node_list[0].select('ul > li')[4].get_text().split(':',1)[-1]
                 marriage = node_list[1].select('ul > li')[0].get_text().split(':',1)[-1]
                 job = node_list[1].select('ul > li')[2].get_text().split(':',1)[-1]
                 web_age = node_list[1].select('ul > li')[4].get_text().split(':',1)[-1]
                 self.user_info['粉丝数'] = fans
                 self.user_info['关注数'] = concerns
                 self.user_info['评论'] = comments
                 self.user_info['笑脸'] = coins
                 self.user_info['婚姻'] = marriage
                 self.user_info['职业'] = job
                 self.user_info['糗龄'] = web_age
                 # print(self.user_info)
                 self.save_json() # 保存数据到json文件中
                 self.save_mongo() # 保存到MongoDB中
         except Exception as e:
             print(e)

     def save_json(self):
         content = json.dumps(self.user_info,ensure_ascii=False) + '\n'  # 每行结束后换行
         self.file.write(content)
         self.file.close()

     def save_mongo(self):
         try:
             self.client = pymongo.MongoClient(host='localhost', port=27107)
             self.col = self.client['qsbk']['qsbk']
             self.col.insert(self.user_info)
         except Exception as e:
             print(e)

 if __name__ == '__main__':
     for i in range(1,3):
         url = 'https://www.qiushibaike.com/8hr/page/{}/'.format(i)
         # print(url)
         qsbk = Qsbk(url)
         qsbk.get_data()