Python 爬虫 (五)

 # 头条街拍图片爬取

 1 import re
 import requests
 from urllib import request
 import json
 import os
 i = 0
 headers = {
     'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
 }
 while True:
     pag_all_url = 'https://www.toutiao.com/search_content/?offset={}&format=json&keyword=%E8%A1%97%E6%8B%8D&autoload=true&count=20&cur_tab=1&from=search_tab'
     i += 20
     full_pag_url = pag_all_url.format(i)
     # print(full_pag_url) # 街拍的首页 根据不同的i来请求ajax,从而获得所有的街拍网址 像这样https://www.toutiao.com/search/?keyword=%E8%A1%97%E6%8B%8D
     pag_html = requests.get(full_pag_url,headers = headers).text
     pag_html_str = str(json.loads(pag_html))
     # print(pag_html_str) #把网页转化为字符串 进行正则匹配
     img_pag_id = re.findall(r'\'item_source_url\': \'\/group\/(\d*)\/\',',pag_html_str)
     # print(img_pag_id) #获得每个街拍的url like this--->https://www.toutiao.com/a6590127156037157379/
     for l in img_pag_id:    #图片下载
         img_all_url = 'https://www.toutiao.com/a{}'
         full_url = img_all_url.format(l)
         # print(full_url)#图片的url  print(full_pag_url)#图片所在的url
         html = requests.get(full_url,headers=headers).text
         pattern = r'gallery: JSON\.parse\((.*)\),'
         ans1 = re.search(pattern,html)
         try:
             ans1_str = json.loads(ans1[1])
             ans1_dic = json.loads(ans1_str)
             # print(ans1_dic)
             # if not os.path.exists('1'):
             #     os.mkdir('1')
             for q in ans1_dic['sub_images']:
                 img_url = q['url']
                 print(img_url)
                 filename = '1/' + img_url.split('/')[-1] + '.jpg'
                 request.urlretrieve(img_url, filename)
         except:continue