2.分析Ajax请求并抓取今日头条街拍美图
import requests
from urllib.parse import urlencode
# 引入异常类
from requests.exceptions import RequestException
# 保存文件时用到json格式
import json
# 正则表达式
import re def get_page_index(offset,keyword):
data = {
'autoload':'true',
'count':'',
'cur_tab':,
'format':'json',
'from':'search_tab',
'keyword':keyword,
'offset':offset
}
url = 'https://www.toutiao.com/search_content/?'+ urlencode(data)
try:
response = requests.get(url)
if response.status_code == :
return response.text
return None
except RequestException:
print('请求索引页出错')
return None
def parse_page_index(html):
data = json.loads(html)
if data and 'data' in data.keys():
for item in data.get('data'):
yield item.get('article_url')
def get_page_detail(url):
try:
response = requests.get(url)
if response.status_code == 200:
return response.text
return None
except RequestException:
print('请求详情页出错',url)
return None
def parse_page_detail(html):
soup = bs(html,'lxml')
title = soup.select('title')[0].get_text()
print(title)
images_pattern = re.compile('gallery: JSON.parse("(.*?);',re.S)
BASE_DATA.galleryInfo = {
title: '路人街拍，个子不高，穿得却很有范，而且时尚潮流',
isOriginal: false,
mediaInfo: BASE_DATA.mediaInfo,
gallery: JSON.parse(
"{\"count\":5,\"sub_images\":[{\"url\":\"http:\\/\\/p3.pstatp.com\\/origin\\/pgc-image\\/15324800265505b953ab972\",\"width\":640,\"url_list\":[{\"url\":\"http:\\/\\/p3.pstatp.com\\/origin\\/pgc-image\\/15324800265505b953ab972\"},{\"url\":\"http:\\/\\/pb9.pstatp.com\\/origin\\/pgc-image\\/15324800265505b953ab972\"},{\"url\":\"http:\\/\\/pb1.pstatp.com\\/origin\\/pgc-image\\/15324800265505b953ab972\"}],\"uri\":\"origin\\/pgc-image\\/15324800265505b953ab972\",\"height\":959},{\"url\":\"http:\\/\\/p3.pstatp.com\\/origin\\/pgc-image\\/1532480004156480c21a63e\",\"width\":640,\"url_list\":[{\"url\":\"http:\\/\\/p3.pstatp.com\\/origin\\/pgc-image\\/1532480004156480c21a63e\"},{\"url\":\"http:\\/\\/pb9.pstatp.com\\/origin\\/pgc-image\\/1532480004156480c21a63e\"},{\"url\":\"http:\\/\\/pb1.pstatp.com\\/origin\\/pgc-image\\/1532480004156480c21a63e\"}],\"uri\":\"origin\\/pgc-image\\/1532480004156480c21a63e\",\"height\":1011},{\"url\":\"http:\\/\\/p3.pstatp.com\\/origin\\/pgc-image\\/1532480016958bdc65dea62\",\"width\":640,\"url_list\":[{\"url\":\"http:\\/\\/p3.pstatp.com\\/origin\\/pgc-image\\/1532480016958bdc65dea62\"},{\"url\":\"http:\\/\\/pb9.pstatp.com\\/origin\\/pgc-image\\/1532480016958bdc65dea62\"},{\"url\":\"http:\\/\\/pb1.pstatp.com\\/origin\\/pgc-image\\/1532480016958bdc65dea62\"}],\"uri\":\"origin\\/pgc-image\\/1532480016958bdc65dea62\",\"height\":929},{\"url\":\"http:\\/\\/p9.pstatp.com\\/origin\\/pgc-image\\/15324800362354f9d1c3ec1\",\"width\":640,\"url_list\":[{\"url\":\"http:\\/\\/p9.pstatp.com\\/origin\\/pgc-image\\/15324800362354f9d1c3ec1\"},{\"url\":\"http:\\/\\/pb1.pstatp.com\\/origin\\/pgc-image\\/15324800362354f9d1c3ec1\"},{\"url\":\"http:\\/\\/pb3.pstatp.com\\/origin\\/pgc-image\\/15324800362354f9d1c3ec1\"}],\"uri\":\"origin\\/pgc-image\\/15324800362354f9d1c3ec1\",\"height\":960},{\"url\":\"http:\\/\\/p3.pstatp.com\\/origin\\/pgc-image\\/15324800476353e6c3e0c25\",\"width\":640,\"url_list\":[{\"url\":\"http:\\/\\/p3.pstatp.com\\/origin\\/pgc-image\\/15324800476353e6c3e0c25\"},{\"url\":\"http:\\/\\/pb9.pstatp.com\\/origin\\/pgc-image\\/15324800476353e6c3e0c25\"},{\"url\":\"http:\\/\\/pb1.pstatp.com\\/origin\\/pgc-image\\/15324800476353e6c3e0c25\"}],\"uri\":\"origin\\/pgc-image\\/15324800476353e6c3e0c25\",\"height\":986}],\"max_img_width\":640,\"labels\":[\"\\u65f6\\u88c5\\u642d\\u914d\"],\"sub_abstracts\":[\"\\u8def\\u4eba\\u8857\\u62cd\\uff0c\\u4e2a\\u5b50\\u4e0d\\u9ad8\\uff0c\\u7a7f\\u5f97\\u5374\\u5f88\\u6709\\u8303\\uff0c\\u800c\\u4e14\\u65f6\\u5c1a\\u6f6e\\u6d41\",\" \",\" \",\" \",\" \"],\"sub_titles\":[\"\\u8def\\u4eba\\u8857\\u62cd\\uff0c\\u4e2a\\u5b50\\u4e0d\\u9ad8\\uff0c\\u7a7f\\u5f97\\u5374\\u5f88\\u6709\\u8303\\uff0c\\u800c\\u4e14\\u65f6\\u5c1a\\u6f6e\\u6d41\",\"\\u8def\\u4eba\\u8857\\u62cd\\uff0c\\u4e2a\\u5b50\\u4e0d\\u9ad8\\uff0c\\u7a7f\\u5f97\\u5374\\u5f88\\u6709\\u8303\\uff0c\\u800c\\u4e14\\u65f6\\u5c1a\\u6f6e\\u6d41\",\"\\u8def\\u4eba\\u8857\\u62cd\\uff0c\\u4e2a\\u5b50\\u4e0d\\u9ad8\\uff0c\\u7a7f\\u5f97\\u5374\\u5f88\\u6709\\u8303\\uff0c\\u800c\\u4e14\\u65f6\\u5c1a\\u6f6e\\u6d41\",\"\\u8def\\u4eba\\u8857\\u62cd\\uff0c\\u4e2a\\u5b50\\u4e0d\\u9ad8\\uff0c\\u7a7f\\u5f97\\u5374\\u5f88\\u6709\\u8303\\uff0c\\u800c\\u4e14\\u65f6\\u5c1a\\u6f6e\\u6d41\",\"\\u8def\\u4eba\\u8857\\u62cd\\uff0c\\u4e2a\\u5b50\\u4e0d\\u9ad8\\uff0c\\u7a7f\\u5f97\\u5374\\u5f88\\u6709\\u8303\\uff0c\\u800c\\u4e14\\u65f6\\u5c1a\\u6f6e\\u6d41\"]}"),
siblingList: [{"comments_count": 72, "media_avatar_url": "//p3.pstatp.com/large/8b6000041e42267916a3",
"is_feed_ad": false, "is_diversion_page": false, "title": "街拍路人，高跟控的时尚穿搭参考，让你展现与众不同的时尚个性",
"single_mode": true, "gallary_image_count": 7, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6577212971246682628/",
"source": "时尚街拍Dog", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p9.pstatp.com/list/300x170/pgc-image/1531376421040010f39e595",
"group_id": "6577212971246682628", "is_related": true, "media_url": "/c/user/5441627343/"},
{"comments_count": 36, "media_avatar_url": "//p6.pstatp.com/large/382f0005b0e7dbc47d8d",
"is_feed_ad": false, "is_diversion_page": false, "title": "北京街拍，都是乱穿衣的时候，怎么好看就怎么穿",
"single_mode": true, "gallary_image_count": 10, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6562767784663581191/",
"source": "海玲时尚", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p1.pstatp.com/list/300x170/pgc-image/1528013395598fac4c4efe1",
"group_id": "6562767784663581191", "is_related": true, "media_url": "/c/user/58444595361/"},
{"comments_count": 9, "media_avatar_url": "//p3.pstatp.com/large/8b6000041e42267916a3",
"is_feed_ad": false, "is_diversion_page": false, "title": "街拍路人，高贵洋气的潮流穿搭参考，让你展现高贵女神气质",
"single_mode": true, "gallary_image_count": 7, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6570846557128622600/",
"source": "时尚街拍Dog", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p3.pstatp.com/list/300x170/pgc-image/1529894002540131cde7818",
"group_id": "6570846557128622600", "is_related": true, "media_url": "/c/user/5441627343/"},
{"comments_count": 16, "media_avatar_url": "//p1.pstatp.com/large/6c310002c52d08feb562",
"is_feed_ad": false, "is_diversion_page": false, "title": "街拍路人，时尚又洋气的穿搭参考，让你气场变大气质飙升",
"single_mode": true, "gallary_image_count": 7, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6570524966184288776/",
"source": "六六大顺1286", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p3.pstatp.com/list/300x170/pgc-image/152981831396458c08bc0c3",
"group_id": "6570524966184288776", "is_related": true, "media_url": "/c/user/81174905516/"},
{"comments_count": 8, "media_avatar_url": "//p1.pstatp.com/large/6c310002c52d08feb562",
"is_feed_ad": false, "is_diversion_page": false, "title": "街拍路人，时尚又减龄的穿搭参考，让你看起来至少年轻好几岁",
"single_mode": true, "gallary_image_count": 8, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6566070892542034445/",
"source": "六六大顺1286", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p1.pstatp.com/list/300x170/pgc-image/1528782064341c24abc274f",
"group_id": "6566070892542034445", "is_related": true, "media_url": "/c/user/81174905516/"},
{"comments_count": 15, "media_avatar_url": "//p3.pstatp.com/large/6eed0004f9299624bf43",
"is_feed_ad": false, "is_diversion_page": false, "title": "路人街拍，美如花开的时尚穿搭参考，让你美如鲜花盛开！",
"single_mode": true, "gallary_image_count": 5, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6576006257222615556/",
"source": "天天美女图", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p9.pstatp.com/list/300x170/pgc-image/1531095687103b57abfdf57",
"group_id": "6576006257222615556", "is_related": true, "media_url": "/c/user/98918905833/"},
{"comments_count": 12, "media_avatar_url": "//p1.pstatp.com/large/6c310002c52d08feb562",
"is_feed_ad": false, "is_diversion_page": false, "title": "街拍路人，时下最新潮的穿搭参考，让你穿出超越女神的气质",
"single_mode": true, "gallary_image_count": 7, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6577224195824943620/",
"source": "六六大顺1286", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p3.pstatp.com/list/300x170/pgc-image/1531378675945f7e5f3dc4e",
"group_id": "6577224195824943620", "is_related": true, "media_url": "/c/user/81174905516/"},
{"comments_count": 23, "media_avatar_url": "//p6.pstatp.com/large/382f0005b0e7dbc47d8d",
"is_feed_ad": false, "is_diversion_page": false, "title": "北京街拍，不对称西服外套与白衬衣的叠穿，更显层次感",
"single_mode": true, "gallary_image_count": 10, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6552352941250970126/",
"source": "海玲时尚", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p3.pstatp.com/list/300x170/pgc-image/1525588437307f582c869c9",
"group_id": "6552352941250970126", "is_related": true, "media_url": "/c/user/58444595361/"},
{"comments_count": 8, "media_avatar_url": "//p1.pstatp.com/large/8b610003fea5104fa804",
"is_feed_ad": false, "is_diversion_page": false, "title": "街拍路人，黑色T永不过时的穿搭参考，让你穿出迷人青春范",
"single_mode": true, "gallary_image_count": 7, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6572015578561643012/",
"source": "每日分享社", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p9.pstatp.com/list/300x170/pgc-image/1530166166448957eb30a67",
"group_id": "6572015578561643012", "is_related": true, "media_url": "/c/user/52415621433/"},
{"comments_count": 14, "media_avatar_url": "//p3.pstatp.com/large/6eed0004f5928763adad",
"is_feed_ad": false, "is_diversion_page": false, "title": "路人街拍，时尚大气的休闲穿搭参考，让你够酷，够美，够大气！",
"single_mode": true, "gallary_image_count": 6, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6575640177157341699/",
"source": "天天美女照", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p9.pstatp.com/list/300x170/pgc-image/1531010389181b3166e603a",
"group_id": "6575640177157341699", "is_related": true, "media_url": "/c/user/6686213364/"},
{"comments_count": 42, "media_avatar_url": "//p3.pstatp.com/large/6eed0002747fdb857784",
"is_feed_ad": false, "is_diversion_page": false, "title": "街拍北京：个子高挑，衣品时尚，搭配潮流，长得好看！",
"single_mode": true, "gallary_image_count": 6, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6581196161003029000/",
"source": "时尚好看19", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p1.pstatp.com/list/300x170/pgc-image/1532304059555dc3da79df0",
"group_id": "6581196161003029000", "is_related": true, "media_url": "/c/user/98512844552/"},
{"comments_count": 7, "media_avatar_url": "//p3.pstatp.com/large/5e7c0000cc291f5e42d3",
"is_feed_ad": false, "is_diversion_page": false, "title": "路人街拍：约会简约搭配方式，纯色连衣裙搭配平底尖脚鞋气质出众",
"single_mode": true, "gallary_image_count": 7, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6577348572038038019/",
"source": "Fashion第六感", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p3.pstatp.com/list/300x170/pgc-image/1531408080685595da561c6",
"group_id": "6577348572038038019", "is_related": true, "media_url": "/c/user/6236329656/"},
{"comments_count": 17, "media_avatar_url": "//p3.pstatp.com/large/6eed0004f9299624bf43",
"is_feed_ad": false, "is_diversion_page": false, "title": "路人街拍，真心养眼的时尚穿搭参考，让你穿的好看，美的养眼！",
"single_mode": true, "gallary_image_count": 5, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6575811480208278030/",
"source": "天天美女图", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p3.pstatp.com/list/300x170/pgc-image/1531050304667eb82ae53fb",
"group_id": "6575811480208278030", "is_related": true, "media_url": "/c/user/98918905833/"},
{"comments_count": 2, "media_avatar_url": "//p3.pstatp.com/large/6eed0004f5928763adad",
"is_feed_ad": false, "is_diversion_page": false, "title": "路人街拍，一身优雅的穿搭，穿出女人端庄的气质",
"single_mode": true, "gallary_image_count": 6, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6579159866898645507/",
"source": "天天美女照", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p3.pstatp.com/list/300x170/pgc-image/1531829948291cec7f1e419",
"group_id": "6579159866898645507", "is_related": true, "media_url": "/c/user/6686213364/"},
{"comments_count": 2, "media_avatar_url": "//p3.pstatp.com/large/6eed0004f9299624bf43",
"is_feed_ad": false, "is_diversion_page": false, "title": "路人街拍，尽显优雅绝伦的时尚穿搭参考，让你美的令人一见倾心",
"single_mode": true, "gallary_image_count": 5, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6575441122204582404/",
"source": "天天美女图", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p1.pstatp.com/list/300x170/pgc-image/1530964112002b59991e98e",
"group_id": "6575441122204582404", "is_related": true, "media_url": "/c/user/98918905833/"},
{"comments_count": 9, "media_avatar_url": "//p1.pstatp.com/large/888e0001ca2d8afc54d4",
"is_feed_ad": false, "is_diversion_page": false, "title": "街拍，休闲+减龄的气质女生夏天穿衣搭配，日常穿搭秒杀路人",
"single_mode": true, "gallary_image_count": 5, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6579356829992616455/",
"source": "街拍美女图", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p9.pstatp.com/list/300x170/pgc-image/1531875794891cbda4529a6",
"group_id": "6579356829992616455", "is_related": true, "media_url": "/c/user/98867532852/"},
{"comments_count": 32, "media_avatar_url": "//p3.pstatp.com/large/6eea0002dbb5ffffa1f1",
"is_feed_ad": false, "is_diversion_page": false, "title": "女生街拍，黄金比例闺蜜，穿黑色凉鞋气质真好！",
"single_mode": true, "gallary_image_count": 10, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6580886941489693191/",
"source": "kk潮品街拍", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p3.pstatp.com/list/300x170/pgc-image/15322320709553059d33c7c",
"group_id": "6580886941489693191", "is_related": true, "media_url": "/c/user/98013496587/"},
{"comments_count": 22, "media_avatar_url": "//p3.pstatp.com/large/8b61000302495b74702e",
"is_feed_ad": false, "is_diversion_page": false, "title": "路人街拍，年轻就是本钱，穿什么都充满时尚美！",
"single_mode": true, "gallary_image_count": 5, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6577151772521923086/",
"source": "天天街拍图", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p9.pstatp.com/list/300x170/pgc-image/1531362350533dbf5878e98",
"group_id": "6577151772521923086", "is_related": true, "media_url": "/c/user/100250154069/"},
{"comments_count": 7, "media_avatar_url": "//p3.pstatp.com/large/6eed0004f9299624bf43",
"is_feed_ad": false, "is_diversion_page": false, "title": "路人街拍，女人要会穿才显漂亮，夏季这样穿搭让你尽显迷人气质美",
"single_mode": true, "gallary_image_count": 5, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6578614959142339075/",
"source": "天天美女图", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p3.pstatp.com/list/300x170/pgc-image/1531702883449de4a8be734",
"group_id": "6578614959142339075", "is_related": true, "media_url": "/c/user/98918905833/"},
{"comments_count": 7, "media_avatar_url": "//p3.pstatp.com/large/6eed0004f5928763adad",
"is_feed_ad": false, "is_diversion_page": false, "title": "路人街拍，好身材也要会穿搭，这样穿让你气质迷人",
"single_mode": true, "gallary_image_count": 5, "middle_mode": false, "has_video": false,
"video_duration_str": null, "source_url": "https://www.toutiao.com/group/6575273365118910983/",
"source": "天天美女照", "more_mode": null, "article_genre": "gallery", "has_gallery": false,
"video_play_count": 0,
"image_url": "//p1.pstatp.com/list/300x170/pgc-image/153092495402580bddfcc1d",
"group_id": "6575273365118910983", "is_related": true, "media_url": "/c/user/6686213364/"}],
publish_time: '2018-07-25 08:55:51',
group_id: '6581952133069799949',
item_id: '6581952133069799949',
share_url: 'https://m.toutiao.com/group/6581952133069799949/',
abstract: ''.replace( / < br \ / > / ig, ''),
repin: 0
}
def main():
html = get_page_index(0,'街拍')
# print(html)
{
"count":20,
"action_label":"click_search",
"return_count":20,
"has_more":1,
"page_id":"/search/",
"request_id":"20180725214120172016032036766212",
"cur_tab":1,
"tokens":[
"街拍"
],
"tab":{
"tab_list":[
{
"tab_name":"综合",
"tab_id":1,
"tab_code":"news"
},
{
"tab_name":"视频",
"tab_id":2,
"tab_code":"video"
},
{
"tab_name":"图集",
"tab_id":3,
"tab_code":"gallery"
},
{
"tab_name":"用户",
"tab_id":4,
"tab_code":"pgc"
},
{
"tab_name":"问答",
"tab_id":5,
"tab_code":"wenda"
}
],
"cur_tab":1
},
"offset":20,
"action_label_web":"click_search",
"show_tabs":1,
"data":[
{
"merge_user":[
{
"screen_name":"她街拍",
"verified_content":"",
"media_id":"1583556350618637",
"description":"全国时尚女性的聚集地。",
"user_id":"75166641448",
"gender":"2",
"follow_count":50336,
"user_verified":"0",
"user_type":"1",
"is_pgc":1,
"user_auth_info":{
"auth_type":2,
"auth_info":"四川数赢信息技术有限公司官方头条号"
},
"avatar_url":"http://p3.pstatp.com/medium/83500012b978513e79a4",
"create_time":"1510196395",
"user_decoration":{
},
"highlight":{
"name":[
[
1,
2
]
]
},
"source_url":"http://www.toutiao.com/c/user/75166641448/",
"verify_content":"四川数赢信息技术有限公司官方头条号",
"id":"75166641448",
"name":"她街拍"
},
{
"screen_name":"街拍滚叔",
"verified_content":"",
"media_id":"5909205287",
"description":"分享每日杭州街拍及全球时尚资讯。",
"user_id":"5909205287",
"gender":"1",
"follow_count":284367,
"user_verified":"0",
"user_type":"1",
"is_pgc":1,
"user_auth_info":{
"auth_type":"0",
"auth_info":"时尚达人"
},
"avatar_url":"http://p3.pstatp.com/medium/12236/4694012685",
"create_time":"1452827046",
"user_decoration":{
},
"highlight":{
"name":[
[
0,
2
]
]
},
"source_url":"http://www.toutiao.com/c/user/5909205287/",
"verify_content":"时尚达人",
"id":"5909205287",
"name":"街拍滚叔"
},
{
"screen_name":"暴走街拍",
"verified_content":"",
"media_id":"5826291222",
"description":"暴走街拍主要以采访高颜值对象为主，风格青春向上，话题有趣贴近生活。",
"user_id":"5826291222",
"gender":"0",
"follow_count":3276997,
"user_verified":"0",
"user_type":"1",
"is_pgc":1,
"user_auth_info":{
"auth_type":"0",
"auth_info":"搞笑自媒体"
},
"avatar_url":"http://p1.pstatp.com/medium/10687/2312895944",
"create_time":"1450370560",
"user_decoration":{
},
"highlight":{
"name":[
[
2,
2
]
]
},
"source_url":"http://www.toutiao.com/c/user/5826291222/",
"verify_content":"搞笑自媒体",
"id":"5826291222",
"name":"暴走街拍"
},
{
"screen_name":"皇城根五爷原创街拍",
"verified_content":"",
"media_id":"6682998606",
"description":"人像捉影 人文纪实 旅游风光 徐飞原创街拍",
"user_id":"6685177416",
"gender":"1",
"follow_count":26649,
"user_verified":"0",
"user_type":"1",
"is_pgc":1,
"user_auth_info":{
"auth_type":"0",
"auth_info":"街拍达人"
},
"avatar_url":"http://p3.pstatp.com/medium/6cb000fd8de5eec5814",
"create_time":"1465346690",
"user_decoration":{
},
"highlight":{
"name":[
[
7,
2
]
]
},
"source_url":"http://www.toutiao.com/c/user/6685177416/",
"verify_content":"街拍达人",
"id":"6685177416",
"name":"皇城根五爷原创街拍"
},
{
"screen_name":"3A街拍",
"verified_content":"",
"media_id":"1570911765335041",
"description":"",
"user_id":"62376452948",
"gender":"2",
"follow_count":38868,
"no_display_pgc_icon":"0",
"user_type":"1",
"is_pgc":1,
"user_auth_info":{
"auth_type":"0",
"auth_info":"摄影达人"
},
"avatar_url":"http://p1.pstatp.com/medium/289a001a1b2868d356ea",
"create_time":"1498137829",
"user_decoration":{
},
"highlight":{
"name":[
[
2,
2
]
]
},
"source_url":"http://www.toutiao.com/c/user/62376452948/",
"verify_content":"摄影达人",
"id":"62376452948",
"user_verified":"0",
"name":"3A街拍"
},
{
"screen_name":"粤B街拍",
"verified_content":"",
"media_id":"6212033896",
"description":"大帅比和小美腻の聚集地。",
"user_id":"6203908847",
"gender":"0",
"follow_count":127757,
"user_verified":"0",
"user_type":"1",
"is_pgc":1,
"user_auth_info":{
"auth_type":"0",
"other_auth":{
"pgc":"头条摄影原创作者"
},
"auth_info":"头条摄影原创作者"
},
"avatar_url":"http://p3.pstatp.com/medium/2b7002107206ebc4a7e",
"create_time":"1459130707",
"user_decoration":{
},
"highlight":{
"name":[
[
2,
2
]
]
},
"source_url":"http://www.toutiao.com/c/user/6203908847/",
"verify_content":"头条摄影原创作者",
"id":"6203908847",
"name":"粤B街拍"
},
{
"screen_name":"街拍潮流",
"media_id":"6933588826",
"description":"推送最新的街拍，引领时尚潮流！",
"user_id":"6933678811",
"gender":"0",
"follow_count":13881,
"no_display_pgc_icon":"0",
"user_type":"1",
"is_pgc":1,
"verified_content":"",
"avatar_url":"http://p1.pstatp.com/medium/97e000cd69a309bbb59",
"create_time":"1468561349",
"user_decoration":{
},
"highlight":{
"name":[
[
0,
2
]
]
},
"source_url":"http://www.toutiao.com/c/user/6933678811/",
"id":"6933678811",
"user_verified":"0",
"name":"街拍潮流"
},
{
"screen_name":"街拍天下",
"media_id":"51026951312",
"description":"资深街拍，专注街拍，分享街拍之美！",
"user_id":"50965704962",
"gender":"1",
"follow_count":17097,
"user_verified":"0",
"user_type":"1",
"is_pgc":1,
"verified_content":"",
"avatar_url":"http://p1.pstatp.com/medium/e580006e8cd68515a95",
"create_time":"1474249162",
"user_decoration":{
},
"highlight":{
"name":[
[
0,
2
]
]
},
"source_url":"http://www.toutiao.com/c/user/50965704962/",
"id":"50965704962",
"name":"街拍天下"
},
{
"screen_name":"街拍美女",
"media_id":"50015024950",
"description":"爱街拍，展现时尚自然美！",
"user_id":"50018626654",
"gender":"0",
"follow_count":31361,
"user_verified":"0",
"user_type":"1",
"is_pgc":1,
"verified_content":"",
"avatar_url":"http://p3.pstatp.com/medium/97e0016cb071b0c0037",
"create_time":"1470016231",
"user_decoration":{
},
"highlight":{
"name":[
[
0,
2
]
]
},
"source_url":"http://www.toutiao.com/c/user/50018626654/",
"id":"50018626654",
"name":"街拍美女"
},
{
"screen_name":"街拍福利",
"media_id":"50020881456",
"description":"把每日的所见所闻，街拍分享给大家。",
"user_id":"50020731766",
"gender":"0",
"follow_count":28449,
"user_verified":"0",
"user_type":"1",
"is_pgc":1,
"verified_content":"",
"avatar_url":"http://p1.pstatp.com/medium/97e001712c15b762881",
"create_time":"1470041090",
"user_decoration":{
},
"highlight":{
"name":[
[
0,
2
]
]
},
"source_url":"http://www.toutiao.com/c/user/50020731766/",
"id":"50020731766",
"name":"街拍福利"
}
],
"cell_type":58,
"id":"75166641448",
"keyword":"街拍"
},
{
"cell_type":26,
"key_info":{
"keyword":"街拍"
},
"keyinfo":{
"keyword":"街拍"
},
"display":{
"hits":[
{
"video_url":"https://aweme.snssdk.com/aweme/v1/play/?video_id=v0200fbd0000bcbi802iv570552f3phg&line=0&app_id=13",
"title":"重庆街拍，最新的小仙女合集来了，双击支持下❤️～",
"detail_schema":"sslocal://awemevideo?group_id=6563754566444125447&group_source=19&item_id=6563754566444125447&request_id=1532526080.86&log_pb=%7B%22impr_id%22%3A%221532526080.86%22%7D&gd_ext_json=%7B%22impr_id%22%3A%221532526080.86%22%7D&category_name=__search__&load_more=7&keyword=%E8%A1%97%E6%8B%8D&size=10&offset=0&from_req_id=20180725214120172016032036766212",
"digg_count":7376,
"large_thumbnail_url":"http://sf6-ttcdn-tos.pstatp.com/img/mosaic-legacy/8abd00042f56be143308~375x375.jpeg",
"play_count":303884,
"thumbnail_url":"http://sf6-ttcdn-tos.pstatp.com/img/mosaic-legacy/8abd00042f56be143308~240x240.jpeg",
"group_id":"6563754566444125447"
},
{
"video_url":"https://aweme.snssdk.com/aweme/v1/play/?video_id=v0200fd30000bbnrni8ckqbvir0trn40&line=0&app_id=13",
"title":"街拍",
"detail_schema":"sslocal://awemevideo?group_id=6552662565971823876&group_source=19&item_id=6552662565971823876&request_id=1532526080.86&log_pb=%7B%22impr_id%22%3A%221532526080.86%22%7D&gd_ext_json=%7B%22impr_id%22%3A%221532526080.86%22%7D&category_name=__search__&load_more=7&keyword=%E8%A1%97%E6%8B%8D&size=10&offset=1&from_req_id=20180725214120172016032036766212",
"digg_count":2660,
"large_thumbnail_url":"http://sf6-ttcdn-tos.pstatp.com/img/mosaic-legacy/815f0004cf3441b51aaa~375x375.jpeg",
"play_count":144648,
"thumbnail_url":"http://sf6-ttcdn-tos.pstatp.com/img/mosaic-legacy/815f0004cf3441b51aaa~240x240.jpeg",
"group_id":"6552662565971823876"
},
{
"video_url":"https://aweme.snssdk.com/aweme/v1/play/?video_id=v0200fbd0000bc3d9q4thbi90j746ctg&line=0&app_id=13",
"title":"重庆街拍，小伙伴们睡前养养眼，明早起来希望看到各位的小心心❤️❤️ 晚安
2.分析Ajax请求并抓取今日头条街拍美图的更多相关文章
- 分析Ajax请求并抓取今日头条街拍美图
项目说明 本项目以今日头条为例,通过分析Ajax请求来抓取网页数据. 有些网页请求得到的HTML代码里面并没有我们在浏览器中看到的内容.这是因为这些信息是通过Ajax加载并且通过JavaScript渲 ...
- 分析 ajax 请求并抓取今日头条街拍美图
首先分析街拍图集的网页请求头部: 在 preview 选项卡我们可以找到 json 文件,分析 data 选项,找到我们要找到的图集地址 article_url: 选中其中一张图片,分析 json 请 ...
- python爬虫知识点总结（十）分析Ajax请求并抓取今日头条街拍美图
一.流程框架
- 15-分析Ajax请求并抓取今日头条街拍美图
流程框架: 抓取索引页内容:利用requests请求目标站点,得到索引网页HTML代码,返回结果. 抓取详情页内容:解析返回结果,得到详情页的链接,并进一步抓取详情页的信息. 下载图片与保存数据库:将 ...
- Python爬虫系列-分析Ajax请求并抓取今日头条街拍图片
1.抓取索引页内容 利用requests请求目标站点,得到索引网页HTML代码,返回结果. 2.抓取详情页内容 解析返回结果,得到详情页的链接,并进一步抓取详情页的信息. 3.下载图片与保存数据库 将 ...
- 【Python爬虫案例学习】分析Ajax请求并抓取今日头条街拍图片
1.抓取索引页内容 利用requests请求目标站点,得到索引网页HTML代码,返回结果. from urllib.parse import urlencode from requests.excep ...
- python爬虫之分析Ajax请求抓取抓取今日头条街拍美图（七）
python爬虫之分析Ajax请求抓取抓取今日头条街拍美图 一.分析网站 1.进入浏览器,搜索今日头条,在搜索栏搜索街拍,然后选择图集这一栏. 2.按F12打开开发者工具,刷新网页,这时网页回弹到综合 ...
- Python Spider 抓取今日头条街拍美图
""" 抓取今日头条街拍美图 """ import os import time import requests from hashlib ...
- 爬虫（八）：分析Ajax请求抓取今日头条街拍美图
(1):分析网页 分析ajax的请求网址,和需要的参数.通过不断向下拉动滚动条,发现请求的参数中offset一直在变化,所以每次请求通过offset来控制新的ajax请求. (2)上代码 a.通过aj ...
随机推荐
- 使用docker踩过的坑
最近需要使用docker,但是win10电脑的系统不是docker windows适用版本,没法在windows上安装 于是就上centos虚拟机里面装了一个docker docker pull文件的 ...
- BZOJ - 2038 小Z的袜子(普通莫队)
题目链接:小Z的袜子 题意:$n$只袜子,$m$个询问,每次回答有多大概率在$[L,R]$区间内抽到两只颜色相同的袜子 思路:普通莫队,如果两个询问左端点在一个块内,则按询问右端点排序,否则按照所在块 ...
- Spark性能调优-基础篇
前言 在大数据计算领域,Spark已经成为了越来越流行.越来越受欢迎的计算平台之一.Spark的功能涵盖了大数据领域的离线批处理.SQL类处理.流式/实时计算.机器学习.图计算等各种不同类型的计算操作 ...
- day46_Webservice学习笔记_02
一.回顾昨天所学 什么是webservice? 什么是远程调用技术?答:系统和系统之间的调用,从远程系统当中获取业务数据. Webservice是web服务,他是用http传输SOAP协议 ...
- docker部署-windows环境
docker部署-windows环境 1. docker windows 1.1. 安装 win7或者win8需要利用docker toolbox来安装,其是一个docker工具集,w ...
- 洛谷P1144最短路计数
题目描述 给出一个NNN个顶点MMM条边的无向无权图,顶点编号为1−N1-N1−N.问从顶点111开始,到其他每个点的最短路有几条. 输入格式 第一行包含222个正整数N,MN,MN,M,为图的顶点数 ...
- 洛谷P1346 电车（需要稍加思索的最短路）
题目描述 在一个神奇的小镇上有着一个特别的电车网络,它由一些路口和轨道组成,每个路口都连接着若干个轨道,每个轨道都通向一个路口(不排除有的观光轨道转一圈后返回路口的可能).在每个路口,都有一个开关决定 ...
- java里自定义分页查询的尝试
public String list(){ try { LoginUser loginUser = getLoginUser();//获取当前登录用户 if(curpage<=0){ curpa ...
- ES-倒排索引
一.什么事倒排索引 二.倒排索引为什么查询速度快 (1)ES全文检索的底层是使用倒排索引实现的 (2)倒排索引会将文档的内容进行关键词分词,可以使用关键词直接定位到文档的内容
- wc、grep 、 cut、paste 和 tr 命令的用法
1 wc 命令 wc 命令是一个统计的工具,主要用来显示文件所包含的行.字和字节数. wc 命令是 word count 的缩写. (1)命令格式 wc [选项] [文件] (2)常用参数 参数 描述 ...