python--爬取豆瓣热门国产电视剧保存为文件
# -*- coding: utf-8 -*-
__author__ = 'Frank Li'
import requests
import json
class HotSpider(object):
def __init__(self):
self.url = "https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_domestic_hot/items?os=android&for_mobile=1&start={}&count=18&loc_id=108288"
self.session = requests.session()
self.headers = {"Referer": "https://m.douban.com/tv/chinese",
"User-Agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Mobile Safari/537.36"}
def parse_2_list_from_str(self,url):
return json.loads(self.session.get(url,headers=self.headers).content.decode())['subject_collection_items']
def save_as_file(self,content_list,file):
with open(file,'a',encoding='utf-8') as f:
for content in content_list:
f.write(json.dumps(content,ensure_ascii=False))
f.write('\n')
def run(self):
url = self.url.format(0)
num = 0
total = 500
while num<total+18:
print(url)
self.save_as_file(self.parse_2_list_from_str(url),'hot.json')
num+=18
url=self.url.format(num)
if __name__ == '__main__':
hot_spider = HotSpider()
hot_spider.run()
使用 xpath 爬取正在热映的 电影保存为 json 文件
# -*- coding: utf-8 -*-
__author__ = 'Frank Li'
import requests
from lxml import etree
import json
url = "https://movie.douban.com/cinema/nowplaying/changsha/"
headers = {"Referer":"https://movie.douban.com/cinema/nowplaying/changsha/",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"}
sess = requests.session()
response = sess.get(url,headers=headers)
html_str = response.content.decode()
element = etree.HTML(html_str)
movie_img_list = element.xpath("//div[@class='mod-bd']/ul[@class='lists']//li[@class='poster']//img/@src")
movie_name_list = element.xpath("//div[@class='mod-bd']/ul[@class='lists']//li[@class='stitle']/a/@title")
movie_addr_list = element.xpath("//div[@class='mod-bd']/ul[@class='lists']//li[@class='stitle']/a/@href")
movie_score_list = element.xpath("//div[@class='mod-bd']/ul[@class='lists']//li[@class='srating']/span[@class='subject-rate']/text()")
for name,img,addr,score in zip(movie_name_list,movie_img_list,movie_addr_list,movie_score_list):
item = {}
item['name'] = name
item['img'] = img
item['addr'] = addr
item['score'] = score
with open('movie.json','a',encoding='utf-8') as f:
item_json = json.dumps(item, ensure_ascii=False, indent=2)
print(item_json)
f.write(item_json)
f.write('\n')
f.flush()
保存下来的 movie.json 文件
{
"name": "碟中谍6:全面瓦解",
"img": "https://img3.doubanio.com/view/photo/s_ratio_poster/public/p2529365085.jpg",
"addr": "https://movie.douban.com/subject/26336252/?from=playing_poster",
"score": "8.3"
}
{
"name": "阿尔法:狼伴归途",
"img": "https://img1.doubanio.com/view/photo/s_ratio_poster/public/p2530871439.jpg",
"addr": "https://movie.douban.com/subject/26810318/?from=playing_poster",
"score": "6.5"
}
{
"name": "蚁人2:黄蜂女现身",
"img": "https://img1.doubanio.com/view/photo/s_ratio_poster/public/p2529389608.jpg",
"addr": "https://movie.douban.com/subject/26636712/?from=playing_poster",
"score": "7.5"
}
{
"name": "传奇的诞生",
"img": "https://img1.doubanio.com/view/photo/s_ratio_poster/public/p2531286907.jpg",
"addr": "https://movie.douban.com/subject/3073268/?from=playing_poster",
"score": "7.6"
}
{
"name": "快把我哥带走",
"img": "https://img3.doubanio.com/view/photo/s_ratio_poster/public/p2531080870.jpg",
"addr": "https://movie.douban.com/subject/30122633/?from=playing_poster",
"score": "7.0"
}
{
"name": "道高一丈",
"img": "https://img1.doubanio.com/view/photo/s_ratio_poster/public/p2530863118.jpg",
"addr": "https://movie.douban.com/subject/26954268/?from=playing_poster",
"score": "5.7"
}
{
"name": "李宗伟:败者为王",
"img": "https://img3.doubanio.com/view/photo/s_ratio_poster/public/p2530870325.jpg",
"addr": "https://movie.douban.com/subject/27195119/?from=playing_poster",
"score": "7.1"
}
{
"name": "西虹市首富",
"img": "https://img1.doubanio.com/view/photo/s_ratio_poster/public/p2529206747.jpg",
"addr": "https://movie.douban.com/subject/27605698/?from=playing_poster",
"score": "6.7"
}
{
"name": "一出好戏",
"img": "https://img3.doubanio.com/view/photo/s_ratio_poster/public/p2529571873.jpg",
"addr": "https://movie.douban.com/subject/26985127/?from=playing_poster",
"score": "7.3"
}
{
"name": "精灵旅社3:疯狂假期",
"img": "https://img3.doubanio.com/view/photo/s_ratio_poster/public/p2530591543.jpg",
"addr": "https://movie.douban.com/subject/26630714/?from=playing_poster",
"score": "6.9"
}
{
"name": "苏丹",
"img": "https://img3.doubanio.com/view/photo/s_ratio_poster/public/p2529570494.jpg",
"addr": "https://movie.douban.com/subject/26728641/?from=playing_poster",
"score": "7.0"
}
{
"name": "巨齿鲨",
"img": "https://img3.doubanio.com/view/photo/s_ratio_poster/public/p2530572643.jpg",
"addr": "https://movie.douban.com/subject/26426194/?from=playing_poster",
"score": "6.0"
}
{
"name": "藏北秘岭-重返无人区",
"img": "https://img3.doubanio.com/view/photo/s_ratio_poster/public/p2532522676.jpg",
"addr": "https://movie.douban.com/subject/30208007/?from=playing_poster",
"score": "6.2"
}
{
"name": "那些女人",
"img": "https://img3.doubanio.com/view/photo/s_ratio_poster/public/p2530146643.jpg",
"addr": "https://movie.douban.com/subject/26574965/?from=playing_poster",
"score": "5.3"
}
{
"name": "草戒指",
"img": "https://img1.doubanio.com/view/photo/s_ratio_poster/public/p2531782507.jpg",
"addr": "https://movie.douban.com/subject/27204180/?from=playing_poster",
"score": "5.6"
}
{
"name": "吻隐者",
"img": "https://img3.doubanio.com/view/photo/s_ratio_poster/public/p2531980221.jpg",
"addr": "https://movie.douban.com/subject/26928809/?from=playing_poster",
"score": "7.6"
}
{
"name": "禹神传之寻找神力",
"img": "https://img3.doubanio.com/view/photo/s_ratio_poster/public/p2532781444.jpg",
"addr": "https://movie.douban.com/subject/30227727/?from=playing_poster",
"score": "6.6"
}
{
"name": "大师兄",
"img": "https://img1.doubanio.com/view/photo/s_ratio_poster/public/p2528842218.jpg",
"addr": "https://movie.douban.com/subject/27201353/?from=playing_poster",
"score": "6.2"
}
简单多线程 图片下载
import requests
from bs4 import BeautifulSoup
import os
import threading
def download_img(src,target=None):
parent_dir = './img'
os.makedirs(parent_dir,exist_ok=True)
r = requests.get(src,stream=True)
target = src.split('/')[-1]
target = os.path.join(parent_dir,target)
print(threading.current_thread(),' start to download img: ',target)
with open(target,'wb') as tar_file:
for chunk in r.iter_content(chunk_size=128):
tar_file.write(chunk)
print('saved {}'.format(target))
if __name__ == '__main__':
URL = 'https://tieba.baidu.com/p/6034793219'
html = requests.get(URL).text
soup = BeautifulSoup(html,'lxml')
# print(html)
imgs = []
srcs = soup.find_all('img',{'class':'BDE_Image'})
for src in srcs:
imgs.append(src['src'])
threads = []
for i,img in enumerate(imgs):
t = threading.Thread(target=download_img,args=(img,),name='Thread-{}'.format(i))
t.start()
threads.append(t)
for t in threads:
t.join()
python--爬取豆瓣热门国产电视剧保存为文件的更多相关文章
- requests库爬取豆瓣热门国产电视剧数据并保存到本地
首先要做的就是去豆瓣网找对应的接口,这里就不赘述了,谷歌浏览器抓包即可,然后要做的就是分析返回的json数据的结构: https://movie.douban.com/j/search_subject ...
- 利用Python爬取豆瓣电影
目标:使用Python爬取豆瓣电影并保存MongoDB数据库中 我们先来看一下通过浏览器的方式来筛选某些特定的电影: 我们把URL来复制出来分析分析: https://movie.douban.com ...
- Python爬取豆瓣指定书籍的短评
Python爬取豆瓣指定书籍的短评 #!/usr/bin/python # coding=utf-8 import re import sys import time import random im ...
- Python爬取豆瓣《复仇者联盟3》评论并生成乖萌的格鲁特
代码地址如下:http://www.demodashi.com/demo/13257.html 1. 需求说明 本项目基于Python爬虫,爬取豆瓣电影上关于复仇者联盟3的所有影评,并保存至本地文件. ...
- Python爬取豆瓣电影top
Python爬取豆瓣电影top250 下面以四种方法去解析数据,前面三种以插件库来解析,第四种以正则表达式去解析. xpath pyquery beaufifulsoup re 爬取信息:名称 评分 ...
- python爬取豆瓣首页热门栏目详细流程
记录一下爬取豆瓣热门专栏的经过,通过这篇文章,你能学会requests,HTMLParser,json的基本使用,以及爬取网页内容的基本思路. 使用模块 1,获取豆瓣首页代码:首先我们需要访问豆瓣页面 ...
- python 爬取豆瓣的美剧
pc版大概有500条记录,mobile大概是50部,只有热门的,所以少一点 url构造很简单,主要参数就是page_limit与page_start,每翻一页,start+=20即可,tag是&quo ...
- python爬取豆瓣电影信息数据
题外话+ 大家好啊,最近自己在做一个属于自己的博客网站(准备辞职回家养老了,明年再战)在家里 琐事也很多, 加上自己 一回到家就懒了(主要是家里冷啊! 广东十几度,老家几度,躲在被窝瑟瑟发抖,) 由于 ...
- python 爬取豆瓣电影短评并wordcloud生成词云图
最近学到数据可视化到了词云图,正好学到爬虫,各种爬网站 [实验名称] 爬取豆瓣电影<千与千寻>的评论并生成词云 1. 利用爬虫获得电影评论的文本数据 2. 处理文本数据生成词云图 第一步, ...
随机推荐
- MySQL 报错 1055
具体报错 of SELECT list is not in GROUP BY clause and contains nonaggregated column 'exer.student.sid' w ...
- synchronized 关键字解析
synchronized 关键字解析 同步锁依赖于对象,每个对象都有一个同步锁. 现有一成员变量 Test,当线程 A 调用 Test 的 synchronized 方法,线程 A 获得 Test 的 ...
- css标准文档流
css标准文档流 所谓的标准文档流指的是网页当中的一个渲染顺序,就如同人类读书一样,从上向下,从左向右.网页的渲染顺序也是如此.而我们使用的标签默认都是存在于标准文档流当中. 标准文档流当中的特性 空 ...
- 动态逆序对[CDQ分治]
题面 luogu cdq分治入门 注意删的是值不是位置! #include <cstdio> #include <algorithm> #include <cmath&g ...
- Windows server 安装 OpenSSH
文件自己网上下载或百度云盘提取即可 执行setupssh.exe,一直 next 下去 把安装目录下的 sshd_config 文件 copy 到 C:\Program Files (x86)\Op ...
- 【BZOJ1831】[AHOI2008]逆序对(动态规划)
[BZOJ1831][AHOI2008]逆序对(动态规划) 题面 BZOJ 洛谷 题解 显然填入的数拎出来是不降的. 那么就可以直接大力\(dp\). 设\(f[i][j]\)表示当前填到了\(i\) ...
- suoi62 网友跳 (暴搜+dp)
传送门 sbw太神啦orz 首先N<=20可以直接暴搜 然后玄学剪枝可以过18个点 那么N<=40的时候,就把它拆成两半分别暴搜,再用dp拼起来 对于前半段,设f[i][j]是开始高度为i ...
- Android 架构 -- Room
gradle依赖: // add for room implementation "android.arch.persistence.room:runtime:1.1.1" // ...
- Vim在图形环境下全屏产生黑边
在终端中运行Vim或运行GVim都会遇到这个问题,当窗口全屏时,左右和底部可能会出现边框,这个边框在终端中的Vim表现为Terminal的背景颜色.下图为SpaceVim+Neovim+Termina ...
- A1009. Product of Polynomials
This time, you are supposed to find A*B where A and B are two polynomials. Input Specification: Each ...