day1之校花网小试牛刀

一 利用生成器来完成爬去校花网视频

　　

import requests
import re
import os
import hashlib
import time

DOWLOAD_PATH=r'D:\DOWNLOAD'

def get_page(url):
    try:
        response=requests.get(url,)
        if response.status_code == 200:
            return response.text
    except Exception:
        pass

def parse_index(index_contents):
    # print(type(index_contents))
    detail_urls=re.findall('class="items".*?href="(.*?)"',index_contents,re.S)
    for detail_url in detail_urls:
        if not detail_url.startswith('http'):
            detail_url='http://www.xiaohuar.com'+detail_url
        yield detail_url

def parse_detail(detail_contents):
    movie_urls=re.findall('id="media".*?src="(.*?)"',detail_contents,re.S)
    if movie_urls:
        movie_url=movie_urls[0]
        if movie_url.endswith('mp4'):
           yield movie_url

def download(movie_url):
    print(movie_url)
    try:
        response=requests.get(movie_url,
                              )
        if response.status_code == 200:
            data=response.content
            m=hashlib.md5()
            m.update(str(time.time()).encode('utf-8'))
            m.update(movie_url.encode('utf-8'))
            filepath=os.path.join(DOWLOAD_PATH,'%s.mp4' %m.hexdigest())
            with open(filepath,'wb') as f:
                f.write(data)
                f.flush()
                print('下载成功',movie_url)
    except Exception:
        pass

def main():
    raw_url='http://www.xiaohuar.com/list-3-{page_num}.html'
    for i in range(5):
        #请求索引页,解析拿到详情页链接
        index_url=raw_url.format(page_num=i)
        index_contents=get_page(index_url)
        detail_urls=parse_index(index_contents)

        #请求详情页，解析拿到视频的链接地址
        for detail_url in detail_urls:
            detail_contents=get_page(detail_url)
            movie_urls=parse_detail(detail_contents)

            #下载视频
            for movie_url in movie_urls:
                download(movie_url)

if __name__ == '__main__':
    t1=time.time()
    main()
    print(time.time()-t1)

二 利用对线程优化上述代码

　　

import requests #pip install requests
import re
import os
import hashlib
import time
from concurrent.futures import ThreadPoolExecutor

pool=ThreadPoolExecutor(50)
DOWLOAD_PATH=r'D:\DOWNLOAD'

def get_page(url):
    try:
        response=requests.get(url,)
        if response.status_code == 200:
            return response.text
    except Exception:
        pass

def parse_index(index_contents):
    index_contents=index_contents.result()
    detail_urls=re.findall('class="items".*?href="(.*?)"',index_contents,re.S)
    for detail_url in detail_urls:
        if not detail_url.startswith('http'):
            detail_url='http://www.xiaohuar.com'+detail_url
        pool.submit(get_page,detail_url).add_done_callback(parse_detail)

def parse_detail(detail_contents):
    detail_contents=detail_contents.result()
    movie_urls=re.findall('id="media".*?src="(.*?)"',detail_contents,re.S)
    if movie_urls:
        movie_url=movie_urls[0]
        if movie_url.endswith('mp4'):
           pool.submit(download,movie_url)

def download(movie_url):
    # print(movie_url)
    try:
        response=requests.get(movie_url,
                              )
        if response.status_code == 200:
            data=response.content
            m=hashlib.md5()
            m.update(str(time.time()).encode('utf-8'))
            m.update(movie_url.encode('utf-8'))
            filepath=os.path.join(DOWLOAD_PATH,'%s.mp4' %m.hexdigest())
            with open(filepath,'wb') as f:
                f.write(data)
                f.flush()
                print('下载成功',movie_url)
    except Exception:
        pass

def main():
    raw_url='http://www.xiaohuar.com/list-3-{page_num}.html'
    for i in range(5):
        #请求索引页,解析拿到详情页链接
        index_url=raw_url.format(page_num=i)
        pool.submit(get_page,index_url).add_done_callback(parse_index)

if __name__ == '__main__':
    t1=time.time()
    main()
    print(time.time()-t1)

牛逼的代码

三 自己根据egon讲的grep命令，类似的道理，写的爬去校花网图片的代码

　　

import requests,re,os
def init(f):
    def inner(*args,**kwargs):
        g=f(*args,**kwargs)
        next(g)
        return g
    return inner
def get(url):
    r=requests.get(url)
    def inner():
        r.encoding='gbk'
        return r.text
    return inner
xiaohua=get('http://www.xiaohuar.com/2014.html')
xiaohua_contend=xiaohua()
def search(target):
    g=re.finditer('<a href=.*? target=.*?><img width=.*?  alt="(?P<name>.*?)" src="(?P<src>.*?)" /></a>',xiaohua_contend,re.S)
    for i in g:
        target.send((i.group('name'),i.group('src')))
@init
def handle(target):
    while True:
        name,src=yield
        if src.startswith('http'):
            pass
        else:
            src='http://www.xiaohuar.com'+src
        target.send((name,src))
@init
def download():
    while True:
        name,src=yield
        r=requests.get(src)
        with open(r'D:\校花网'+'\\'+name+'.jpg','wb')as f:
            f.write(r.content)
search(handle((download())))

总结：

egon授课。

生成器与协程有紧密的联系。

生成器可以通过yield接收参数，通过send传值。

生成器与多线程也有关系吗？没有吧。

普通函数爬取视频也是可以用到多线程的。

优化的余地：可以加上进度条，利用类实现。大概就是这个样式，copy的。

def download_file(url, path):
    with closing(requests.get(url, stream=True)) as r:
        chunk_size = *
        content_size = int(r.headers['content-length'])
        print '下载开始'
        with open(path, "wb") as f:
            p = ProgressData(size = content_size, unit='Kb', block=chunk_size)
            for chunk in r.iter_content(chunk_size=chunk_size):
                f.write(chunk)
                p.output()

class ProgressData(object):

    def __init__(self, block,size, unit, file_name='', ):
        self.file_name = file_name
        self.block = block/1000.0
        self.size = size/1000.0
        self.unit = unit
        self.count =
        self.start = time.time()
    def output(self):
        self.end = time.time()
        self.count +=
        speed = self.block/(self.end-self.start) if (self.end-self.start)> else
        self.start = time.time()
        loaded = self.count*self.block
        progress = round(loaded/self.size, )
        if loaded >= self.size:
            print u'%s下载完成\r\n'%self.file_name
        else:
            print u'{0}下载进度{1:.2f}{2}/{3:.2f}{4} 下载速度{5:.2%} {6:.2f}{7}/s'.\
                  format(self.file_name, loaded, self.unit,\
                  self.size, self.unit, progress, speed, self.unit)
            print '%50s'%('/'*int((-progress)*))