百度贴吧python吧抓取用户名和图片

原理就是将贴吧条数中的用户提取出来并在此爬取用户中的图片

#!/usr/bin/env python
#coding:utf-8

import requests
import urllib2
import re
import socket
import logging
import os
import threading
import urllib
import sys
import time
import redis
reload(sys)
sys.setdefaultencoding('utf8')
# 设置超时时间
socket.setdefaulttimeout(30)

# 设置日志级别、格式和日期时间
logging.basicConfig(level=logging.INFO,
                format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
                datefmt='%a, %d %b %Y %H:%M:%S',
                filename='mz_teacher_spider.log',
                filemode='w')
def details(block_count, block_size, total_size):
    # 总大小(KB)
        total_kb = total_size / 1024
        # 已下载(KB)
        downloaded_kb = (block_count * block_size) / 1024
        par = 100.0 * block_count * block_size/ total_size
        if block_count * block_size <= total_size:
                print (u'\r进度：%.2f%%, 总大小：%dKB, 已下载：%dKB\r' % ( par,total_kb, downloaded_kb))

def get_content(url):
    headers={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36'}
    try:
        request=urllib2.Request(url,headers=headers)
        # print r.content
        response = urllib2.urlopen(request, timeout=3)
        content=response.read()
        return content
    except urllib2.URLError as e:
        # 写异常日志
        logging.info('该地址不能访问('+str(e)+')：'+url)
    except urllib2.HTTPError as e:
        # 写异常日志
        logging.info('该地址访问出错('+str(e)+')：'+url)
    except socket.timeout:
        # 写异常日志
        logging.info('该地址访问超时：'+url)

def parser(content):
    pattern='a data-field=.*? class="frs-author-name j_user_card " href="(.*?)" target="_blank">(.*?)</a>'
    user_dict={}
    users=[]
    regex = re.compile(pattern)
    items=re.findall(regex,content)
    items=set(items)
    for item in items:
        #print item[0]+item[1]
        user=requests.get('http://tieba.baidu.com'+item[0])
        pattern2='<a href="javascript:;" style="" class="userinfo_head"><img src="(.*?)"/></a>'
        regex2 = re.compile(pattern2)
        imgurl=re.findall(regex2,user.content)
        name=item[1]
        #判断用户是否存在
        if imgurl:
            user_dict['name']=name
            user_dict['imgurl']=imgurl[0]
            users.append(user_dict)
            user_dict={}
            #print user_dict
        else:
            print '该用户不存在'

        # #
        #
        #
        #
    #print users
    return users

def urlretrieve(users,folder,rdb):
    if not os.path.isdir(folder):
        os.mkdir(folder)

    for user in users:
        path=unicode(folder+'/'+user['name']+'.jpg', 'utf-8')
        rdb.hset('tieba_user_info',user['name'],user['imgurl'])
        #print path
        #print path
        print u'线程:%s 正在下载图片: %s \r' %(threading.current_thread(),user['imgurl'])
        urllib.urlretrieve(user['imgurl'],path,reporthook=details)
    rdb.save()
     #
    print '下载完成'
def run(users,folder,rdb):
    urlretrieve(users=users,folder=folder,rdb=rdb)

     #def _fname():
        #return f
     #

if __name__ == '__main__':
    rdb = redis.Redis(host='localhost',port=6379,db=0,password='test')
    r=get_content('http://tieba.baidu.com/f?kw=python&fr=ala0&tpl=5')
    users=parser(r)
    run(users=users,folder='img',rdb=rdb)