Python爬虫爬取qq视频等动态网页全代码

环境：py3.4.4 32位

需要插件：selenium BeautifulSoup xlwt

# coding = utf-8

from selenium import webdriver
from bs4 import BeautifulSoup
import time
import re #re模块提供正则表达式支持
import xlwt

url = 'http://v.qq.com/vplus/huilanyujia/videos'
tudou = webdriver.Firefox()
tudou.get(url)

#创建workbook和sheet对象
workbook = xlwt.Workbook() #注意Workbook的开头W要大写
sheet1 = workbook.add_sheet('优酷',cell_overwrite_ok=True)
count1 = count2 = 3
sheet1.write(0,0,'由于bug，暂时！！最后一页数据需要自己手动统计')
sheet1.write(1,0,'如有技术问题，请联系陈鼎，微信chending2012')

#开始写入文件
for num in range(2,22):
pageNum='pager_num_0_'+str(num)
tudou.find_element_by_id(pageNum).click()

i = tudou.page_source#获取读取到的网页资源
soup = BeautifulSoup(i,"html.parser")
i1 = soup.find_all("strong",class_="figure_title figure_title_two_row")
i2 = soup.find_all("span",class_="info_inner")

#以上通过beautifulsoup 做一个初略的筛选

for each in i1:
p =r'(target="_blank">)(.+)(</a>)'
play_name =re.search(p,str(each)).group(2)
sheet1.write(count1,0,play_name)
count1 += 1

for each in i2:
play_num = ''
p = re.compile(r'\d+\.?万?')
play_num0 = p.findall(str(each))
for each1 in play_num0:
play_num +=str(each1)
sheet1.write(count2,1,play_num)
count2 += 1

time.sleep(2)

#最后一页 ，因为最后一页的元素地址有点不一样，所以特地写一篇
pageNum = 'pager_last_0'
tudou.find_element_by_id(pageNum).click()

i = tudou.page_source#获取读取到的网页资源
soup = BeautifulSoup(i,"html.parser")
i1 = soup.find_all("strong",class_="figure_title figure_title_two_row")
i2 = soup.find_all("span",class_="info_inner")

for each in i1:
p =r'(target="_blank">)(.+)(</a>)'#用正则表达式匹配
play_name =re.search(p,str(each)).group(2)
sheet1.write(count1,0,play_name)
count1 += 1

for each in i2:
play_num = ''
p = re.compile(r'\d+\.?万?')
play_num0 = p.findall(str(each))
for each1 in play_num0:
play_num +=str(each1)
sheet1.write(count2,1,play_num)
count2 += 1

#保存该excel文件,有同名文件时直接覆盖
Nowtime = time.strftime('%Y-%m-%d',time.localtime(time.time()))
excel_name = str(Nowtime)+'.xls'
workbook.save(excel_name)

print('done')
tudou.quit()

暂时写这么多，后期会优化代码，编写界面。

这里有一个bug，selenium 翻页以后，获取到的网页内容为前一页的，而非当前页。望大神指点。

-------------------------附录

1. python高手之路python处理excel文件(方法汇总)  http://www.jb51.net/article/77626.htm

2. python模块介绍- xlwt 创建xls文件（excel） http://www.cnblogs.com/snake-hand/p/3153158.html

3. seleniumwebdriver(python)第三版   http://wenku.baidu.com/view/cd580331b6360b4c2e3f5727a5e9856a5612268d

4. Beautiful Soup 中文文档   https://www.crummy.com/software/BeautifulSoup/bs4/doc/index.zh.html