scrapy模拟浏览器爬取验证码页面

使用selenium模块爬取验证码页面，selenium模块需要另外安装这里不讲环境的配置，我有一篇博客有专门讲ubuntn下安装和配置模拟浏览器的开发

spider的代码

 # -*- coding: utf-8 -*-
 from selenium import webdriver
 import scrapy
 from scrapy.selector import Selector
 from time import sleep

 class MydoubanSpider(scrapy.Spider):
     name = "mydouban_moni"

     def __init__(self, ):
         super(MydoubanSpider, self).__init__()
         self.start_urls = ['https://www.douban.com/']
         self.driver = webdriver.Chrome()
         self.driver.get("https://accounts.douban.com/login")
         sleep(1)

     def parse(self, response):
         yanzhengma = raw_input('请输入验证码：')
         name = self.driver.find_element_by_xpath('//*[@id="email"]')
         name.send_keys('username用户名')
         password = self.driver.find_element_by_xpath('//*[@id="password"]')
         password.send_keys('password密码')
         key = self.driver.find_element_by_xpath('//*[@id="captcha_field"]')
         key.send_keys(yanzhengma)
         summit = self.driver.find_element_by_xpath('//*[@id="lzform"]/div[7]/input')
         summit.click()
         sleep(1)
         sel = Selector(text=self.driver.page_source)
         myname = sel.response.xpath('//*[@id="db-global-nav"]/div/div[1]/ul/li[2]/a/span[1]//text()').extract()
         print ''.join(myname)
         print '====================='
         pass

　　如果出现这个HTTP status code is not handled or not allowed  错误 说明还有spider的头需要配置这个可以用伪装代理解决在setting.py中设置

github完整代码地址：https://github.com/sea1234/myyangzhengma