最近学了下网络爬虫，打算从一个网站上提取点东西，自己练练手，刚开始还从这个网站上取了正确的html，后来百般尝试还是不能取正确的html，希望能得到大家的帮助~

我刚开始的代码是：

1     url="http://www.karger.com/Collections/Hospital";

2     data = urllib.request.urlopen(url).read();

3     data=data.decode('gb2312');

4     data=BeautifulSoup(data);

5     print(data);

后来改成下面这样：

 1 url="http://www.karger.com/Collections/Hospital";

 2

 3 headers = [('Host','www.karger.com'),

 4             ('Connection', 'keep-alive'),

 5             ('Cache-Control', 'max-age=0'),

 6              ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'),

 7               ('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0'),

 8               ('Accept-Encoding','gzip, deflate'),

 9                ('Accept-Language', 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3'),

10                ('If-None-Match', '90101f995236651aa74454922de2ad74'),

11                ('Referer','"http://www.karger.com/Collections/Hospital"'),

12                ('If-Modified-Since', 'Thu, 01 Jan 1970 00:00:00 GMT')]

13 opener = urllib.request.build_opener()

14 opener.addheaders = headers

15 data = opener.open(url).read();

16 print(data)

得到的执行结果都是下面这样：

b'<html>\r\n<head>\r\n<META NAME="robots" CONTENT="noindex,nofollow">\r\n<script>\r\n(function() {  function getSessionCookies() {   cookieArray = new Array();   var cName = /^\\s?incap_ses_/;   var c = document.cookie.split(";");   for (var i = 0; i < c.length; i++) {    key = c[i].substr(0, c[i].indexOf("="));    value = c[i].substr(c[i].indexOf("=") + 1, c[i].length);    if (cName.test(key)) {     cookieArray[cookieArray.length] = value    }   }   return cookieArray  }  function setIncapCookie(vArray) {   try {    cookies = getSessionCookies();    digests = new Array(cookies.length);    for (var i = 0; i < cookies.length; i++) {     digests[i] = simpleDigest((vArray) + cookies[i])    }    res = vArray + ",digest=" + (digests.join())   } catch (e) {    res = vArray + ",digest=" + (encodeURIComponent(e.toString()))   }   createCookie("___utmvc", res, 20)  }  function simpleDigest(mystr) {   var res = 0;   for (var i = 0; i < mystr.length; i++) {    res += mystr.charCodeAt(i)   }   return res  }  function createCookie(name, value, seconds) {   if (seconds) {    var date = new Date();    date.setTime(date.getTime() + (seconds * 1000));    var expires = "; expires=" + date.toGMTString()   } else {    var expires = ""   }   document.cookie = name + "=" + value + expires + "; path=/"  }  function test(o) {   var res = "";   var vArray = new Array();   for (var j = 0; j < o.length; j++) {    var test = o[j][0]    switch (o[j][1]) {    case "exists_boolean":     try { \t if(typeof(eval(test)) != "undefined"){ \t\tvArray[vArray.length] = encodeURIComponent(test + "=true") \t } \t else{ \t\tvArray[vArray.length] = encodeURIComponent(test + "=false") \t }     } catch (e) {      vArray[vArray.length] = encodeURIComponent(test + "=false")     }     break;    case "exists":     try {      vArray[vArray.length] = encodeURIComponent(test + "=" + typeof(eval(test)))     } catch (e) {      vArray[vArray.length] = encodeURIComponent(test + "=" + e)     }     break;    case "value":     try {      vArray[vArray.length] = encodeURIComponent(test + "=" + eval(test).toString())     } catch (e) {      vArray[vArray.length] = encodeURIComponent(test + "=" + e)     }     break;     case "plugins":     try{         p=navigator.plugins         pres=""         for (a in p){pres+=(p[a][\'description\']+" ").substring(0,20)}         vArray[vArray.length] = encodeURIComponent("plugins=" + pres);         }     catch(e){         vArray[vArray.length] = encodeURIComponent("plugins=" +e);         } \tbreak;      case "plugin":     try {      a = navigator.plugins;      for (i in a) {       f = a[i]["filename"].split(".");       if (f.length == 2) {        vArray[vArray.length] = encodeURIComponent("plugin=" + f[1]);        break       }      }     } catch (e) {      vArray[vArray.length] = encodeURIComponent("plugin=" + e)     }     break    }   }   vArray = vArray.join();   return vArray  }  var o = [   ["navigator", "exists_boolean"],   ["navigator.vendor", "value"],   ["opera", "exists_boolean"],   ["ActiveXObject", "exists_boolean"],   ["navigator.appName", "value"],   ["platform", "plugin"],   ["webkitURL", "exists_boolean"],   ["navigator.plugins.length==0", "value"],   ["_phantom", "exists_boolean"] ];  try {   setIncapCookie(test(o));   document.createElement("img").src = "/_Incapsula_Resource?SWKMTFSR=1&e=" + Math.random()  } catch (e) {   img = document.createElement("img");   img.src = "/_Incapsula_Resource?SWKMTFSR=1&e=" + e  } })();\r\n</script>\r\n<script>\r\n(function() { \r\nvar z="";var b="7472797B766172207868723B76617220743D6E6577204461746528292E67657454696D6528293B766172207374617475733D227374617274223B7661722074696D696E673D6E65772041727261792833293B77696E646F772E6F6E756E6C6F61643D66756E6374696F6E28297B74696D696E675B325D3D22723A222B286E6577204461746528292E67657454696D6528292D74293B646F63756D656E742E637265617465456C656D656E742822696D6722292E7372633D222F5F496E63617073756C615F5265736F757263653F4553324C555243543D363726743D373826643D222B656E636F6465555249436F6D706F6E656E74287374617475732B222028222B74696D696E672E6A6F696E28292B222922297D3B69662877696E646F772E584D4C4874747052657175657374297B7868723D6E657720584D4C48747470526571756573747D656C73657B7868723D6E657720416374697665584F626A65637428224D6963726F736F66742E584D4C4854545022297D7868722E6F6E726561647973746174656368616E67653D66756E6374696F6E28297B737769746368287868722E72656164795374617465297B6361736520303A7374617475733D6E6577204461746528292E67657454696D6528292D742B223A2072657175657374206E6F7420696E697469616C697A656420223B627265616B3B6361736520313A7374617475733D6E6577204461746528292E67657454696D6528292D742B223A2073657276657220636F6E6E656374696F6E2065737461626C6973686564223B627265616B3B6361736520323A7374617475733D6E6577204461746528292E67657454696D6528292D742B223A2072657175657374207265636569766564223B627265616B3B6361736520333A7374617475733D6E6577204461746528292E67657454696D6528292D742B223A2070726F63657373696E672072657175657374223B627265616B3B6361736520343A7374617475733D22636F6D706C657465223B74696D696E675B315D3D22633A222B286E6577204461746528292E67657454696D6528292D74293B6966287868722E7374617475733D3D323030297B706172656E742E6C6F636174696F6E2E72656C6F616428297D627265616B7D7D3B74696D696E675B305D3D22733A222B286E6577204461746528292E67657454696D6528292D74293B7868722E6F70656E2822474554222C222F5F496E63617073756C615F5265736F757263653F535748414E45444C3D313134343931363832343539363439303734382C31343832303638373230383234383031333036362C31343339363333303438303432363939313530392C3735353431222C66616C7365293B7868722E73656E64286E756C6C297D63617463682863297B7374617475732B3D6E6577204461746528292E67657454696D6528292D742B2220696E6361705F6578633A20222B633B646F63756D656E742E637265617465456C656D656E742822696D6722292E7372633D222F5F496E63617073756C615F5265736F757263653F4553324C555243543D363726743D373826643D222B656E636F6465555249436F6D706F6E656E74287374617475732B222028222B74696D696E672E6A6F696E28292B222922297D3B";for (var i=0;i<b.length;i+=2){z=z+parseInt(b.substring(i, i+2), 16)+",";}z = z.substring(0,z.length-1); eval(eval(\'String.fromCharCode(\'+z+\')\'));})();\r\n</script></head>\r\n<body>\r\n<iframe style="display:none;visibility:hidden;" src="//content.incapsula.com/jsTest.html" id="gaIframe"></iframe>\r\n</body></html>'

希望大家能够帮我找到问题，谢谢大家~

python3.4 伪装成浏览器获取页面信息失败的更多相关文章

  1. 爬取斗图网图片,使用xpath格式来匹配内容,对请求伪装成浏览器, Referer 防跨域请求

    6.21自我总结 一.爬取斗图网 1.摘要 使用xpath匹配规则查找对应信息文件 将请求伪装成浏览器 Referer 防跨域请求 2.爬取代码 #导入模块 import requests #爬取网址 ...

  2. 爬虫实例——爬取煎蛋网OOXX频道（反反爬虫——伪装成浏览器）

    煎蛋网在反爬虫方面做了不少工作,无法通过正常的方式爬取,比如用下面这段代码爬取无法得到我们想要的源代码. import requests url = 'http://jandan.net/ooxx' ...

  3. Python+Selenium自动化-获取页面信息

    Python+Selenium自动化-获取页面信息   1.获取页面title title:获取当前页面的标题显示的字段 from selenium import webdriver import t ...

  4. 8021x 获取IP信息失败，请检查锐捷认证客户端当前配置是否符合所在网络的要求，检查完毕后尝试重新认证

    早上一起床,登陆锐捷客户端上网,谁知道错问题了.不能联网了,锐捷登陆成功,但是一会儿就提示失败,获取IP信息失败了.下面我描述一下问题原因: 锐捷登陆后有认证提示,和往常正常情况一样的,不过有个小感叹 ...

  5. scrapy获取页面信息

    本例子用命令行调试的方式,演示如何获取页面的特定信息: 0) 示例页面 1) 使用scrapy shell获取目标页面: scrapy shell http://bj.lianjia.com/ersh ...

  6. 通过jquery获取页面信息

    获取浏览器显示区域(可视区域)的高度 : $(window).height(); 获取浏览器显示区域(可视区域)的宽度 :$(window).width(); 获取页面的文档高度 $(document ...

  7. Python3类和实例之获取对象信息

    当我们拿到一个对象的引用时,如何知道这个对象是什么类型,有哪些方法呢 使用type() 判断对象类型使用type()函数 基本类型都可以用type()判断 <class 'int'> &g ...

  8. java，利用Selenium调用浏览器，动态模拟浏览器事件，动态获取页面信息

    1.环境搭建 jdk1.6版本:selenium 2.4版本. jdk1.8版本:selenium3.14版本. (1)selenium的jar包下载: 地址:http://selenium-rele ...

  9. 通过js触发launch事件获取页面信息

    注:图片如果损坏,点击文章链接:https://www.toutiao.com/i6814776265602499080/ 承接上一篇文档<页面Cookie的JS文件编写> 思路 继续之前 ...

随机推荐

  1. scala学习笔记-类型参数中协变(+)、逆变（-）、类型上界(<:)和类型下界(>:)的使用

    转载自  fineqtbull   http://fineqtbull.iteye.com/blog/477994 有位je上的同学来短信向我问起了Scala类型参数中协变.逆变.类型上界和类型下界的 ...

  2. 读mongoose api 记录

    mongoose 需要在Schemas基础上进行使用 var mongoose = require('mongoose'); var Schema = mongoose.Schema; var blo ...

  3. 使用片上XRAM需要进行的初始化

    现在,流行的51单片机大多把on-chip expanded RAM(以下简称XRAM)作为基本配置,容量有些差别.厂商在给出芯片特性时,往往把XRAM和标准52芯片的256字节内部RAM加在一起统称 ...

  4. 【转】adb push&pull bug --- Permission denied----不错

    原文网址:http://blog.csdn.net/hengkong_horse/article/details/8708076 问题背景: adb push  E:\\rrr.txt  /data/ ...

  5. SEO的URL如何优化才是最佳

    原文地址:http://www.chinaz.com/web/2007/0413/6841.shtml 很多人都知道URL对SEO的重要之处,但是很多站点却忽略了站点的路径优化.今天本人在这里写几点关 ...

  6. OpenStackCLI调试及术语识记

    1,Project are organizational units in the cloud,and are also known as tenants or accounts.Each user ...

  7. Unity 脚本函数生命周期

    Awake(),一般我们在这里做一些组件的获得,比如使用getcomponent方法. Start(),我们可以在这里给变量赋值. FixUpdate(),固定更新,因为这里得更新速度为固定(可以在T ...

  8. hdu 5245 Joyful(期望的计算，好题)

    Problem Description Sakura has a very magical tool to paint walls. One day, kAc asked Sakura to pain ...

  9. Linux各种包安装及命令

    1.Locate yum -y install mlocate 若出现问题: locate: can not stat () `/var/lib/mlocate/mlocate.db': 没有那个文件 ...

  10. 如何在cmd窗口启动Tomcat

    平时,一般使用tomcat/bin/startup.bat目录在windows环境启动Tomcat,或者使用IDE配置后启动. 下面来简单介绍下如果在cmd窗口直接输入命令启动Tomcat: 1.将t ...