python爬虫之趟雷

python爬虫之趟雷整理

雷一：URLError

　　问题具体描述：urllib.error.URLError: <urlopen error [Errno 11004] getaddrinfo failed

 import urllib.request

 def load_message():
     url = 'http://www.baidu.com'

     request = urllib.request.Request(url)
     response = urllib.request.urlopen(request)
     response_str = response.read().decode('utf-8')

     return response.headers, request.headers, response_str

 response_header, request_header, response_data = load_message()
 print(request_header)
 print('----------------------------------------')
 print(response_header)
 print('----------------------------------------')
 print(response_data)

　　分析：报错原因为URLError，产生原因为URL，简单来说，就是URL资源无法访问或者访问不了。具体问题出在三个方向，URL本身，客户端，服务器。

　　解决办法：第一点，检查URL书写是否正确；第二点，检查客户端网络连接状态；第三点，使用URL在浏览器地址栏访问验证服务器是否存在。

　　问题具体描述：urllib.error.URLError: <urlopen error [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1045)>

 #!/usr/bin/env python
 # -*- coding=utf-8 -*-
 # Author: Snow

 import urllib.request

 def create_cookie():
     url = 'https://www.yaozh.com/member/'
     headers = {
         'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko\
 Chrome/69.0.3497.92 Safari/537.36',
         'Cookie': 'think_language=zh-CN; _ga=GA1.2.179792116.1550119571; _gat=1; acw_tc=2f624a2115501195808648935e4f2de7e89205315a7c9e8934c938389d8999; _gid=GA1.2.111857803.1550119581; yaozh_logintime=1550119751; yaozh_user=692948%09snown_1; yaozh_userId=692948; yaozh_uidhas=1; acw_tc=2f624a2115501195808648935e4f2de7e89205315a7c9e8934c938389d8999; MEIQIA_VISIT_ID=1H9g97Ef1WpjYsWf4b7UlGe3wel; PHPSESSID=5itl5rejqnekb07bfrtmuvr3l6; yaozh_mylogin=1550196658; MEIQIA_VISIT_ID=1HCCOYdyjR0FalzMfFm4vYsqevT; Hm_lvt_65968db3ac154c3089d7f9a4cbb98c94=1550119570%2C1550119584%2C1550119751%2C1550196659; Hm_lpvt_65968db3ac154c3089d7f9a4cbb98c94=1550196663'
     }

     request = urllib.request.Request(url, headers=headers)
     response = urllib.request.urlopen(request)
     response_data = response.read().decode('utf-8')

     return response_data

 result = create_cookie()
 with open('cookies.html', 'w', encoding='utf-8') as f:
     f.write(result)

　　分析：问题产生原因python使用urllib.request,urlopen()打开https链接时，需要验证SSL证书，如果网站使用自签名的证书会抛出异常。

　　解决办法：第一点，使用SSL创建context验证上下文，传入urlopen()中context上下文参数；第二点，取消证书验证。

 #!/usr/bin/env python
 # -*- coding=utf-8 -*-
 # Author: Snow

 import urllib.request
 import ssl    #导入ssl模块

 def create_cookie():
     url = 'https://www.yaozh.com/member/'
     headers = {
         'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko\
 Chrome/69.0.3497.92 Safari/537.36',
         'Cookie': 'think_language=zh-CN; _ga=GA1.2.179792116.1550119571; _gat=1; acw_tc=2f624a2115501195808648935e4f2de7e89205315a7c9e8934c938389d8999; _gid=GA1.2.111857803.1550119581; yaozh_logintime=1550119751; yaozh_user=692948%09snown_1; yaozh_userId=692948; yaozh_uidhas=1; acw_tc=2f624a2115501195808648935e4f2de7e89205315a7c9e8934c938389d8999; MEIQIA_VISIT_ID=1H9g97Ef1WpjYsWf4b7UlGe3wel; PHPSESSID=5itl5rejqnekb07bfrtmuvr3l6; yaozh_mylogin=1550196658; MEIQIA_VISIT_ID=1HCCOYdyjR0FalzMfFm4vYsqevT; Hm_lvt_65968db3ac154c3089d7f9a4cbb98c94=1550119570%2C1550119584%2C1550119751%2C1550196659; Hm_lpvt_65968db3ac154c3089d7f9a4cbb98c94=1550196663'
     }

     context = ssl._create_unverified_context()  # 创建验证SSL上下文

     request = urllib.request.Request(url, headers=headers)
     response = urllib.request.urlopen(request， context=context)  # 传入context参数
     response_data = response.read().decode('utf-8')

     return response_data

 result = create_cookie()
 with open('cookies.html', 'w', encoding='utf-8') as f:
     f.write(result)

 #!/usr/bin/env python
 # -*- coding=utf-8 -*-
 # Author: Snow

 import urllib.request
 import ssl

 def create_cookie():
     url = 'https://www.yaozh.com/member/'
     headers = {
         'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko\
 Chrome/69.0.3497.92 Safari/537.36',
         'Cookie': 'think_language=zh-CN; _ga=GA1.2.179792116.1550119571; _gat=1; acw_tc=2f624a2115501195808648935e4f2de7e89205315a7c9e8934c938389d8999; _gid=GA1.2.111857803.1550119581; yaozh_logintime=1550119751; yaozh_user=692948%09snown_1; yaozh_userId=692948; yaozh_uidhas=1; acw_tc=2f624a2115501195808648935e4f2de7e89205315a7c9e8934c938389d8999; MEIQIA_VISIT_ID=1H9g97Ef1WpjYsWf4b7UlGe3wel; PHPSESSID=5itl5rejqnekb07bfrtmuvr3l6; yaozh_mylogin=1550196658; MEIQIA_VISIT_ID=1HCCOYdyjR0FalzMfFm4vYsqevT; Hm_lvt_65968db3ac154c3089d7f9a4cbb98c94=1550119570%2C1550119584%2C1550119751%2C1550196659; Hm_lpvt_65968db3ac154c3089d7f9a4cbb98c94=1550196663'
     }

     ssl._create_default_https_context = ssl._create_unverified_context  # 缺省context参数不做验证，取消验证ssl证书

     request = urllib.request.Request(url, headers=headers)
     response = urllib.request.urlopen(request)
     response_data = response.read().decode('utf-8')

     return response_data

 result = create_cookie()
 with open('cookies.html', 'w', encoding='utf-8') as f:
     f.write(result)

雷二：HTTPError

　　问题具体描述：urllib.error.HTTPError: HTTP Error 503: Service Temporarily Unavailable

 #!/usr/bin/env python
 # -*- coding=utf-8 -*-
 # Author: Snow

 import urllib.request

 def fee_proxy():
     url = 'https://www.xicidaili.com/nn/'

     # 付费代理IP第一种方式
     # proxy_1 = {
     #     'http': 'user_name:passswor@121.61.1.222:9999'
     # }

     # 付费代理IP第二种方式
     user_name = 'admin'
     password = ''
     proxy_ip = '121.61.1.222:9999'
     proxy_manage = urllib.request.HTTPPasswordMgrWithDefaultRealm()  # 密码管理器
     proxy_manage.add_password(None, proxy_ip, user_name, password)

     # proxy_handler = urllib.request.ProxyHandler(proxy_1)
     proxy_handler = urllib.request.ProxyBasicAuthHandler(proxy_manage)  # 代理IP验证处理器
     proxy_openner = urllib.request.build_opener(proxy_handler)

     response = proxy_openner.open(url)
     response_str = response.read().decode('utf-8')

     return response_str

 data = fee_proxy()
 print(data)

　　分析：

　　解决办法：