学习爬虫的day01

反扒

1.浏览器伪装
加一个协议头（即浏览器的协议头）

火狐的浏览器协议头='User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'

headers={'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'};
url="http://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E5%8C%97%E4%BA%AC&kw=python&sm=0&p=1";

req_timeout=5;
req=Request(url=url,headers=headers)
f=urlopen(req,None,req_timeout)
爬虫中 空格，换行也算

2.三种方式爬取数据

#正则提取

d=re.findall('<a style=\"font-weight: bold\" par=\"(.*)\" href=\"(.*)\" target=\"_blank\">(.*)</a>',str(s));
print(d);
问题：无法处理换行问题，查找后乱码
解决：d.decode('utf-8')

#beautifulsoup提取

# soup=BeautifulSoup(s,'html.parser');
# aList=soup.find_all("tr")
# for items in aList:
#     # print(items);
#     aList1=items.find_all("a")
#     for item1 in aList1:
#         print(item1.get('href'));# a 下的所有href属性
#         print(item1.get_text());# 所有标签包含的内容
#         break;#处理一列的数据‘
　　问题：有未知的JavaScript脚本

#lxml提取

selector=etree.HTML(s);
links=selector.xpath('//tr/td[@class="zwmc"]/div/a/@href|//tr/td[@class="zwmc"]/div/a/text()');
for link in links:
    print(link);
完整代码：

# 1.浏览器伪装
# 加一个协议头（即浏览器的协议头）
#添加模拟浏览器协议头
from urllib.request import Request
from urllib.request import urlopen
from lxml import etree
from bs4 import BeautifulSoup
import re;
headers = {'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}
url = "http://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E5%8C%97%E4%BA%AC&kw=python&sm=0&p=1"
req_timeout=5;
req=Request(url=url,headers=headers);
f=urlopen(req,None,req_timeout);
s=f.read();
#正则提取
# d=re.findall('<a style=\"font-weight: bold\" par=\"(.*)\" href=\"(.*)\" target=\"_blank\">(.*)</a>',str(s));
# print(d);
#beautifulsoup提取
# soup=BeautifulSoup(s,'html.parser');
# aList=soup.find_all("tr")
# for items in aList:
#     # print(items);
#     aList1=items.find_all("a")
#     for item1 in aList1:
#         print(item1.get('href'));# a 下的所有href属性
#         print(item1.get_text());# 所有标签包含的内容
#         break;
#lxml提取
selector=etree.HTML(s);
links=selector.xpath('//tr/td[@class="zwmc"]/div/a/@href|//tr/td[@class="zwmc"]/div/a/text()');
for link in links:
    print(link);