Python爬虫第一个成功版

爬取http://www.mzitu.com/all里面的图片

 import urllib.request
 import re
 import os

 url = 'http://www.mzitu.com/all/' # 爬虫入口
 req = urllib.request.Request(url)
 req.add_header('Referer','http://www.mzitu.com/all/')
 req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36')

 html = urllib.request.urlopen(req).read().decode('utf-8') # 得到入口页面的HTML

 reg = re.compile(r'<a.+?href="(.+?)"')

 hrlist = re.findall(reg,html) # 得到所有图片的页面链接
 print(hrlist)

 def getImgUrl(url): # 根据图片页面的链接得到图片链接
     html = getHtml(url).decode('utf-8')
     reg = re.compile(r'<img.+?src="(.+?\.jpg)"')
     imgUrl = re.findall(reg,html)
     return imgUrl # 返回得到的图片链接

 def getHtml(url): # 得到HTML页面信息
     req = urllib.request.Request(url)
     req.add_header('Referer','http://www.mzitu.com/all/')
     req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36')

     res = urllib.request.urlopen(req)
     html = res.read() # 得到入口页面的HTML

     return html # 返回得到的字符串形式的HTML页面

 # 根据图片的链接下载图片
 def download(url):
     filename = url.split('/')[-1]
     with open(filename,'wb') as f:
         img = getHtml(url)
         f.write(img)

 # 保存所有图片
 def save_all(folder='mm'):
     os.mkdir(folder)
     os.chdir(folder)
     for each in hrlist:
         imgUrl_list = getImgUrl(each)
         for imgUrl in imgUrl_list:
             download(imgUrl)

 if __name__=='__main__':
     save_all()