python 学习之爬虫练习

通过学习python，写两个简单的爬虫，没用线程，本地抓取速度还不错，有些瑕疵就是抓的图片有些显示不出来，代码做个笔记记录下：

# -*- coding:utf-8 -*-

import re
import urllib.request
import os

url = "http://www.58pic.com/yuanchuang/0/day-"

def getHtml(url):
    page = urllib.request.urlopen(url)
    html = page.read().decode('gbk')
    return html

def getImg(html,num):
    reg = r'src="(.*?)" '
    imgre = re.compile(reg)
    imglist = re.findall(imgre,html)
    x = 0
    os.mkdir(r"G:\collect/%d" % num)
    filePath = r"G:\collect/%d/" % num
    for imgurl in imglist:
        f=open(filePath+str(x)+".jpg",'wb')
        req=urllib.request.urlopen(imgurl)
        buf=req.read()
        f.write(buf)
        x+=1

for i in range(1,10):
    getUrl = url+"%d.html" % i
    print(getUrl)
    html = getHtml(getUrl)
    #print(html)
    print(getImg(html,i))

最终的结果如下图：

根据上面的初步代码，优化后加强版的爬虫代码，对于链接的状态异常的抛出异常后在继续执行程序。代码如下：

# -*- coding:utf-8 -*-

import re
import urllib.request
import os

url = "http://www.58pic.com/psd/"

def getHtml(url):
    page = urllib.request.urlopen(url)
    html = page.read().decode('gbk')
    return html

def getImg(html,num):
    reg = r'src="(.+?\.jpg)" class="show-area-pic" id="show-area-pic" alt="(.*?)"'
    imgre = re.compile(reg)
    imglist = re.findall(imgre,html)
    print(imglist)
    filePath = r"F:\Py/collect/%d/" % num
    isCreate = os.path.exists(filePath)
    if isCreate == False :
        os.mkdir(r"F:\Py/collect/%d" % num)
        for img in imglist:
            title = img[1]
            f=open(filePath+title+".jpg",'wb')
            req=urllib.request.urlopen(img[0])
            buf=req.read()
            f.write(buf)

for i in range(22797263,22797666):
    getUrl = url+"%d.html" % i
    #status = urllib.request.urlopen(getUrl).code
    try:
        html = getHtml(getUrl)
        #print(html)
        getImg(html,i)
    except urllib.request.URLError as e:
        print(e.code)
        print(e.reason)