python爬虫-爬取你想要的小姐姐

一、准备

1. 原地址

2. 检查html发现，网页是有规则的分页, 最大图片的class为pic-large

二、代码

 import requests
 import os
 from bs4 import BeautifulSoup

 url = 'http://www.win4000.com/wallpaper_detail_157712.html'
 imgmkdir = 'D://Download//ghost_1//'

 # 获取网页url
 def getUrlList():
     imgUrlList = []
     for i in range(0, 10):
         imgUrl = ''
         url_split = url.split('.html')
         if not i == 0:
             imgUrl += url_split[0] + '_' + str(i) + '.html'
             # print(imgUrl)
             imgUrlList.append(imgUrl)

     return imgUrlList

 # 下载图片
 def downImg(imgUrl):
     try:
         if not os.path.exists(imgmkdir):
             os.mkdir(imgmkdir)
         if not os.path.exists(imgUrl):
             r = requests.get(imgUrl)
             r.raise_for_status()
             # 使用with语句可以不用自己手动关闭已经打开的文件流
             imgpath = imgmkdir + imgUrl.split('/')[-1]
             # 开始写文件, wb表示写二进制文件
             with open(imgpath, 'wb') as f:
                 f.write(r.content)
             print(imgUrl + '【爬取完成】')
         else:
             print(imgUrl.split('/')[-1] + '【文件已存在】')
     except Exception as e:
         print("爬取失败" + str(e))

 # 获取imgHtml标签
 def getcontent(soup):
     for i in soup.find_all('img', class_='pic-large'):
         imgsrc = i['src']
         if imgsrc.find('http') >= 0 or imgsrc.find('https') >= 0:
             # 下载图片
             downImg(imgsrc)

 # 根据url获取html源码
 def getHtmlByUrl(htmlUrl):
     htmlText = requests.get(htmlUrl).content
     # 使用beautifulSoup解析html
     soup = BeautifulSoup(htmlText, 'lxml')

     return soup

 def main():
     htmlUrlList = getUrlList()
     for url in htmlUrlList:
         htmltext = getHtmlByUrl(url)
         getcontent(htmltext)

 if __name__ == '__main__':
     main()

三、结果

四、总结

　　代码用比较笨的方法来获取，先试水