使用Jsoup爬取网站图片

 package com.test.pic.crawler;
 
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URL;
 import java.net.URLConnection;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import org.apache.commons.lang3.concurrent.BasicThreadFactory;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.*;
 import org.jsoup.select.Elements;
 
 /**
  * @Title: PicCrawler.java
  *
  * @Package com.test.pic.crawler
  *
  * @Description: 爬取指定网站的指定Tag下的图片或者全部Tag图片
  *
  * @author CoderZZ
  *
  * @date 2018年1月12日 下午11:22:41
  *
  * @version V1.0
  *
  */
 public class PicCrawler implements Runnable{
     private static String pathString = "G:/test/pic/";//存储目录
     //存储真正的爬取页面
     static BlockingQueue<String> urlBlockingQueue = new LinkedBlockingDeque<String>(1000);
     static int threadNum = 10;
 //    public PicCrawler(String url){
 //        this.url = url;
 //    }
 
     /**
      * @Title: main
      *
      * @Description: TODO(这里用一句话描述这个方法的作用)
      *
      * @param @param args    设定文件
      *
      * @return void    返回类型
      *
      * @throws
      *
      */
     public static void main(String[] args) {
         String homeurlString = "https://www.xxxx.com";//爬取页面的基本地址
         String tagPageUrl = "https://www.xxxx.com/tag/";//tag分页地址
         //Tag标签的完整路径
         Set<String> tagFullHrefSet = new HashSet<String>(16);
         //想要爬取哪些tag，如果为空，则全部爬取;否则只配置对应的tag
         String[] crawlerTagArray = {"风景"};
         List<String> crawlerTagList = Arrays.asList(crawlerTagArray);
         try {
             //1.获取想要的tag完整的url
             Document tagListDocument = Jsoup.connect(tagPageUrl).get();
             Elements tagsListDivElements = tagListDocument.getElementsByClass("tags_list");
             for(Element element:tagsListDivElements){
                 Elements aElements = element.getElementsByTag("a");
                 for(Element a:aElements){
                     if(crawlerTagList.size() == 0 || crawlerTagList.contains(a.text())){
                         String tagUrlString = homeurlString+a.attr("href");
                         //https://www.xxxx.com/tag/fengjing.html
                         tagUrlString = tagUrlString.substring(0, tagUrlString.lastIndexOf("."))+"/1.html";
                         tagFullHrefSet.add(tagUrlString);
                     }
                 }
             }
             //2.获取图片链接页面地址，分页爬取
             for(String tagUrl:tagFullHrefSet){
                 String tempTagUrlString = tagUrl;
                 int currentPageNum = 1;
                 while(true){
                     try{
                         Document imagePageDocument = Jsoup.connect(tempTagUrlString).get();
                         Elements imageListElements = imagePageDocument.getElementsByClass("Pli-litpic");
                         if(imageListElements.size() == 0){
                             break;
                         }
                         for(Element image:imageListElements){
                             urlBlockingQueue.offer(homeurlString+image.attr("href"));
                         }
                         //https://www.xxxx.com/tag/fengjing/1.html
                         tempTagUrlString = tempTagUrlString.substring(0, tempTagUrlString.lastIndexOf("/")+1)+(++currentPageNum)+".html";
                     }catch(Exception e){
                         break;
                     }
                 }
             }
             ScheduledExecutorService excutor = new ScheduledThreadPoolExecutor(threadNum,new BasicThreadFactory.Builder().namingPattern("my-crawler-thread-%d").daemon(false).build());
             for(int i=0;i<threadNum;i++){
 //                excutor.schedule(new PicCrawler(urlArray[i]), 1, TimeUnit.SECONDS);
 //                excutor.execute(new PicCrawler(urlArray[i]));
                 excutor.submit(new PicCrawler());
             }
         } catch (IOException e) {
             // TODO Auto-generated catch block
             e.printStackTrace();
         }
     }
     @Override
     public void run() {
         while (true) {
             try {
                 long begin = System.currentTimeMillis();
                 String url = urlBlockingQueue.poll();
                 if(null != url){
                     Document doc = Jsoup.connect(url).get();
                     Elements titleElements =doc.select("#photos > h1");
                     if(null != titleElements && null != titleElements.get(0)){
                         Set<String> imgSrcSet = new HashSet<String>(16);
                         Element titleElement = titleElements.get(0);
                         String foldNameString = titleElement.text();
                         String[] nameArray = foldNameString.split("\\(");
                         foldNameString = nameArray[0];
                         nameArray = nameArray[1].split("/");
                         int totalPaggs = Integer.parseInt(nameArray[1].replace(")", ""));
                         for(int i=1;i<=totalPaggs;i++){
                             String urlTemp = url.replace(".html", "_"+i+".html");
                             Document docTemp = Jsoup.connect(urlTemp).get();
                             Element element = docTemp.getElementById("big-pic");
                             Elements imgElements = element.getElementsByTag("img");
                             for(Element imgElement:imgElements){
                                 imgSrcSet.add(imgElement.attr("src"));
                             }
                         }
                         if(imgSrcSet.size()>0){
                             for(String imgSrc:imgSrcSet){
                                 // 构造URL
                                 URL imgurl = new URL(imgSrc);
                                 // 打开连接
                                 URLConnection con = imgurl.openConnection();
                                 //设置请求超时为10s
                                 con.setConnectTimeout(10*1000);
                                 // 输入流
                                 InputStream is = con.getInputStream();
                                 // 500k的数据缓冲
                                 byte[] bs = new byte[1024*500];
                                 // 读取到的数据长度
                                 int len;
                                 // 输出的文件流
                                 File sf=new File(pathString+"\\"+foldNameString);
                                 if(!sf.exists()){
                                     sf.mkdirs();
                                 }
                                 String filename = imgSrc.split("/")[imgSrc.split("/").length-1];
                                 OutputStream os = new FileOutputStream(sf.getPath()+"\\"+filename);
                                 // 开始读取
                                 while ((len = is.read(bs)) != -1) {
                                     os.write(bs, 0, len);
                                 }
                                 // 完毕，关闭所有链接
                                 os.close();
                                 is.close();
                                 System.out.println(imgSrc+"下载完成!!!");
                             }
                         }
                         long end = System.currentTimeMillis();
                         System.out.println("================================================================");
                         System.out.println(Thread.currentThread().getName()+"******************已全部下载完成，用时："+((end-begin)/1000)+"S");
                     }
                 }else{
                     System.out.println("========================BlockingQueue已空，已全部抓取完成!=======================");
                 }
             } catch (Exception e) {
                 System.out.println("========================抓取异常=======================");
             }
         }
     }
 }