webcollector + selenium 爬取空间相册图片

 package cn.hb.util;

 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;

 import org.apache.commons.io.IOUtils;
 import org.openqa.selenium.By;
 import org.openqa.selenium.Cookie;
 import org.openqa.selenium.JavascriptExecutor;
 import org.openqa.selenium.Keys;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.WebElement;
 import org.openqa.selenium.firefox.FirefoxDriver;
 import org.openqa.selenium.firefox.FirefoxOptions;
 import org.openqa.selenium.interactions.Actions;
 import cn.edu.hfut.dmic.webcollector.conf.Configuration;
 import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
 import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
 import cn.edu.hfut.dmic.webcollector.model.Page;
 import cn.edu.hfut.dmic.webcollector.net.HttpRequest;
 import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
 import cn.edu.hfut.dmic.webcollector.util.FileUtils;

 /**
  * 爬取空间图片 selenium登录后提取链接给webcollector处理即可
  *
  * @author tele
  *
  */
 public class QZoneCrawler extends BreadthCrawler {
     static String url = "https://user.qzone.qq.com/qq号";
     static String cookies = "";
     static final int pageSize = 98;
     static List<String> crawdataList = new ArrayList<String>();
     static File baseDir = new File("F:/qz/image");

     public QZoneCrawler(String crawlPath, boolean autoParse) {
         super(crawlPath, autoParse);
     }

     @Override
     public void visit(Page page, CrawlDatums next) {
         try {
             Thread.sleep(3000);
         } catch (InterruptedException e) {
             e.printStackTrace();
         }
         String name = UUID.randomUUID().toString() + ".jpg";
         try {
             FileUtils.write(new File(baseDir, name), page.content());
         } catch (FileNotFoundException e) {
             e.printStackTrace();
         } catch (IOException e) {
             e.printStackTrace();
         }
     }

     String userAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0";

     // 设置cookies
     @Override
     public Page getResponse(CrawlDatum crawlDatum) throws Exception {
         HttpRequest request = new HttpRequest(crawlDatum);
         request.setCookie(cookies);
         request.setUserAgent(userAgent);
         return request.responsePage();
     }

     public static void main(String[] args) throws Exception {

         QZoneCrawler qz = new QZoneCrawler("F:/qz/image/webcollector", true);

         Configuration conf = Configuration.copyDefault();
         conf.setAutoDetectImg(true);
         conf.setConnectTimeout(5000);
         conf.setReadTimeout(10000);

         // 线程爬取间隔
         conf.setExecuteInterval(5000);
         qz.setConf(conf);
         qz.setThreads(100);

         login();
         qz.addSeed(crawdataList);
         qz.start(1);

     }

     /**
      * 登录
      *
      * @throws InterruptedException
      * @throws IOException
      */
     public static void login() throws InterruptedException, IOException {
         System.setProperty("webdriver.gecko.driver", "D:/browserdriver/geckodriver.exe");

         FirefoxOptions options = new FirefoxOptions();
         options.setBinary("F:/ff/firefox.exe");

         WebDriver driver = new FirefoxDriver(options);
         driver.manage().window().maximize();
         // 超时
         try {
             driver.manage().timeouts().pageLoadTimeout(3, TimeUnit.SECONDS);
             driver.manage().timeouts().setScriptTimeout(3, TimeUnit.SECONDS);
             driver.get(url);
         } catch (Exception e) {
             System.out.println("所需元素已出现,停止加载页面");
         } finally {
             // 切换到登录login
             driver.switchTo().frame("login_frame");

             WebElement switcher_plogin = driver.findElement(By.id("switcher_plogin"));
             System.out.println(switcher_plogin.getText());
             if (switcher_plogin.isDisplayed()) {
                 switcher_plogin.click();
             }
             // 用户名
             driver.findElement(By.id("u")).clear();
             driver.findElement(By.id("u")).sendKeys("账号");

             // 密码
             driver.findElement(By.id("p")).clear();
             driver.findElement(By.id("p")).sendKeys("密码");

             // 登录
             try {
                 driver.findElement(By.id("login_button")).click();
                 Thread.sleep(3000);
             } catch (Exception e) {
                 e.printStackTrace();
             } finally {
                 if ("https://i.qq.com/".equals(driver.getCurrentUrl())) {
                     System.out.println("登录失败!5秒后再次尝试登录");
                     Thread.sleep(5000);
                     driver.findElement(By.id("login_button")).click();
                 }
             }

             // 退出frame
             driver.switchTo().defaultContent();

             System.out.println(driver.getCurrentUrl());

             JavascriptExecutor jsExecutor = (JavascriptExecutor) driver;

             // 如果有亲密度提示
             /*
              * try { WebElement fs_guide = driver.findElement(By.xpath(
              * "//div[@id='friendship_promote_layer']/table[@class='tbl-fs-guide']//a"
              * )); if(fs_guide != null && fs_guide.isDisplayed()) {
              * fs_guide.click(); } } catch (Exception e) { e.printStackTrace();
              * }finally {
              *
              * }
              */

             // 点击相册
             driver.findElement(By.cssSelector("#menuContainer ul.head-nav-menu>li.menu_item_4>a")).click();

             Thread.sleep(2000);

             // 切换到frame
             driver.switchTo().frame(driver.findElement(By.className("app_canvas_frame")));

             // 进入图片列表(说说相册)
             // driver.findElement(By.xpath("//ul[@class='js-album-list-ul']/li[1]/div[1]/div[1]/a")).click();

             // 拼接cookie
             StringBuilder builder = new StringBuilder();
             Set<Cookie> cookieSet = driver.manage().getCookies();
             cookieSet.forEach(c -> builder.append(c.getName()).append("=").append(c.getValue()).append("; "));
             cookies = builder.toString();

             // 获得相册列表
             List<WebElement> photoList = driver.findElements(By.xpath("//ul[@class='js-album-list-ul']/li"));
             if (photoList == null || photoList.size() == 0) {
                 throw new RuntimeException("定位相册列表元素失败!");
             }

             // 构造不同相册的xpath路径
             List<String> xpathList = new ArrayList<String>();
             for (int i = 0; i < photoList.size(); i++) {
                 xpathList.add("//ul[@class='js-album-list-ul']/li[" + (i + 1) + "]");
             }

             // 窗口句柄
             List<String> allHandles = new ArrayList<String>(driver.getWindowHandles());

             // 遍历xpath
             String newUrl = driver.getCurrentUrl();
             for (int i = 0; i < xpathList.size(); i++) {
                 // 打开新标签页
                 jsExecutor.executeScript("window.open('" + newUrl + "');");
                 allHandles = new ArrayList<String>(driver.getWindowHandles());

                 Thread.sleep(2000);
                 String xpath = xpathList.get(i);

                 // 句柄切换需要时间
                 driver.switchTo().window(allHandles.get(i + 1));
                 Thread.sleep(2000);

                 List<String> urlList = getImageUrl(driver, xpath);
                 if (urlList == null) {
                     break;
                 }
                 crawdataList.addAll(urlList);
             }

             System.out.println("所有相册图片链接提取完毕,退出浏览器");
             driver.quit();

         }
     }

     /**
      * 提取图片url
      *
      * @param driver
      * @param xpath
      * @throws InterruptedException
      * @throws IOException
      */
     public static List<String> getImageUrl(WebDriver driver, String xpath) throws InterruptedException, IOException {
         List<String> urlList = new ArrayList<String>();

         // 点击相册
         driver.findElement(By.cssSelector("#menuContainer ul.head-nav-menu>li.menu_item_4>a")).click();

         // 切换到图片的frame
         driver.switchTo().frame(driver.findElement(By.className("app_canvas_frame")));
         Thread.sleep(1000);

         // 获得相册名称
         String photo_name = driver.findElement(By.xpath(xpath + "//a[@class='c-tx2 js-album-desc-a']")).getText();

         //// 文件夹检测
         File imageUrl = new File("f:/qz/" + photo_name + ".txt");
         if (!imageUrl.getParentFile().exists()) {
             imageUrl.mkdirs();
         } else {
             imageUrl.delete();
         }

         // 获得图片总数,每页最多98张图片
         WebElement span = driver.findElement(By.xpath(xpath + "/div[1]/div[1]/a" + "/span"));
         String text = span.getText();
         int count = Integer.parseInt(text);

         // 进入列表
         driver.findElement(By.xpath(xpath + "/div[1]/div[1]/a")).click();
         Thread.sleep(3000);

         // 计算页数
         int totalPage = (int) Math.ceil((double) count / (double) pageSize);
         System.out.println(photo_name + "图片总数为----" + count + "张,共计---" + totalPage + "页");

         FileWriter fileWriter = new FileWriter(imageUrl, true);
 　　　　　Actions actions = new Actions(driver);
         for (int i = 0; i < totalPage; i++) {

             // 模拟按键加载图片
      //       Actions actions = new Actions(driver);
             for (int j = 0; j < 50; j++) {
                 if (j % 5 == 0) {
                     Thread.sleep(1000);
                 }
                 actions.sendKeys(Keys.ARROW_DOWN).perform();
             }

             // 提取本页的image链接
             List<WebElement> list = driver.findElements(
                     By.xpath("//a[@class='item-cover j-pl-photoitem-imgctn']/img[@class='j-pl-photoitem-img']"));
             if (list == null || list.size() == 0) {
                 // 相册无权限访问或定位失败
                 System.out.println("无法提取图片链接!");
                 return null;
             }
             for (WebElement element : list) {
                 String src = element.getAttribute("src") + "\n";
                 IOUtils.write(src, fileWriter);
                 System.out.println(src);
                 // 添加链接
                 urlList.add(src);
             }
             System.out.println("第" + (i + 1) + "页图片链接提取完毕");
             Thread.sleep(1000);
             // 跳转到下一页
             if ((i + 2) <= totalPage) {
                 driver.findElement(By.xpath("//a[@id='pager_num_1_" + (i + 2) + "']")).click();
                 ;
             }
         }

         fileWriter.close();
         return urlList;
     }

 }

运行环境与上篇博文相同https://www.cnblogs.com/tele-share/p/9595265.html爬取结果