selenium 爬取空间说说

 package cn.hb.util;

 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import org.openqa.selenium.By;
 import org.openqa.selenium.Cookie;
 import org.openqa.selenium.JavascriptExecutor;
 import org.openqa.selenium.Keys;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.WebElement;
 import org.openqa.selenium.firefox.FirefoxDriver;
 import org.openqa.selenium.firefox.FirefoxOptions;
 import org.openqa.selenium.interactions.Actions;

 /**
  * 爬取说说写入到txt中,爬取100条
  *
  * @author tele
  *
  */
 public class QZTwitterCrawler {
     static String url = "https://user.qzone.qq.com/1350560858";
     static int maxSize = ;
     static int pageSize = ;
     static String userName="qq";
     static String pwd = "密码";
     public static void main(String[] args) throws InterruptedException, IOException {
         login();
     }

     /**
      * 登录
      *
      * @throws InterruptedException
      * @throws IOException
      */
     public static void login() throws InterruptedException, IOException {
         System.setProperty("webdriver.gecko.driver", "D:/browserdriver/geckodriver.exe");

         FirefoxOptions options = new FirefoxOptions();
         options.setBinary("F:/ff/firefox.exe");

         WebDriver driver = new FirefoxDriver(options);
         driver.manage().window().maximize();
         // 超时
         try {
             driver.manage().timeouts().pageLoadTimeout(, TimeUnit.SECONDS);
             driver.manage().timeouts().setScriptTimeout(, TimeUnit.SECONDS);
             driver.get(url);
         } catch (Exception e) {
             System.out.println("所需元素已出现,停止加载页面");
         } finally {
             // 切换到登录login
             driver.switchTo().frame("login_frame");

             WebElement switcher_plogin = driver.findElement(By.id("switcher_plogin"));
             System.out.println(switcher_plogin.getText());
             if (switcher_plogin.isDisplayed()) {
                 switcher_plogin.click();
             }
             // 用户名
             driver.findElement(By.id("u")).clear();
             driver.findElement(By.id("u")).sendKeys(userName);

             // 密码
             driver.findElement(By.id("p")).clear();
             driver.findElement(By.id("p")).sendKeys(pwd);

             // 登录
             try {
                 driver.findElement(By.id("login_button")).click();
                 Thread.sleep();
             } catch (Exception e) {
                 e.printStackTrace();
             } finally {
                 if ("https://i.qq.com/".equals(driver.getCurrentUrl())) {
                     System.out.println("登录失败!5秒后再次尝试登录");
                     Thread.sleep();
                     driver.findElement(By.id("login_button")).click();
                 }
             }

             // 退出frame
             driver.switchTo().defaultContent();

             System.out.println(driver.getCurrentUrl());

             JavascriptExecutor jsExecutor = (JavascriptExecutor) driver;
             // 如果有亲密度提示

               try { WebElement fs_guide = driver.findElement(By.xpath(
               "//div[@id='friendship_promote_layer']/table[@class='tbl-fs-guide']//a"
              )); if(fs_guide != null && fs_guide.isDisplayed()) {
               fs_guide.click(); } } catch (Exception e) { e.printStackTrace();
               }finally {

               }

             // 点击说说
             driver.findElement(By.cssSelector("#menuContainer ul.head-nav-menu>li.menu_item_311>a")).click();

             Thread.sleep();

             // 切换到frame
             driver.switchTo().frame(driver.findElement(By.className("app_canvas_frame")));

             Thread.sleep();

             // 拼接cookie
         /*    StringBuilder builder = new StringBuilder();
             Set<Cookie> cookieSet = driver.manage().getCookies();
             cookieSet.forEach(c -> builder.append(c.getName()).append("=").append(c.getValue()).append("; "));
             cookies = builder.toString();*/

             // 保存
             saveTwitter(driver);

             System.out.println("内容提取完毕,退出浏览器");
             driver.quit();

         }
     }

     /**
      * 序列化
      * @param driver
      * @return
      * @throws InterruptedException
      * @throws IOException
      */
     public static void saveTwitter(WebDriver driver) throws InterruptedException, IOException {
         File file = new File("f:/qz/twitter.txt");

         // 文件夹检测
         if (!file.getParentFile().exists()) {
             file.mkdirs();
         } else {
             file.delete();
         }

         FileWriter fileWriter = new FileWriter(file, true);

         String xpath;
         // 模拟按键进行滚动
         Actions actions = new Actions(driver);

         //说说总量
         String totalNumStr = driver.findElement(By.xpath("//div[@class='feed_num']/a")).getText();
         int totalNum = Integer.parseInt(totalNumStr);

         // 计算页数
         int totalPage = (int) Math.ceil((double)Math.min(maxSize, totalNum) / (double) pageSize);

         // 构造xpath
         for (int i = ; i < totalPage; i++) {

             for (int j = ; j < pageSize; j++) {
                 xpath = "//ol[@id='msgList']/li[" + (j + ) + "]/div[3]/div[2]/pre[@class='content']";
                 // 获取说说内容
                 try {
                     WebElement element = driver.findElement(By.xpath(xpath));
                     String text = element.getText();
                     System.out.println("本页第" + (j + ) + "条   :" + text);
                     fileWriter.write(text, , text.length());

                 } catch (Exception e) {
                     e.printStackTrace();
                 } finally {

                 }
                 if (j %  == ) {
                     actions.sendKeys(Keys.ARROW_DOWN).perform();
                 }
             }
             System.out.println("第" + (i + ) + "页说说爬取完毕");
             // 分页
             if ((i + ) <= totalPage) {
                 driver.findElement(By.xpath("//a[@id='pager_num_" + i + "_" + (i + ) + "']")).click();
                 // 等待页面加载
                 Thread.sleep();
             }
         }

         if (fileWriter != null) {
             fileWriter.close();
         }
     }

 }

比爬取相册简单点,唯一有点弯的是页码的构造了,我写的这个只支持获取文字,可以用来生成词云