Java爬虫学习（3）之用对象保存新浪微博博文

 package com.mieba;

 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.Site;
 import us.codecraft.webmagic.processor.PageProcessor;

 public class SinaPageProcessor implements PageProcessor
 {
     public static final String URL_LIST = "http://blog\\.sina\\.com\\.cn/s/articlelist_1487828712_0_\\d+\\.html";

     public static final String URL_POST = "http://blog\\.sina\\.com\\.cn/s/blog_\\w+\\.html";

     private Site site = Site.me().setTimeOut(10000).setRetryTimes(3).setSleepTime(1000).setCharset("UTF-8").setUserAgent(

             "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");;

     @Override
     public Site getSite()
     {
         // TODO Auto-generated method stub
         return site;
     }

     @Override
     public void process(Page page)
     {
         // TODO Auto-generated method stub
         // 列表页

         if (page.getUrl().regex(URL_LIST).match())
         {
             // 从页面发现后续的url地址来抓取
             page.addTargetRequests(page.getHtml().xpath("//div[@class=\"articleList\"]").links().regex(URL_POST).all());

             page.addTargetRequests(page.getHtml().links().regex(URL_LIST).all());

             // 文章页

         } else
         {
             String title = new String();
             String content = new String();
             Article ar = new Article(title, content);
             // 定义如何抽取页面信息，并保存下来
             ar.setTitle(page.getHtml().xpath("//div[@class='articalTitle']/h2/text()").toString());

             ar.setContent(
                     page.getHtml().xpath("//div[@id='articlebody']//div[@class='articalContent']/text()").toString());
             System.out.println("title:"+ar.getTitle());
             System.out.println(ar.getContent());
             page.putField("repo", ar);
 //                    page.putField("date", page.getHtml().xpath("//div[@id='articlebody']//span[@class='time SG_txtc']/text()").regex("\\((.*)\\)"));

         }
     }

 }

 package com.mieba;

 import java.io.FileNotFoundException;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.util.Vector;

 import us.codecraft.webmagic.ResultItems;
 import us.codecraft.webmagic.Task;
 import us.codecraft.webmagic.pipeline.Pipeline;

 public class SinaPipeline implements Pipeline
 {

     @Override
     public void process(ResultItems resultItems, Task arg1)
     {
         // TODO Auto-generated method stub
         Article vo = resultItems.get("repo");
         PrintWriter pw = null;
         try
         {
             pw = new PrintWriter(new FileWriter("sina.txt", true));

                 pw.println(vo);
                 pw.flush();

         }catch(FileNotFoundException e) {
             e.printStackTrace();
         }catch (IOException e)
         {
             e.printStackTrace();
         } finally
         {
             pw.close();
         }
     }

 }

 package com.mieba;

 public class Article
 {
 private String title;
 private String content;
 public String getTitle()
 {
     return title;
 }
 public void setTitle(String title)
 {
     this.title = title;
 }
 public String getContent()
 {
     return content;
 }
 public void setContent(String content)
 {
     this.content = content;
 }
 public Article(String title, String content)
 {
     super();
     this.title = title;
     this.content = content;
 }
 @Override
 public String toString()
 {
     return "Article [title=" + title + ", content=" + content + "]";
 }

 }

 package com.mieba;

 import us.codecraft.webmagic.Spider;

 public class Demo
 {

     public static void main(String[] args)
     { // 爬取开始
         Spider
         // 爬取过程
         .create(new SinaPageProcessor())
         // 爬取结果保存
         .addPipeline(new SinaPipeline())
         // 爬取的第一个页面
         .addUrl("http://blog.sina.com.cn/s/articlelist_1487828712_0_1.html")
         // 启用的线程数
         .thread(5).run();
         }
 }

运行结果

爬取到的数据

总结：

关于简单的页面基本可以实现爬取，并且用对象进行存储数据，并最终保存为txt文档。

目前存在的问题，在于一些前端渲染的页面，还找不到url链接去完成相应的爬取，还需要进一步学习模拟登录页面，以获得隐藏的url等数据。