crawler4j源码学习(1)：搜狐新闻网新闻标题采集爬虫

crawler4j是用Java实现的开源网络爬虫。提供了简单易用的接口，可以在几分钟内创建一个多线程网络爬虫。下面实例结合jsoup，采集搜狐新闻网（http://news.sohu.com/）新闻标题信息。

所有的过程仅需两步完成：

第一步：建立采集程序核心部分

 /**
  * @date 2016年8月20日 上午11:52:13
  * @version
  * @since JDK 1.8
  */
 public class MyCrawler extends WebCrawler {

     //链接地址过滤//
     private final static Pattern FILTERS = Pattern.compile(".*(\\.(css|js|gif|jpg" + "|png|mp3|mp3|zip|gz))$");

     @Override
     public boolean shouldVisit(Page referringPage, WebURL url) {
         String href = url.getURL().toLowerCase();
         return !FILTERS.matcher(href).matches() && href.startsWith("http://news.sohu.com/");
     }

     /**
      * This function is called when a page is fetched and ready to be processed
      * by your program.
      */
     @Override
     public void visit(Page page) {
         String url = page.getWebURL().getURL();
         logger.info("URL: " + url);

         if (page.getParseData() instanceof HtmlParseData) {
             HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
             String text = htmlParseData.getText();
             String html = htmlParseData.getHtml();
             Set<WebURL> links = htmlParseData.getOutgoingUrls();

             logger.debug("Text length: " + text.length());
             logger.debug("Html length: " + html.length());
             logger.debug("Number of outgoing links: " + links.size());
             logger.info("Title: " + htmlParseData.getTitle());

         }
     }

 }

第二步：建立采集程序控制部分

 /**
  * @date 2016年8月20日 上午11:55:56
  * @version
  * @since JDK 1.8
  */
 public class MyController {

     /**
      * @param args
      * @since JDK 1.8
      */
     public static void main(String[] args) {
         // TODO Auto-generated method stub

         //本地嵌入式数据库，采用berkeley DB
         String crawlStorageFolder = "data/crawl/root";
         int numberOfCrawlers = 3;

         CrawlConfig config = new CrawlConfig();
         config.setCrawlStorageFolder(crawlStorageFolder);

         /*
          * Instantiate the controller for this crawl.
          */
         PageFetcher pageFetcher = new PageFetcher(config);
         RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
         RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
         CrawlController controller;
         try {
             controller = new CrawlController(config, pageFetcher, robotstxtServer);
             controller.addSeed("http://news.sohu.com/");
             controller.start(MyCrawler.class, numberOfCrawlers);
         } catch (Exception e) {
             // TODO Auto-generated catch block
             e.printStackTrace();
         }

     }

 }

采集结果展示：