webcollector 2.x 爬取搜狗搜索结果页

 /**
  * 使用搜狗搜索检索关键字并爬取结果集的标题
  * @author tele
  *
  */
 public class SougouCrawler extends RamCrawler{

     public SougouCrawler() {

     }

     public SougouCrawler(String keyword,int maxnum) {
         for(int i=1;i<=maxnum;i++) {
             //拼接url
             String url ="https://www.sogou.com/web?query="+keyword+"&s_from=result_up&cid=&page="+ i +"&ie=utf8&p=40040100&dp=1&w=01029901&dr=1";
             CrawlDatum crawlDatum = new CrawlDatum(url).meta("pageNum",i);
             addSeed(crawlDatum);
             addRegex(".*");
         }
     }

     @Override
     public void visit(Page page, CrawlDatums next) {
         String pageNum = page.meta("pageNum");
         Elements results = page.doc().select("div.results div[^class] h3 a");
         for(int i=0;i<results.size();i++) {
             System.out.println("第"+ pageNum +"页第"+ (i+1) +"条结果------" + results.get(i).text());
         }
     }

     public static void main(String[] args) throws Exception {
         String keyword="淘宝";
         SougouCrawler crawler  = new SougouCrawler(keyword,3);
         crawler.setThreads(8);

         Configuration conf = Configuration.copyDefault();
         conf.setExecuteInterval(3000);
         conf.setReadTimeout(5000);
         conf.setWaitThreadEndTime(3000);

         crawler.setConf(conf);
         crawler.start(1);//只有一层
     }
 }

输出截图(部分)

验证第二页数据标题