爬虫代码实现四：采用Hbase存储爬虫数据(2)

导入hbase的jar包，在maven仓库找：进入http://mavenrepository.com/,输入hbase client，选择apache hbase client，

点击进入，选择

点击进入：

将这段配置

<!-- https://mvnrepository.com/artifact/org.apache.hbase/hbase-client -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>0.98.11-hadoop2</version>
</dependency>

复制到pom.xml中

准备好以上工作后，代码操作如下：

1.新建一个Hbase工具类，这里直接用已经写好的现成工具。

2.Page实体类添加属性tvId，并且添加set/get方法。

之前我们通过观察，点击不同的电视剧，url分别为：

这里我们如何获取电视剧的id？采用正则表达式。因此，我们在配置文件中添加一个变量idRegex=http://list.youku.com/show/id_([\\w]+).html，为了获取电视剧的id

youku.properties如下：

allnumberRegex=(?<=\u603B\u64AD\u653E\u6570\uFF1A)[\\d,]+
commentnumberRegex=(?<=\u8BC4\u8BBA\uFF1A)[\\d,]+
supportnumberRegex=(?<=\u9876\uFF1A)[\\d,]+
parseAllNumber=/body/div/div/div/div/div/ul/li[11]
parseCommentNumber=//div[@class=\"p-base\"]/ul/li[12]
parseSupportNumber=//div[@class=\"p-base\"]/ul/li[13]
idRegex=http://list.youku.com/show/id_([\\w]+).html

我们知道，RegexUtil的工具类中封装的方法是public static String getPageInfoByRegex(String content,Pattern pattern,int groupNo)，需要传入匹配的内容以及正则表达式以及groupNo，

而这里正则表达式是idRegex=http://list.youku.com/show/id_([\\w]+).html，它对应的匹配内容是page.getUrl,因此，我们在调用RegexUtil.getPageInfoByRegex方法前，还得去设置page.url。所以，

1.回到HttpClientDownLoadService：

package com.dajiangtai.djt_spider.service.impl;

import com.dajiangtai.djt_spider.entity.Page;
import com.dajiangtai.djt_spider.service.IDownLoadService;
import com.dajiangtai.djt_spider.util.PageDownLoadUtil;

public class HttpClientDownLoadService implements IDownLoadService {

public Page download(String url) {

Page page = new Page();
page.setContent(PageDownLoadUtil.getPageContent(url));
page.setUrl(url); //添上这句代码
return page;
}
}

2.YOUKUProcessService 中，添加代码，即通过正则表达式获取电视剧id。通过调用RegexUtil.getPageInfoByRegex方法，注意，groupNo为1，表示，对于匹配的内容，我们使用正则表达式，我们不需要所有的匹配到的字符串如http://list.youku.com/show/id_z9cd2277647d311e5b692.html，我们只需要([\\w]+)正则对应的z9cd2277647d311e5b692。由于这里未使用到XPath，仅仅是正则匹配，因此只需要使用到正则表达式工具。

package com.dajiangtai.djt_spider.service.impl;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.htmlcleaner.XPatherException;

import com.dajiangtai.djt_spider.entity.Page;
import com.dajiangtai.djt_spider.service.IProcessService;
import com.dajiangtai.djt_spider.util.HtmlUtil;
import com.dajiangtai.djt_spider.util.LoadPropertyUtil;
import com.dajiangtai.djt_spider.util.RegexUtil;

public class YOUKUProcessService implements IProcessService {

//这里解析得到的是 ： 总播放数：16,931,628,832，因此采用正则表达式获取数字
// private String allnumberRegex = "(?<=总播放数：)[\\d,]+";
// private String commentnumberRegex = "(?<=评论：)[\\d,]+";
// private String supportnumberRegex = "(?<=顶：)[\\d,]+";
//
// private String parseAllNumber = "/body/div/div/div/div/div/ul/li[11]";
// private String parseCommentNumber = "//div[@class=\"p-base\"]/ul/li[12]";
// private String parseSupportNumber = "//div[@class=\"p-base\"]/ul/li[13]";

public void process(Page page) {
String content = page.getContent();
HtmlCleaner htmlCleaner = new HtmlCleaner();
//利用htmlCleaner对网页进行解析，得到根节点
TagNode rootNode = htmlCleaner.clean(content);

try {
// /html/body/div[4]/div/div[1]/div[2]/div[2]/ul/li[11]
//对XPath做相应的调整，使其有效，如果不该写，则使用debug模式，会发现evaluateXPath为[]
//总播放数
// String allnumber = HtmlUtil.getFieldByRegex(rootNode, parseAllNumber, allnumberRegex);
String allnumber = HtmlUtil.getFieldByRegex(rootNode, LoadPropertyUtil.getYOUKY("parseAllNumber"), LoadPropertyUtil.getYOUKY("allnumberRegex"));
//System.out.println("总播放数数量为:"+allnumber);
page.setAllnumber(allnumber);

//总播放数
String commentnumber = HtmlUtil.getFieldByRegex(rootNode, LoadPropertyUtil.getYOUKY("parseCommentNumber"), LoadPropertyUtil.getYOUKY("commentnumberRegex"));
//System.out.println("总评论数量为:"+commentnumber);
page.setCommentnumber(commentnumber);

//总播放数
String supportnumber = HtmlUtil.getFieldByRegex(rootNode, LoadPropertyUtil.getYOUKY("parseSupportNumber"), LoadPropertyUtil.getYOUKY("supportnumberRegex"));
//System.out.println("总评论数量为:"+supportnumber);
page.setSupportnumber(supportnumber);

page.setDaynumber("0");
page.setAgainstnumber("0");
page.setCollectnumber("0");

//通过正则表达式获取电视剧id
//Pattern numberPattern = Pattern.compile(regex,Pattern.DOTALL);
Pattern numberPattern = Pattern.compile(LoadPropertyUtil.getYOUKY("idRegex"),Pattern.DOTALL);
page.setTvId("tvId_"+RegexUtil.getPageInfoByRegex(page.getUrl(), numberPattern, 1));
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}

3.ConsoleStoreService中添加电视剧id打印信息。

package com.dajiangtai.djt_spider.service.impl;

import com.dajiangtai.djt_spider.entity.Page;
import com.dajiangtai.djt_spider.service.IStoreService;

public class ConsoleStoreService implements IStoreService {

public void store(Page page) {
System.out.println("电视剧id:"+page.getTvId());
System.out.println("总播放数:"+page.getAllnumber());
System.out.println("评论数："+page.getCommentnumber());
System.out.println("赞:"+page.getSupportnumber());
System.out.println("踩："+page.getAgainstnumber());
System.out.println("收藏："+page.getCollectnumber());
System.out.println("每日播放增量："+page.getDaynumber());

}

}

4.运行StartDSJCount的main方法进行测试。

package com.dajiangtai.djt_spider.start;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.dajiangtai.djt_spider.entity.Page;
import com.dajiangtai.djt_spider.service.IDownLoadService;
import com.dajiangtai.djt_spider.service.IProcessService;
import com.dajiangtai.djt_spider.service.IStoreService;
import com.dajiangtai.djt_spider.service.impl.ConsoleStoreService;
import com.dajiangtai.djt_spider.service.impl.HttpClientDownLoadService;
import com.dajiangtai.djt_spider.service.impl.YOUKUProcessService;

/**
* 电视剧爬虫入口类
* @author Administrator
*
*/
public class StartDSJCount {

//页面下载接口
private IDownLoadService downLoadService;
//页面解析接口
private IProcessService processService;
//数据存储接口
private IStoreService storeService;

public static void main(String[] args) {
StartDSJCount dsj = new StartDSJCount();
dsj.setDownLoadService(new HttpClientDownLoadService());
dsj.setProcessService(new YOUKUProcessService());
dsj.setStoreService(new ConsoleStoreService());
String url = "http://list.youku.com/show/id_z9cd2277647d311e5b692.html?spm=a2h0j.8191423.sMain.5~5~A!2.iCUyO9";
//下载页面
Page page = dsj.downloadPage(url);
dsj.processPage(page);
//存储页面信息
dsj.storePageInfo(page);

}

//下载页面方法
public Page downloadPage(String url){
return this.downLoadService.download(url);
}

//解析页面方法
public void processPage(Page page){
this.processService.process(page);
}

//存储页面信息方法
public void storePageInfo(Page page){
this.storeService.store(page);
}
public IDownLoadService getDownLoadService() {
return downLoadService;
}

public void setDownLoadService(IDownLoadService downLoadService) {
this.downLoadService = downLoadService;
}

public IProcessService getProcessService() {
return processService;
}

public void setProcessService(IProcessService processService) {
this.processService = processService;
}

public IStoreService getStoreService() {
return storeService;
}

public void setStoreService(IStoreService storeService) {
this.storeService = storeService;
}

}

5.控制台输出：

电视剧id:tvId_z9cd2277647d311e5b692
总播放数:17,130,904,276
评论数：1,259,621
赞:13,954,307
踩：0
收藏：0
每日播放增量：0