HBase常用操作-HBaseUtil
package com.zhen.hbase; import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.Map.Entry; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.PageFilter;
import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.Logger; public class HBaseUtil { //@Value("${hbase.zookeeper.quorum}")
private String addr="HDP1,HDP2,HDP3"; //@Value("${hbase.zookeeper.property.clientPort}")
private String port="2181"; Logger logger = Logger.getLogger(getClass()); private static Connection connection; @SuppressWarnings("static-access")
public void getConnection(){
Configuration conf = HBaseConfiguration.create(); conf.set("hbase.zookeeper.quorum",addr);
conf.set("hbase.zookeeper.property.clientPort", port);
try {
this.connection = ConnectionFactory.createConnection(conf);
} catch (IOException e) {
e.printStackTrace();
}
} public HBaseUtil() {
getConnection();
} /*
* 判断表是否存在
*
*
* @tableName 表名
*/ public boolean isExist(String tableName) throws IOException {
TableName table_name = TableName.valueOf(tableName);
Admin admin = connection.getAdmin();
boolean exit = admin.tableExists(table_name);
admin.close();
return exit;
} /**
* 单行添加
* @param tableName
* @param rowKey
* @param family
* @param keyValue
* @throws IOException
*/
@SuppressWarnings("unused")
private static void addRow(String tableName, String rowKey, String family, Map<String, String> keyValue) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Put put = new Put(Bytes.toBytes(rowKey));
for (Entry<String, String> entry : keyValue.entrySet()) {
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(entry.getKey()), Bytes.toBytes(entry.getValue()));
}
table.put(put);
table.close();
keyValue.clear();
} public static void addRows(String tableName, String rowFamilySeparator, Map<String, Map<String, String>> keyValues) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
List<Put> puts = new ArrayList<Put>();
for (Entry<String, Map<String, String>> entry : keyValues.entrySet()) {
String key = entry.getKey();
if (null == rowFamilySeparator || rowFamilySeparator.isEmpty()) {
rowFamilySeparator = "_";
}
String rowKey = key.split(rowFamilySeparator)[0];
String family = key.split(rowFamilySeparator)[1];
Map<String, String> keyValue = entry.getValue();
Put put = new Put(Bytes.toBytes(rowKey), System.currentTimeMillis());
for (Entry<String, String> entry2 : keyValue.entrySet()) {
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(entry2.getKey()), Bytes.toBytes(entry2.getValue()));
}
puts.add(put);
}
table.put(puts);
table.close();
keyValues.clear();
} /**
* 单行删除
* @param tableName
* @param rowKey
* @param family
* @throws IOException
*/
public static void deleteByRowKey(String tableName, String rowKey) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Delete delete = new Delete(Bytes.toBytes(rowKey));
table.delete(delete);
table.close();
} /**
* 查询所有
* @param tableName
* @param family
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForScan(String tableName, String family) throws IOException {
List<Map<String, String>> rows = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(family));
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell)));
}
rows.add(row);
}
} finally {
rs.close();
}
return rows;
} /**
* 查询所有rowKey
* @param tableName
* @return
* @throws IOException
*/
public static List<String> queryAllRowKeyForScan(String tableName) throws IOException {
List<String> result = new ArrayList<String>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
if (null == cells || cells.length <= 0) {
continue;
}
Cell cell = cells[0];
String rowKey = new String(CellUtil.cloneRow(cell));
result.add(rowKey);
}
} finally {
rs.close();
}
return result;
} /**
* 查询所有字段
* @param tableName
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForScan(String tableName) throws IOException {
List<Map<String, String>> rows = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put("rowKey", new String(CellUtil.cloneRow(cell)));
row.put("family", new String(CellUtil.cloneFamily(cell)));
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell)));
}
rows.add(row);
}
} finally {
rs.close();
}
return rows;
} /**
* 根据时间范围
* @param tableName
* @param family
* @param minStamp
* @param maxStamp
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForTimeRange(String tableName, String family, long minStamp, long maxStamp) throws IOException {
List<Map<String, String>> rows = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(family));
scan.setTimeRange(minStamp, maxStamp);
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell)));
}
rows.add(row);
}
} finally {
rs.close();
}
return rows;
} /**
* 根据RowKey查询
* @param tableName
* @param rowKey
* @param family
* @return
* @throws IOException
*/
public static Map<String, String> queryForRowKey(String tableName, String rowKey, String family) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Get get = new Get(Bytes.toBytes(rowKey));
get.addFamily(Bytes.toBytes(family));
Scan scan = new Scan(get);
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell),"UTF-8"));
}
}
} finally {
rs.close();
}
return row;
} /**
* 根据多个RowKey查询
* @param tableName
* @param rowKeys
* @param family
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForRowKeys(String tableName, List<String> rowKeys, String family) throws IOException {
List<Map<String, String>> resultList = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
List<Get> getList = new ArrayList();
for (String rowKey : rowKeys) {
Get get = new Get(Bytes.toBytes(rowKey));
get.addFamily(Bytes.toBytes(family));
getList.add(get);
}
Result[] results = table.get(getList);
for (Result result : results){//对返回的结果集进行操作
Map<String, String> row = new HashMap<String, String>();
for (Cell kv : result.rawCells()) {
row.put(new String(CellUtil.cloneQualifier(kv)), new String(CellUtil.cloneValue(kv),"UTF-8"));
}
resultList.add(row);
}
return resultList;
} /**
* 根据RowKey范围查询
* @param tableName
* @param family
* @param startRow
* @param stopRow
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForRowKeyRange(String tableName, String family, String startRow, String stopRow) throws IOException {
List<Map<String, String>> rows = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(family));
scan.setStartRow(Bytes.toBytes(startRow));
scan.setStopRow(Bytes.toBytes(stopRow));
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put("timestamp", cell.getTimestamp() + "");
row.put("rowKey", new String(CellUtil.cloneRow(cell)));
row.put("family", new String(CellUtil.cloneFamily(cell)));
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell)));
}
rows.add(row);
}
} finally {
rs.close();
}
return rows;
} /**
* 根据指定列名匹配列值
* @param tableName
* @param family
* @param qualifier
* @param value
* @return
* @throws IOException
*/
public static Collection<Map<String, String>> queryForQuilfier(String tableName, String family, String column, String value) throws IOException {
Map<String, Map<String, String>> rows = new HashMap<String, Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
SubstringComparator comp = new SubstringComparator(value);
SingleColumnValueFilter filter = new SingleColumnValueFilter(family.getBytes(), column.getBytes(), CompareOp.EQUAL, comp);
filter.setFilterIfMissing(true);
PageFilter p = new PageFilter(5);
scan.setFilter(filter);
scan.setFilter(p);
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
String rowKey = new String(CellUtil.cloneRow(cell));
if (null == row || !rows.containsKey(rowKey)) {
row = new HashMap<String, String>();
}
row.put("timestamp", cell.getTimestamp() + "");
row.put("rowKey", rowKey);
row.put("family", new String(CellUtil.cloneFamily(cell)));
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell),"UTF-8"));
rows.put(rowKey,row);
}
}
} finally {
rs.close();
}
return rows.values();
} /**
* 根据指定列名完全匹配列值
* @param tableName
* @param family
* @param qualifier
* @param value
* @return
* @throws IOException
*/
public static Collection<Map<String, String>> queryForQuilfierExactly(String tableName, String family, String column, String value) throws IOException {
Map<String, Map<String, String>> rows = new HashMap<String, Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
SingleColumnValueFilter filter = new SingleColumnValueFilter(family.getBytes(), column.getBytes(), CompareOp.EQUAL,value.getBytes());
filter.setFilterIfMissing(true);
scan.setFilter(filter);
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
String rowKey = new String(CellUtil.cloneRow(cell));
if (null == row || !rows.containsKey(rowKey)) {
row = new HashMap<String, String>();
}
row.put("timestamp", cell.getTimestamp() + "");
row.put("rowKey", rowKey);
row.put("family", new String(CellUtil.cloneFamily(cell)));
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell),"UTF-8"));
rows.put(rowKey,row);
}
}
} finally {
rs.close();
}
return rows.values();
} /**
* 获取列名匹配的rowkey
* @param tableName
* @param qualifier 列名
* @param pageSize 数量
* @return
* @throws IOException
*/
public static List<String> queryForQuilfierName(String tableName, String qualifier, long pageSize) throws IOException {
Set<String> rowKeys = new HashSet<String>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
Filter filter = new QualifierFilter(CompareOp.EQUAL,new BinaryComparator(Bytes.toBytes(qualifier)));
PageFilter pageFilter = new PageFilter(pageSize);
FilterList filterList = new FilterList();
filterList.addFilter(filter);
filterList.addFilter(pageFilter);
scan.setFilter(filterList);
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
String rowKey = new String(CellUtil.cloneRow(cell));
rowKeys.add(rowKey);
}
}
} finally {
rs.close();
}
List<String> rows = new ArrayList<String>(rowKeys);
return rows;
} /**
* 获取多个列值匹配的rowkey
* @param tableName
* @param family
* @param qualifierMap 列名:列值
* @param pageSize 数量
* @return
* @throws IOException
*/
public static List<String> queryForMultiQuilfierName(String tableName, String family, Map<String, String> qualifierMap, long pageSize) throws IOException {
Set<String> rowKeys = new HashSet<String>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
for (Entry<String, String> entry : qualifierMap.entrySet()) {
SingleColumnValueFilter filter = new SingleColumnValueFilter(family.getBytes(), entry.getKey().getBytes(), CompareOp.EQUAL,entry.getValue().getBytes());
filter.setFilterIfMissing(true);
filterList.addFilter(filter);
}
PageFilter pageFilter = new PageFilter(pageSize);
filterList.addFilter(pageFilter);
scan.setFilter(filterList);
scan.addFamily(Bytes.toBytes(family));
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
String rowKey = new String(CellUtil.cloneRow(cell));
rowKeys.add(rowKey);
}
}
} finally {
rs.close();
}
List<String> rows = new ArrayList<String>(rowKeys);
return rows;
} public static void qualifierFilter() throws IOException {
Table mTable = connection.getTable(TableName.valueOf("portrait"));
QualifierFilter columnsNameFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator("basictag-14".getBytes()));
Scan scan = new Scan();
scan.setFilter(columnsNameFilter);
ResultScanner rs = mTable.getScanner(scan);
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
System.out.println("==== "+new String(CellUtil.cloneRow(cell))+"\t"+
new String(CellUtil.cloneFamily(cell))+"\t"+
new String(CellUtil.cloneQualifier(cell))+"\t"+
new String(CellUtil.cloneValue(cell)));
}
}
} /**
* 获取列名匹配的rowkey
* @param tableName
* @param qualifier 列名
* @param pageSize 数量
* @return
* @throws IOException
*/
public static List<String> queryForQuilfierName(String tableName, String family, String qualifier, long pageSize) throws IOException {
Set<String> rowKeys = new HashSet<String>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
Filter qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL,new BinaryComparator(qualifier.getBytes()));
PageFilter pageFilter = new PageFilter(pageSize);
FilterList list = new FilterList();
list.addFilter(pageFilter);
list.addFilter(qualifierFilter);
scan.addFamily(Bytes.toBytes(family));
scan.setFilter(list);
System.out.println(scan.toJSON());
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
System.out.println("==== "+new String(CellUtil.cloneRow(cell))+"\t"+
new String(CellUtil.cloneFamily(cell))+"\t"+
new String(CellUtil.cloneQualifier(cell))+"\t"+
new String(CellUtil.cloneValue(cell)));
String rowKey = new String(CellUtil.cloneRow(cell));
rowKeys.add(rowKey);
}
}
} finally {
rs.close();
}
List<String> rows = new ArrayList<String>(rowKeys);
return rows;
} /**
* 根据指定列名匹配列值条数
* @param tableName
* @param family
* @param qualifier
* @param value
* @return
* @throws IOException
*/
public static long queryForQuilfierCount(String tableName, String family, String column, String value) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
SubstringComparator comp = new SubstringComparator(value);
SingleColumnValueFilter filter = new SingleColumnValueFilter(family.getBytes(), column.getBytes(), CompareOp.EQUAL, comp);
filter.setFilterIfMissing(true);
scan.setFilter(filter);
ResultScanner rs = table.getScanner(scan);
long count = 0;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
count++;
}
} finally {
rs.close();
}
return count;
} /*
* 关闭连接
*
*/
public void close() {
/**
* close connection
**/
if (connection != null) {
try {
connection.close();
} catch (IOException e) {
e.printStackTrace();
}
}
} }
HBase常用操作-HBaseUtil的更多相关文章
- Hbase常用操作记录
Hbase常用操作记录 Hbase 创建表 查看表结构 修改表结构 删除表 创建表 语法:create <table>, {NAME => <family>, VERSI ...
- Hbase常用操作(增删改查)
Hbase常用操作(增删改查) [日期:2014-01-03] 来源:Linux社区 作者:net19880504 [字体:大 中 小] 运行Eclipse,创建一个新的Java工程“HBa ...
- HBase常用操作之namespace
1.介绍 在HBase中,namespace命名空间指对一组表的逻辑分组,类似RDBMS中的database,方便对表在业务上划分.Apache HBase从0.98.0, 0.95.2两个版本开始支 ...
- Hbase常用操作
下面我们看看HBase Shell的一些基本操作命令,我列出了几个常用的HBase Shell命令,如 名称 命令表达式 创建表 create '表名称', '列名称1','列名称2','列名称N' ...
- HBASE常用操作增删改查
http://javacrazyer.iteye.com/blog/1186881 http://www.cnblogs.com/invban/p/5667701.html
- Hbase到Solr同步常用操作
Hbase到Solr同步常用操作 1. 整体流程 2. 常用操作 Hbase常用操作 Solr常用操作 hbase-index常用操作 3. 其他资料 Lily HBase Indexer使用整理 h ...
- HBase Shell操作
Hbase 是一个分布式的.面向列的开源数据库,其实现是建立在google 的bigTable 理论之上,并基于hadoop HDFS文件系统. Hbase不同于一般的关系型数据库(RDBMS ...
- HBase常用操作命令
HBase常用操作命令 1.进入HBase脚本客户端 #hbase shell #进入HBase脚本客户端 > whoami #查看当前登录用户 > status ...
- HBase伪分布式安装(HDFS)+ZooKeeper安装+HBase数据操作+HBase架构体系
HBase1.2.2伪分布式安装(HDFS)+ZooKeeper-3.4.8安装配置+HBase表和数据操作+HBase的架构体系+单例安装,记录了在Ubuntu下对HBase1.2.2的实践操作,H ...
随机推荐
- SpringCloud系列二：硬编码实现简单的服务提供者与服务消费者
从本文开始,以一个电影售票系统为例讲解Spring Cloud 1. 版本 jdk:1.8 SpringBoot:2.0.0.RELEASE SpringCloud:Finchley.M8 2. 系统 ...
- 【POJ 2400】 Supervisor, Supervisee（KM求最小权匹配）
[POJ 2400] Supervisor, Supervisee(KM求最小权匹配) Supervisor, Supervisee Time Limit: 1000MS Memory Limit ...
- Atitit. Object-c语言 的新的特性 attilax总结
Atitit. Object-c语言 的新的特性 attilax总结 1.1. Object-C语言由 Brad J.Cox于20世纪80年代早期设计,1 1.2. Object-C新增的数据结构: ...
- Android实现一键获取课程成绩dome
欢迎转载但请标明出处:http://blog.csdn.net/android_for_james/article/details/50984493 两周废寝忘食的创作最终成功了,如今拿出来分享一下. ...
- How to Use HTML5 FUll Screen API（怎样使用HTML5全屏接口）
原文链接:http://www.sitepoint.com/use-html5-full-screen-api/ 假设你不太喜欢变化太快的东西,那么web开发可能不适合你. 我曾在2012年末有写过F ...
- kafka eagle 使用教程
下载 地址:http://download.smartloli.org/ github:https://github.com/smartloli/kafka-eagle 环境 Windows: 安装J ...
- IDEA 去掉 ALT + / 自动补全
取消掉Alt + / 自动补全 setting -> keymap -> main menu -> code -> completion -> cyclic Expand ...
- Java反射学习系列-绪论
Java反射学习系列-绪论 https://blog.csdn.net/hanchao5272/article/details/79358924
- mysql 与mongodb的特点与优劣
首先我们来分析下mysql 与mongodb的特点与优劣.下面是我以前做的ppt的部分截图. 再来分析下应用场景,a.如果需要将mongodb作为后端db来代替mysql使用,即这里mysql与mon ...
- ios输出想保留的整数位（占位符）
int startHour=5; int startMinute=4; //输出前面补0,不管你输入的数据前面有没有0 sp;NSLog(@"====>%@",[NSStri ...