在单机上运行hbase 二级索引：

import java.io.IOException;

import java.util.HashMap;

import java.util.Map;

import java.util.Set;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.hbase.HBaseConfiguration;

import org.apache.hadoop.hbase.client.Put;

import org.apache.hadoop.hbase.client.Result;

import org.apache.hadoop.hbase.client.Scan;

import org.apache.hadoop.hbase.io.ImmutableBytesWritable;

import org.apache.hadoop.hbase.mapreduce.MultiTableOutputFormat;

import org.apache.hadoop.hbase.mapreduce.TableInputFormat;

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;

import org.apache.hadoop.hbase.mapreduce.TableMapper;

import org.apache.hadoop.hbase.util.Bytes;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import com.tansun.di.core.di.core.hbase.HBaseDaoPool;

public class IndexCreateExample extends

		TableMapper<ImmutableBytesWritable, Put> {

	public static Configuration conf = null;

	private String familyName;

	// 原始表

	private Map<byte[], ImmutableBytesWritable> indexes = new HashMap<byte[], ImmutableBytesWritable>();

	 static {

	        conf = new Configuration();

	        String filePath = "hbase-site.xml";

	        Path path = new Path( filePath );

	        conf.addResource( path );

	        conf = HBaseConfiguration.create( conf );

	    }

//	public static class IndexMapper extends

	@Override

	protected void setup(

			Mapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, Put>.Context context)

			throws IOException, InterruptedException {

		String tableName = conf.get("tableName ");

		// String family = conf.get("familyName");

		// 获得列族

		familyName = conf.get("familyName");

		// 获得列

		String[] qualifiers = conf.getStrings("qualifiers");

		for (String qualifier : qualifiers) {

			// 建立一个映射，为每一个列创建一个表，表的名字tableName+"-"+qualifier

			// 原始表的列 索引表新建表名

			indexes.put(Bytes.toBytes(qualifier), new ImmutableBytesWritable(

					Bytes.toBytes(tableName + "-" + qualifier)));

		}

	}

	public static void main(String[] args) {

		if (args.length < 3) {

			System.err

					.println("Usage: IndexCreateExample <table_name><cf><cn>");

			System.exit(-1);

		}

		// 表名

		String tableName = args[0];

		// 列族

		String columnFamily = args[1];

		conf.set("tableName", tableName);

		conf.set("columnFamily", columnFamily);

		// 列 可能存在多个列

		String[] qualifiers = new String[args.length - 2];

		for (int i = 0; i < qualifiers.length; i++) {

			qualifiers[i] = args[i + 2];

		}

		// 设置列

		conf.setStrings("qualifiers", qualifiers);

		@SuppressWarnings("deprecation")

		Job job;

		try {

			job = new Job(conf, tableName);

			job.setJarByClass(IndexCreateExample.class);

			job.setMapperClass(IndexCreateExample.class);

			job.setNumReduceTasks(0);// 由于不需要执行reduce阶段

			job.setInputFormatClass(TableInputFormat.class);

			job.setOutputFormatClass(MultiTableOutputFormat.class);

			Scan scan = new Scan();

			TableMapReduceUtil.initTableMapperJob(tableName, scan,

					IndexCreateExample.class, ImmutableBytesWritable.class,

					Put.class, job);

			job.waitForCompletion(true);

		} catch (Exception e1) {

			e1.printStackTrace();

		}

	}

	@Override

	protected void map(

			ImmutableBytesWritable key,

			Result value,

			Mapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, Put>.Context context)

			throws IOException, InterruptedException {

		// 获得tableName 和

		Set<byte[]> keys = indexes.keySet();

		for (byte[] k : keys) {

			// index tableName

			ImmutableBytesWritable indexTableName = indexes.get(k);

			//

			//

			byte[] value2 = value.getValue(Bytes.toBytes(familyName), k);

			if (value2 != null) {

				// 索引表

				Put put = new Put(value2);// 索引表行键

				// 列族 列 原始表的行键

				put.add(Bytes.toBytes("f1"), Bytes.toBytes("id"), key.get());

				context.write(indexTableName, put);

			}

		}

	}

}

出现异常信息：内存不足

or more detailed output, check application tracking page:http://ts.node2.com:8088/proxy/application_1472537544791_0007/Then, click on links to logs of each attempt.

Diagnostics: Container [pid=3984,containerID=container_1472537544791_0007_02_000001] is running beyond physical memory limits. Current usage: 285.6 MB of 256 MB physical memory used; 1.5 GB of 537.6 MB virtual memory used. Killing container.

Dump of the process-tree for container_1472537544791_0007_02_000001 :

	|- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE

	|- 3984 3982 3984 3984 (bash) 0 0 108617728 338 /bin/bash -c /usr/java/jdk1.7.0_79/bin/java -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/var/log/hadoop-yarn/container/application_1472537544791_0007/container_1472537544791_0007_02_000001 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA  -Djava.net.preferIPv4Stack=true -Xmx825955249 org.apache.hadoop.mapreduce.v2.app.MRAppMaster 1>/var/log/hadoop-yarn/container/application_1472537544791_0007/container_1472537544791_0007_02_000001/stdout 2>/var/log/hadoop-yarn/container/application_1472537544791_0007/container_1472537544791_0007_02_000001/stderr

	|- 3999 3984 3984 3984 (java) 2183 69 1504305152 72786 /usr/java/jdk1.7.0_79/bin/java -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/var/log/hadoop-yarn/container/application_1472537544791_0007/container_1472537544791_0007_02_000001 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Djava.net.preferIPv4Stack=true -Xmx825955249 org.apache.hadoop.mapreduce.v2.app.MRAppMaster 

Container killed on request. Exit code is 143

Container exited with a non-zero exit code 143

Failing this attempt. Failing the application.

16/09/01 11:07:56 INFO mapreduce.Job: Counters: 0

异常2：

16/08/31 20:40:03 ERROR mapreduce.TableInputFormat: java.io.IOException: java.lang.reflect.InvocationTargetException

	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:240)

	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:218)

	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:119)

	at org.apache.hadoop.hbase.mapreduce.TableInputFormat.initialize(TableInputFormat.java:183)

	at org.apache.hadoop.hbase.mapreduce.TableInputFormatBase.getSplits(TableInputFormatBase.java:230)

	at org.apache.hadoop.hbase.mapreduce.TableInputFormat.getSplits(TableInputFormat.java:237)

	at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:597)

	at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:614)

	at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:492)

	at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1306)

	at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1303)

	at java.security.AccessController.doPrivileged(Native Method)

	at javax.security.auth.Subject.doAs(Subject.java:415)

	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)

	at org.apache.hadoop.mapreduce.Job.submit(Job.java:1303)

	at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1324)

	at com.tansun.di.core.di.core.hbase.second.IndexCreateExample.main(IndexCreateExample.java:101)

	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)

	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

	at java.lang.reflect.Method.invoke(Method.java:606)

	at org.apache.hadoop.util.RunJar.run(RunJar.java:221)

	at org.apache.hadoop.util.RunJar.main(RunJar.java:136)

Caused by: java.lang.reflect.InvocationTargetException

	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)

	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)

	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)

	at java.lang.reflect.Constructor.newInstance(Constructor.java:526)

	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:238)

	... 22 more

Caused by: java.lang.UnsupportedOperationException: Unable to find org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory

	at org.apache.hadoop.hbase.util.ReflectionUtils.instantiateWithCustomCtor(ReflectionUtils.java:36)

	at org.apache.hadoop.hbase.ipc.RpcControllerFactory.instantiate(RpcControllerFactory.java:58)

	at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.createAsyncProcess(ConnectionManager.java:2220)

	at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.<init>(ConnectionManager.java:676)

	at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.<init>(ConnectionManager.java:618)

	... 27 more

Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory

	at java.net.URLClassLoader$1.run(URLClassLoader.java:366)

	at java.net.URLClassLoader$1.run(URLClassLoader.java:355)

	at java.security.AccessController.doPrivileged(Native Method)

	at java.net.URLClassLoader.findClass(URLClassLoader.java:354)

	at java.lang.ClassLoader.loadClass(ClassLoader.java:425)

	at java.lang.ClassLoader.loadClass(ClassLoader.java:358)

	at java.lang.Class.forName0(Native Method)

	at java.lang.Class.forName(Class.java:191)

	at org.apache.hadoop.hbase.util.ReflectionUtils.instantiateWithCustomCtor(ReflectionUtils.java:32)

	... 31 more

16/08/31 20:40:03 INFO mapreduce.JobSubmitter: Cleaning up the staging area /tmp/hadoop-yarn/staging/root/.staging/job_1472695767533_0003

16/08/31 20:40:03 WARN security.UserGroupInformation: PriviledgedActionException as:root (auth:SIMPLE) cause:java.io.IOException: Cannot create a record reader because of a previous error. Please look at the previous logs lines from the task's full log for more details.

java.io.IOException: Cannot create a record reader because of a previous error. Please look at the previous logs lines from the task's full log for more details.

	at org.apache.hadoop.hbase.mapreduce.TableInputFormatBase.getSplits(TableInputFormatBase.java:241)

	at org.apache.hadoop.hbase.mapreduce.TableInputFormat.getSplits(TableInputFormat.java:237)

	at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:597)

	at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:614)

	at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:492)

	at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1306)

	at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1303)

	at java.security.AccessController.doPrivileged(Native Method)

	at javax.security.auth.Subject.doAs(Subject.java:415)

	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)

	at org.apache.hadoop.mapreduce.Job.submit(Job.java:1303)

	at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1324)

	at com.tansun.di.core.di.core.hbase.second.IndexCreateExample.main(IndexCreateExample.java:101)

	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)

	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

	at java.lang.reflect.Method.invoke(Method.java:606)

	at org.apache.hadoop.util.RunJar.run(RunJar.java:221)

	at org.apache.hadoop.util.RunJar.main(RunJar.java:136)

Caused by: java.lang.IllegalStateException: The input format instance has not been properly initialized. Ensure you call initializeTable either in your constructor or initialize method

	at org.apache.hadoop.hbase.mapreduce.TableInputFormatBase.getTable(TableInputFormatBase.java:389)

	at org.apache.hadoop.hbase.mapreduce.TableInputFormatBase.getSplits(TableInputFormatBase.java:236)

创建原始表

hbase(main):002:0> create 'studentinfo','f1'

0 row(s) in 0.6520 seconds

=> Hbase::Table - studentinfo

hbase(main):003:0> put 'studentinfo','1','f1:name','zhangsan'

0 row(s) in 0.1640 seconds

hbase(main):004:0> put 'studentinfo','2','f1:name','lisi'

0 row(s) in 0.0240 seconds

hbase(main):005:0> put 'studentinfo','3','f1:name','wangwu'

0 row(s) in 0.0290 seconds

hbase(main):006:0> scan 'studentinfo'

ROW                      COLUMN+CELL

 1                       column=f1:name, timestamp=1436262175823, value=zhangsan

 2                       column=f1:name, timestamp=1436262183922, value=lisi

 3                       column=f1:name, timestamp=1436262189250, value=wangwu

3 row(s) in 0.0530 seconds

创建索引表 hbase(main):007:0> create 'studentinfo-name','f1'

0 row(s) in 0.7740 seconds => Hbase::Table - studentinfo-name

hbase 二级索引创建的更多相关文章

  1. HBase二级索引的设计(案例讲解)

    摘要 最近做的一个项目涉及到了多条件的组合查询,数据存储用的是HBase,恰恰HBase对于这种场景的查询特别不给力,一般HBase的查询都是通过RowKey(要把多条件组合查询的字段都拼接在RowK ...

  2. HBase二级索引方案总结

    转自:http://blog.sina.com.cn/s/blog_4a1f59bf01018apd.html 附hbase如何创建二级索引以及创建二级索引实例:http://www.aboutyun ...

  3. HBase二级索引的设计

    摘要 最近做的一个项目涉及到了多条件的组合查询,数据存储用的是HBase,恰恰HBase对于这种场景的查询特别不给力,一般HBase的查询都是通过RowKey(要把多条件组合查询的字段都拼接在RowK ...

  4. HBase之八--(1)：HBase二级索引的设计(案例讲解)

    摘要 最近做的一个项目涉及到了多条件的组合查询,数据存储用的是HBase,恰恰HBase对于这种场景的查询特别不给力,一般HBase的查询都是通过RowKey(要把多条件组合查询的字段都拼接在RowK ...

  5. HBase二级索引、读写流程

    HBase二级索引.读写流程 一.HBse二级索引方案 1.1 基于Coprocessor方案 1.2 Phoenix二级索引特点 1.3 Phoenix 二级索引方案 二.HBase读写流程 2.1 ...

  6. [转]HBASE 二级索引

    1.二级索引的核心思想是什么?2.二级索引由谁来管理?3.在主表中插入某条数据后,hbase如何将索引列写到索引表中去?4.scan查询的时候,coprocessor钩子的作用是什么?5.在split ...

  7. HBase之八--(2)：HBase二级索引之Phoenix

    1. 介绍 Phoenix 是 Salesforce.com 开源的一个 Java 中间件,可以让开发者在Apache HBase 上执行 SQL 查询.Phoenix完全使用Java编写,代码位于 ...

  8. HBase 二级索引与Coprocessor协处理器

    Coprocessor简介 (1)实现目的 HBase无法轻易建立“二级索引”: 执行求和.计数.排序等操作比较困难,必须通过MapReduce/Spark实现,对于简单的统计或聚合计算时,可能会因为 ...

  9. CDH使用Solr实现HBase二级索引

      一.为什么要使用Solr做二级索引二.实时查询方案三.部署流程3.1 安装HBase.Solr3.2 增加HBase复制功能3.3创建相应的 SolrCloud 集合3.4 创建 Lily HBa ...

随机推荐

  1. IOS 开发调试方法

    0.警告 尽量一个警告都不要有 1.错误 1)红色提示 编译过不去的原因大部分是语法,检查括号的匹配,变量名称,作用域范围 2)编译可以通过,可以运行 a.运行过程中程序崩溃 在debug区域的右侧, ...

  2. redis学习研究--Redis作者谈Redis应用场景

    毫无疑问,Redis开创了一种新的数据存储思路,使用Redis,我们不用在面对功能单调的数据库时,把精力放在如何把大象放进冰箱这样的问题上,而是利用Redis灵活多变的数据结构和数据操作,为不同的大象 ...

  3. oracle 联表更新

    依 a 表 cate_pub_id  为依据 更新 v 表的 cate_pub_id update td_tobrel_cate_pub_attrval v set v.CATE_PUB_ID=(se ...

  4. 微信开发之门店管理{"errcode":40097,"errmsg":"invalid args hint: [xxxxxxx]"}

    最近在做微信端开发,做到门店开发部分,在创建门店的时候遇到40097问题{"errcode":40097,"errmsg":"invalid args ...

  5. 利用radio实现纯css选项卡切换

    aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAAtIAAABCCAIAAAD18aaXAAAG70lEQVR4nO3dO5KrPBCGYRY0VV4OEX

  6. BC 65 ZYB's Premutation (线段树+二分搜索)

    题目简述:有一个全排列,一直每个前缀区间的逆序对数,还原这个排列. fi记录逆序对数,pi记录该位置数值,则k=fi-f(i-1)表示前i-1个数比pi大的数的个数,那么只要在剩余元素求出按大小顺序第 ...

  7. C语言中预定义符 __FILE__, __LINE__, __FUNCTION__, __DATE__, __TIME__ 的使用演示

    本文演示了C语言中预定义符 __FILE__, __LINE__, __FUNCTION__, __DATE__, __TIME__ 的使用. 这几个预定义符的名称就没必要再介绍了,顾名思义嘛. // ...

  8. 找不到Qt5Cored.dll（Release和Debug版连接了不同的库）

    Qt5Cored.dll和Qt5Core.dll文件分别用于Qt软件的Debug版和Release版. 通常会有两个Qt5Core.dll文件,分别位于Qti安装目录下的“Qt5.1.0\5.1.0\ ...

  9. Linux下GDB调试C/C++

    首先先编译程序并生成调试符号: gcc -g -c main.cpp gcc -o exefile main.o 以上的exefile为可执行程序的文件名 然后: gdb exefile 可以开始gd ...

  10. [置顶] ruby复制对象的方法(dup 和 clone)

    Ruby内置的方法Object#clone和Object#dup可以用来copy一个对象,两者区别是dup只复制对象的内容,而clone还复制与对象相关联的内容,如singleton method [ ...