hadoop生态搭建（3节点）-07.hive配置

# http://archive.apache.org/dist/hive/hive-2.1.1/

# ==================================================================安装 hive

tar -zxvf apache-hive-2.1.1-bin.tar.gz -C /usr/local
mv /usr/local/apache-hive-2.1.1-bin /usr/local/hive-2.1.1
rm –r ~/apache-hive-2.1.1-bin.tar.gz

cp ~/mysql-connector-java-5.1.46.jar /usr/local/hive-2.1.1/lib/

# 配置环境变量
# ==================================================================node1 node2 node3

vi /etc/profile

# 在export PATH USER LOGNAME MAIL HOSTNAME HISTSIZE HISTCONTROL下添加
export JAVA_HOME=/usr/java/jdk1.8.0_111
export ZOOKEEPER_HOME=/usr/local/zookeeper-3.4.12
export HADOOP_HOME=/usr/local/hadoop/hadoop-2.7.6
export MYSQL_HOME=/usr/local/mysql
export HBASE_HOME=/usr/local/hbase-1.2.4
export HIVE_HOME=/usr/local/hive-2.1.1

export PATH=$PATH:$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$ZOOKEEPER_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$MYSQL_HOME/bin:$HBASE_HOME/bin:$HIVE_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

export HADOOP_INSTALL=$HADOOP_HOME
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native

# ==================================================================node1

# 使环境变量生效
source /etc/profile

# 查看配置结果
echo $HIVE_HOME

mkdir -p $HIVE_HOME/data/hive
mkdir -p $HIVE_HOME/data/hive/operaion_logs
mkdir -p $HIVE_HOME/data/resources

# 配置 hive
cp $HIVE_HOME/conf/hive-env.sh.template $HIVE_HOME/conf/hive-env.sh
cp $HIVE_HOME/conf/hive-default.xml.template $HIVE_HOME/conf/hive-site.xml
cp $HIVE_HOME/conf/hive-exec-log4j2.properties.template $HIVE_HOME/conf/hive-exec-log4j2.properties
cp $HIVE_HOME/conf/hive-log4j2.properties.template $HIVE_HOME/conf/hive-log4j2.properties

# ${system:java.io.tmpdir}/${system:user.name} 替换为本机路径 /usr/local/hive-2.1.1/data/hive
# ${system:java.io.tmpdir}/${hive.session.id}_resources替换为本机路径 /usr/local/hive-2.1.1/data/resources
# ${system:java.io.tmpdir}/${system:user.name}/operation_logs 替换为本机路径 /usr/local/hive-2.1.1/data/hive/operation_logs

vi $HIVE_HOME/conf/hive-site.xml

# esc 后输入
:%s#${system:java.io.tmpdir}/${system:user.name}#/usr/local/hive-2.1.1/data/hive#

:%s#${system:java.io.tmpdir}/${hive.session.id}_resources#/usr/local/hive-2.1.1/data/resources#

# esc / 输入 hive.exec.scratchdir 找到后<value> 点击 Insert 键后进行修改

<property>
    <name>hive.exec.scratchdir</name>
    <value>/hive/tmp</value>
</property>

<property>
    <name>hive.metastore.warehouse.dir</name>
    <value>/hive/warehouse</value>
</property>

<!-- 通过jdbc协议连接mysql的hive库 -->
<property>
    <name>javax.jdo.option.ConnectionURL</name>
    <value>jdbc:mysql://node1:3306/hive?createDatabaseIfNotExist=true&amp;useSSL=false</value>
</property>

<!-- jdbc的mysql驱动 -->
<property>
    <name>javax.jdo.option.ConnectionDriverName</name>
    <value>com.mysql.jdbc.Driver</value>
</property>

<!-- mysql用户名 -->
<property>
    <name>javax.jdo.option.ConnectionUserName</name>
    <value>hive</value>
</property>

<!-- mysql用户密码 -->
<property>
    <name>javax.jdo.option.ConnectionPassword</name>
    <value>Hive-123</value>
</property>

<!--hiveserver2的HA-->
<property>
    <name>hive.server2.support.dynamic.service.discovery</name>
    <value>true</value>
</property>

<property>
    <name>hive.zookeeper.quorum</name>
    <value>node1:2181,node2:2181,node3:2181</value>
</property>

<!-- hive的web页面暂时不配置 -->
<property>
    <name>hive.hwi.war.file</name>
    <value>/usr/local/hive-2.1.1/lib/hive-hwi-2.1.1.jar</value>
</property>

# 从hbase/lib下复制必要jar包到hive/lib下
cp $HBASE_HOME/lib/hbase-client-1.2.4.jar $HBASE_HOME/lib/hbase-common-1.2.4.jar $HIVE_HOME/lib

# 同步hive和hadoop的jline版本
cp $HIVE_HOME/lib/jline-2.12.jar $HADOOP_HOME/share/hadoop/yarn/lib

# 查看版本
# cd $HADOOP_HOME/share/hadoop/yarn/lib
# find ./ -name "*jline*jar"

# 删除低版本的jline 0.9
# rm jline-0.9.94.jar

# 复制jdk的tools.jar到hive/lib下
cp $JAVA_HOME/lib/tools.jar $HIVE_HOME/lib

# rm -f $HIVE_HOME/lib/log4j-slf4j-impl-2.4.1.jar

vi $HIVE_HOME/conf/hive-env.sh

HADOOP_HOME=/usr/local/hadoop-2.7.6
export HIVE_HOME=/usr/local/hive-2.1.1
export HIVE_CONF_DIR=/usr/local/hive-2.1.1/conf
export HIVE_AUX_JARS_PATH=/usr/local/hive-2.1.1/lib

# 如果hadoop之前没有配置hadoop.proxyuser.root.groups需配置

# ==================================================================node1 node2 node3
# 如果没有权限，在Hadoop的core-site.xml中增加配置：
vi $HADOOP_HOME/etc/hadoop/core-site.xml

<property>
	<name>hadoop.proxyuser.root.groups</name>
	<value>*</value>
</property>
<property>
	<name>hadoop.proxyuser.root.hosts</name>
	<value>*</value>
</property>

# ==================================================================node1
# 使用超级用户刷新配置
yarn rmadmin -refreshSuperUserGroupsConfiguration
hdfs dfsadmin -refreshSuperUserGroupsConfiguration

# ==================================================================node1 node2
# 如果是对namenode做过HA，则需要在主备namenode上执行
hdfs dfsadmin -fs hdfs://appcluster -refreshSuperUserGroupsConfiguration

# ==================================================================node2 node3

# 使环境变量生效
source /etc/profile

# 查看配置结果
echo $HIVE_HOME

# ==================================================================node1

$HIVE_HOME/bin/schematool -initSchema -dbType mysql

scp -r $HIVE_HOME node2:/usr/local/
scp -r $HIVE_HOME node3:/usr/local/

# 启动

# ==================================================================node1 node2 node3
# 启动 zookeeper
zkServer.sh start
zkServer.sh status

# ==================================================================node1
# 启动hadoop所有进程
$HADOOP_HOME/sbin/start-all.sh

$HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc

# ==================================================================node2
$HADOOP_HOME/sbin/yarn-daemon.sh start resourcemanager
$HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc

# ==================================================================node1
# 要实现Hbase的HA（High Availablity）
$HBASE_HOME/bin/hbase-daemon.sh start master

# 启动Hbase（start-hbase.sh）
$HBASE_HOME/bin/start-hbase.sh

# ==================================================================node2
# 开启Hbase的HA
$HBASE_HOME/bin/hbase-daemon.sh start master

# ==================================================================node1
$HIVE_HOME/bin/hiveserver2

# ==================================================================node2
$HIVE_HOME/bin/hiveserver2

# ==================================================================node1
zkCli.sh

ls /hiveserver2

get /hiveserver2/serverUri=node1:10000;version=2.1.1;sequence=0000000000

$HIVE_HOME/bin/beeline -u "jdbc:hive2://node1:2181,node2:2181,node3:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2" root 123456

# $HIVE_HOME/bin/beeline
# > !connect jdbc:hive2://node1:2181,node2:2181,node3:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2 root "123456"

> create external table user_info(user_id int comment 'userID',user_name string comment 'userName')row format delimited fields terminated by '\t' lines terminated by '\n';

> show tables;

mkdir /root/hive

vi /root/hive/user_info.txt

1001	zhangsan
1002	lisi
1003	wangwu

> load data local inpath '/root/hive/user_info.txt' into table user_info;

> select * from user_info;

> quit;

hdfs dfs -ls /

hdfs dfs -ls /hive/warehouse

hdfs dfs -cat /hive/warehouse/user_info/user_info.txt

hadoop fs -mkdir /hive_input_data

vi /root/hive/user_info.txt

1001	zhangsan
1002	lisi
1003	wangwu
1004	liuliu
1005	qiqi

hadoop fs -put /root/hive/user_info.txt /hive_input_data

hdfs dfs -ls /hive_input_data

# hdfs -dfs  -chmod 777 /hive_input_data

> select * from user_info;

> load data inpath '/hive_input_data/user_info.txt' overwrite into table user_info;

> select * from user_info;

# ==================================================================node1
# stop已经启动的进程
$HBASE_HOME/bin/stop-hbase.sh

$HADOOP_HOME/sbin/stop-all.sh

# ==================================================================node1 node2 node3
# 停止 zookeeper
zkServer.sh stop

# ==================================================================node1
$HADOOP_HOME/sbin/hadoop-daemon.sh stop zkfc

# ==================================================================node2
$HADOOP_HOME/sbin/yarn-daemon.sh stop resourcemanager
$HADOOP_HOME/sbin/hadoop-daemon.sh stop zkfc

shutdown -h now
# 快照 hive_hiveserver2集群