Hadoop记录-安装ambari hdp集群

#!/bin/sh

#配置用户sudo权限（参考/etc/sudoers文件，在/etc/sudoers.d/新建一个用户配置文件，注意要注销）
#需要在/etc/sudoers末尾追加：sfapp ALL=(ALL) NOPASSWD:ALL
#JDK配置,三台机器已配置，略
#修改hostname和加入主机映射（/etc/hosts）
#配置网络/etc/sysconfig/network
#配置无密登录
#ssh免秘钥登录需在sfapp用户下完成；
#在10.202.107.27 cnsz22VLK5444执行
#ssh-keygen #回车
#ssh-copy-id -i /home/sfapp/.ssh/id_rsa.pub sfapp@10.202.107.28 #拷贝公钥
#ssh-copy-id -i /home/sfapp/.ssh/id_rsa.pub sfapp@10.202.107.211 #拷贝公钥
#ssh sfapp@10.202.107.28 #测试登录
#ssh sfapp@10.202.107.211 #测试登录

#ssh-keygen -t rsa
#ssh-copy-id slave1
#ssh-copy-id slave2
#ssh-copy-id master

#在 master机器生成公钥文件
#ssh-keygen -t rsa
#cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
#chmod 600 ~/.ssh/authorized_keys
#拷贝到其他主机上
#scp ~/.ssh/authorized_keys root@slave1:~/.ssh/
#scp ~/.ssh/authorized_keys root@slave2:~/.ssh/
#ssh cnsz22VLK5444 date
#ssh cnsz22VLK5445 date
#ssh cnsz22VLK7714 date

#修改文件打开限制（所有机器）
#vim /etc/security/limits.conf
# End of file
#* soft nofile 65536
#* hard nofile 65536
#* soft nproc 131072
#* hard nproc 131072
#sysctl -p
#关闭防火墙和selinux
#ssh sfapp@10.202.107.27 "sudo systemctl stop firewalld;sudo systemctl disable firewalld"
#ssh sfapp@10.202.107.27 "sudo setenforce 0;sudo sed -ri '/^SELINUX/cSELINUX=disabled' /etc/selinux/config"
#ssh sfapp@10.202.107.28 "sudo systemctl stop firewalld;sudo systemctl disable firewalld"
#ssh sfapp@10.202.107.28 "sudo setenforce 0;sudo sed -ri '/^SELINUX/cSELINUX=disabled' /etc/selinux/config"
#ssh sfapp@10.202.107.211 "sudo systemctl stop firewalld;sudo systemctl disable firewalld"
#ssh sfapp@10.202.107.211 "sudo setenforce 0;sudo sed -ri '/^SELINUX/cSELINUX=disabled' /etc/selinux/config"
#配置ntp时钟(master做时钟服务器，其他slave同步时钟)
#sudo yum -y install ntp
#vim /etc/ntp.conf
#restrict 10.202.0.0 mask 255.255.255.0
#注释以下内容
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst
#server 127.127.1.0
#fudge 127.127.1.0 stratum 10
#systemctl start ntpd.service #启动服务
#systemctl enable ntpd.service #开机自启动
#从节点同步主节点的十种
#crontab -e
#0-59/10 * * * * /usr/sbin/ntpdate 10.202.107.27
#crontab -l
#0-59/10 * * * * /usr/sbin/ntpdate 10.202.107.27
#其他系统设置
#sudo sh -c "echo umask 0022 >> /etc/profile"
#yum -y install httpd
#vim /etc/httpd/conf/httpd.conf 修改端口80为81，修改DocumentRoot 为"/app/ambari"
#sudo systemctl restart httpd.service
#chown -R sfapp. /app/ambari 设置sfapp所有
#浏览器测试登录
#下载文件
#HDP：http://public-repo-1.hortonworks.com/HDP/centos7/2.x/updates/2.6.3.0/HDP-2.6.3.0-centos7-rpm.tar.gz
#ambari：http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/2.6.0.0/ambari-2.6.0.0-centos7.tar.gz
#HDP-UTILS：http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos7/HDP-UTILS-1.1.0.21-centos7.tar.gz
#上传文件ambari-2.6.0.0-centos7.tar.gz HDP-2.6.3.0-centos7-rpm.tar.gz HDP-UTILS-1.1.0.21-centos7.tar.gz
#tar -zxvf ambari-2.6.0.0-centos7.tar.gz
#tar -zxvf HDP-2.6.3.0-centos7-rpm.tar.gz
#mkdir HDP-UTILS
#tar -zxvf HDP-UTILS-1.1.0.21-centos7.tar.gz -C HDP-UTILS
#浏览器测试是否能访问到文件
#制作本地源
#yum install yum-utils createrepo yum-plugin-priorities -y
#createrepo ./
##ambari.repo
#[ambari-2.6.0.0]
#name=ambari Version - ambari-2.6.0.0
#baseurl=http://10.202.107.27:81/ambari/centos7/2.6.0.0-267
#gpgcheck=0
#gpgkey=http://10.202.107.27:81/ambari/centos7/2.6.0.0-267/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
#enabled=1
#priority=1
##hdp.repo
#[HDP-2.6.3.0]
#name=HDP Version - HDP-2.6.3.0
#baseurl=http://10.202.107.27:81/HDP/centos7/2.6.3.0-235
#gpgcheck=1
#gpgkey=http://10.202.107.27:81/HDP/centos7/2.6.3.0-235/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
#enabled=1
#priority=1

#[HDP-UTILS-1.1.0.21]
#name=HDP-UTILS Version - HDP-UTILS-1.1.0.21
#baseurl=http://10.202.107.27:81/HDP-UTILS
#gpgcheck=1
#gpgkey=http://10.202.107.27:81/HDP-UTILS/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
#enabled=1
#priority=1

##cp ambari.repo hdp.repo /etc/yum.repos.d/
##scp ambari.repo hdp.repo slave1:/etc/yum.repos.d/

#yum clean all
#yum makecache
#yum repolist

#安装和配置ambari-server
#yum -y install ambari-server
#ambari-server setup
#n-3-n
#ambari-server start
#浏览器测试：ip:8080 admin/admin

##安装mysql
#yum -y install mariadb mariadb-server
#systemctl start mariadb
#systemctl enable mariadb
#mysql_secure_installation
#Enter current password for root (enter for none):<–初次运行直接回车
#Set root password? [Y/n] <– 是否设置root用户密码，输入y并回车或直接回车
#New password: <– 设置root用户的密码
#Re-enter new password: <– 再输入一次你设置的密码
#Remove anonymous users? [Y/n] <– 是否删除匿名用户，回车
#Disallow root login remotely? [Y/n] <–是否禁止root远程登录,回车,
#Remove test database and access to it? [Y/n] <– 是否删除test数据库，回车
#Reload privilege tables now? [Y/n] <– 是否重新加载权限表，回车
#mysql -uroot -ppassword
#vim /etc/my.cnf
#在[mysqld]标签下添加
init_connect='SET collation_connection = utf8_unicode_ci' init_connect='SET NAMES utf8' character-set-server=utf8
collation-server=utf8_unicode_ci
skip-character-set-client-handshake
vim /etc/my.cnf.d/client.cnf
#在[client]中添加
default-character-set=utf8
vim /etc/my.cnf.d/mysql-clients.cnf
#在[mysql]中添加
default-character-set=utf8
#全部配置完成，重启mariadb
systemctl restart mariadb
#之后进入MariaDB查看字符集
mysql> show variables like "%character%";show variables like "%collation%";
mysql>create user username@localhost identified by 'password';
mysql>grant all on *.* to username@localhost indentified by 'password';
mysql>grant all privileges on *.* to username@'%' identified by 'password';
mysql>grant all privileges on *.* to username@'hostname' identified by 'password' with grant option;
#其中只授予部分权限把 其中 all privileges或者all改为select,insert,update,delete,create,drop,index,alter,grant,references,reload,shutdown,process,file其中一部分。
####创建数据库参考
CREATE DATABASE ambari;
use ambari;
CREATE USER 'ambari'@'%' IDENTIFIED BY 'ambarizk123';
GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'%';
CREATE USER 'ambari'@'localhost' IDENTIFIED BY 'ambarizk123';
GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'localhost';
CREATE USER 'ambari'@'master' IDENTIFIED BY 'ambarizk123';
GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'master';
FLUSH PRIVILEGES;
source /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql
show tables;
use mysql;
select Host User Password from user where user='ambari';
CREATE DATABASE hive;
use hive;
CREATE USER 'hive'@'%' IDENTIFIED BY 'hive';
GRANT ALL PRIVILEGES ON *.* TO 'hive'@'%';
CREATE USER 'hive'@'localhost' IDENTIFIED BY 'hive';
GRANT ALL PRIVILEGES ON *.* TO 'hive'@'localhost';
CREATE USER 'hive'@'master' IDENTIFIED BY 'hive';
GRANT ALL PRIVILEGES ON *.* TO 'hive'@'master';
FLUSH PRIVILEGES;
CREATE DATABASE oozie;
use oozie;
CREATE USER 'oozie'@'%' IDENTIFIED BY 'oozie';
GRANT ALL PRIVILEGES ON *.* TO 'oozie'@'%';
CREATE USER 'oozie'@'localhost' IDENTIFIED BY 'oozie';
GRANT ALL PRIVILEGES ON *.* TO 'oozie'@'localhost';
CREATE USER 'oozie'@'master' IDENTIFIED BY 'oozie';
GRANT ALL PRIVILEGES ON *.* TO 'oozie'@'master';
FLUSH PRIVILEGES;
###配置jdbc驱动
mkdir /usr/share/java
#下载并上传mysql-connector-java-5.1.40.jar重命令为mysql-connector-java.jar
cp /usr/share/java/mysql-connector-java.jar /var/lib/ambari-server/resources/mysql-jdbc-driver.jar
vim /etc/ambari-server/conf/ambari.properties
#添加server.jdbc.driver.path=/usr/share/java/mysql-connector-java.jar
ambari-server setup
#y-输入用户sfapp-3-配置jdk路径-y-3-配置mysql信息（主机、端口、数据库名、用户、密码）-y
ambari-server stop
ambari-server start

###网页引导安装
#Launch Install Wizard-输入集群名字-选择版本、选择操作系统、配置仓库路径
http://10.202.107.27:81/HDP/centos7/2.6.3.0-235
http://10.202.107.27:81/HDP-UTILS
#输入主机名或IP地址，导入主服务器的ssh私钥
#或者不导入私钥选第二个，在从服务器安装yum -y install ambari-agent sudo ambari-agent restart
#如果这里出现问题执行
ambari-server stop
ambari-server reset #重置命令
ambari-server setup #重新设置
ambari-server start
tail -300f /var/log/ambari-server/ambari-server.log
#选择安装的服务、配置参数
#安装完成

array=("sfapp@cnsz22VLK5444" "sfapp@cnsz22VLK5445" "sfapp@cnsz22VLK7714")
for i in "${array[@]}"
do
if [ ! -n $i ]
then
echo "HOSTNAME IS NULL"
else
ssh "$i" "ntpdate -u 6.6.1.52" #ssh登录
echo "$i"
fi
done

beeline -u " jdbc:hive2://10.202.77.201:10000" -n hive -p hive
set hive.execution.engine=tez;

hive -service hivemetastore
hive -service hiveserver2

hive -S -e "select * from xxx"   --S静音模式不打印MR2的进度信息   e加载hql查询语句
hive -f test.hql --加载一个hql文件
source  test.hql

for f in 'rpm -qa | grep xxx';do rpm -e --nodeps  ${f} done;