CentOS7下的CDH 6.2.0 安装过程

#install OS centos 7.5
#install lsb package
yum install -y redhat-lsb

#install net-tools package

yum install -y net-tools

#Set the hostname to a unique name (not localhost).

sudo hostnamectl set-hostname cdh-1.test.hugedata.com.cn
sudo hostnamectl set-hostname cdh-2.test.hugedata.com.cn
sudo hostnamectl set-hostname cdh-3.test.hugedata.com.cn
sudo hostnamectl set-hostname steamsets.test.hugedata.com.cn

#Edit /etc/hosts with the IP address and fully qualified domain name (FQDN) of each host in the cluster.
192.168.88.35 cdh-1.test.hugedata.com.cn cdh-1
192.168.88.36 cdh-2.test.hugedata.com.cn cdh-2
192.168.88.37 cdh-3.test.hugedata.com.cn cdh-3
192.168.88.38 steamsets.test.hugedata.com.cn steamsets

#Edit /etc/sysconfig/network with the FQDN of this host only:

HOSTNAME=cdh-1.test.hugedata.com.cn

HOSTNAME=cdh-2.test.hugedata.com.cn

HOSTNAME=cdh-3.test.hugedata.com.cn

HOSTNAME=steamsets.test.hugedata.com.cn

#stop and disable firewalld service
systemctl stop firewalld
systemctl disable firewalld

#disable selinux
sed -i 's/SELINUX=enforcing/SELINUX=disabled/'  /etc/selinux/config
setenforce 0

#set ntp
yum install -y ntp

#Edit the /etc/ntp.conf file  master
server 120.25.108.11 iburst
server 182.92.12.11 iburst

#other server

vim /etc/ntp.conf

server 192.168.88.35
fudge 192.168.88.35 stratum 10

#check
ntpdate -u 120.25.108.11

sudo systemctl start ntpd
sudo systemctl enable ntpd
hwclock --systohc

# all server
systemctl stop chronyd.service
systemctl disable chronyd.service

#Installing Cloudera Manager, CDH, and Managed Services on manager-server
#enable httpd
sudo yum -y install httpd
sudo systemctl start httpd

#Downloading and Publishing the Package Repository
sudo mkdir -p /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cm6/6.2.0/redhat7/ -P /var/www/html/cloudera-repos
sudo chmod -R ugo+rX /var/www/html/cloudera-repos/cm6

# CDH 6
sudo mkdir -p /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/redhat7/ -P /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/gplextras6/6.2.0/redhat7/ -P /var/www/html/cloudera-repos
sudo chmod -R ugo+rX /var/www/html/cloudera-repos/cdh6
sudo chmod -R ugo+rX /var/www/html/cloudera-repos/gplextras6

#install java sdk
#uninstall open jdk1
rpm -qa | grep jdk | xargs rpm -e --nodeps
#install jdk
rpm -ivh /var/www/html/cloudera-repos/cm6/6.2.0/redhat7/yum/RPMS/x86_64/oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm
#update java_home
vim /etc/profile
JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera
CLASSPATH=$JAVA_HOME/lib/
PATH=$PATH:$JAVA_HOME/bin
export PATH JAVA_HOME CLASSPATH

#refresh profile
source /etc/profile
#install postgresql-server
yum install -y postgresql-server.x86_64

#初始化postgresql数据库
sudo service postgresql initdb

#yum 源
#create repo
cd /var/www/html/cloudera-repos/
createrepo .
#所有机器上配置源
vim /etc/yum.repos.d/cloudera-repo.repo

[cloudera-repo]
name=cloudera-repo
baseurl=http://192.168.88.35/cloudera-repos/
enabled=1
gpgcheck=0

#创建缓存
yum makecache

#检查是否成功
yum search cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server

#manager上安装
sudo yum install -y cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server

#auto-TLS
sudo JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera /opt/cloudera/cm-agent/bin/certmanager setup --configure-services

#查看是否创建成功
more /var/log/cloudera-scm-agent/certmanager.log

#Install and Configure Databases
#MariaDB-server
vim /etc/yum.repos.d/MariaDB.repo
[mariadb]
name = MariaDB
baseurl = http://yum.mariadb.org/10.1/centos7-amd64
gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB
gpgcheck=1

#安装新版本mariadb
sudo yum install -y MariaDB-server MariaDB-client

#Configuring and Starting the MariaDB Server
sudo systemctl stop mariadb
mkdir /root/innoDBLogFiles
mv /var/lib/mysql/ib_logfile0 /root/innoDBLogFiles/
mv /var/lib/mysql/ib_logfile1 /root/innoDBLogFiles/

#config mariadb
#backup config
mv /etc/my.cnf /etc/my.cnf.bak
#example
vim /etc/my.cnf
输入下面的配置文件的内容：
###########################配置文件开始################################################
[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
transaction-isolation = READ-COMMITTED
# Disabling symbolic-links is recommended to prevent assorted security risks;
# to do so, uncomment this line:
symbolic-links = 0
# Settings user and group are ignored when systemd is used.
# If you need to run mysqld under a different user or group,
# customize your systemd unit file for mariadb according to the
# instructions in http://fedoraproject.org/wiki/Systemd

key_buffer = 16M
key_buffer_size = 32M
max_allowed_packet = 32M
thread_stack = 256K
thread_cache_size = 64
query_cache_limit = 8M
query_cache_size = 64M
query_cache_type = 1

max_connections = 550
#expire_logs_days = 10
#max_binlog_size = 100M

#log_bin should be on a disk with enough free space.
#Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your
#system and chown the specified folder to the mysql user.
log_bin=/var/lib/mysql/mysql_binary_log

#In later versions of MariaDB, if you enable the binary log and do not set
#a server_id, MariaDB will not start. The server_id must be unique within
#the replicating group.
server_id=1

binlog_format = mixed

read_buffer_size = 2M
read_rnd_buffer_size = 16M
sort_buffer_size = 8M
join_buffer_size = 8M

# InnoDB settings
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit  = 2
innodb_log_buffer_size = 64M
innodb_buffer_pool_size = 4G
innodb_thread_concurrency = 8
innodb_flush_method = O_DIRECT
innodb_log_file_size = 512M

[mysqld_safe]
log-error=/var/log/mariadb/mariadb.log
pid-file=/var/run/mariadb/mariadb.pid

#
# include all files from the config directory
#
!includedir /etc/my.cnf.d

###########################配置文件结束################################################

#开机自启动mariaDB
sudo systemctl enable mariadb

#Start the MariaDB server:
sudo systemctl start mariadb

#secure_installation

sudo /usr/bin/mysql_secure_installation

###################按照示例操作###################################
[...]
Enter current password for root (enter for none):
OK, successfully used password, moving on...
[...]
Set root password? [Y/n] Y
New password:
Re-enter new password:
[...]
Remove anonymous users? [Y/n] Y
[...]
Disallow root login remotely? [Y/n] N
[...]
Remove test database and access to it [Y/n] Y
[...]
Reload privilege tables now? [Y/n] Y
[...]
All done!  If you've completed all of the above steps, your MariaDB
installation should now be secure.

Thanks for using MariaDB!
#################################################################

#Installing the MySQL JDBC Driver for MariaDB
wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz
tar zxvf mysql-connector-java-5.1.46.tar.gz
sudo mkdir -p /usr/share/java/
cd mysql-connector-java-5.1.46
sudo cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar

#Creating Databases for Cloudera Software
mysql -u root -p

CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE hive DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY 'scm';
GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY 'amon';
GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY 'rman';
GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY 'hue';
GRANT ALL ON hive.* TO 'hive'@'%' IDENTIFIED BY 'hive';
GRANT ALL ON sentry.* TO 'sentry'@'%' IDENTIFIED BY 'sentry';
GRANT ALL ON nav.* TO 'nav'@'%' IDENTIFIED BY 'nav';
GRANT ALL ON navms.* TO 'navms'@'%' IDENTIFIED BY 'navms';
GRANT ALL ON oozie.* TO 'oozie'@'%' IDENTIFIED BY 'oozie';

#检查数据库创建
show databases;

#检查用户授权
SHOW GRANTS FOR 'scm'@'%';
SHOW GRANTS FOR 'amon'@'%';
SHOW GRANTS FOR 'rman'@'%';
SHOW GRANTS FOR 'hue'@'%';
SHOW GRANTS FOR 'hive*'@'%';
SHOW GRANTS FOR 'sentry'@'%';
SHOW GRANTS FOR 'nav'@'%';
SHOW GRANTS FOR 'navms'@'%';
SHOW GRANTS FOR 'oozie'@'%';

#Set up the Cloudera Manager Database in shell ( execute this command one by one)

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  scm scm

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  amon amon

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  rman rman

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  hue hue

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  hive hive

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  sentry sentry

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  nav nav

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  navms navms

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  oozie oozie

#Install CDH and Other Software
#Start Cloudera Manager Server:
sudo systemctl start cloudera-scm-server
#Wait several minutes for the Cloudera Manager Server to start. To observe the startup process, run the following on the Cloudera Manager Server host:
sudo tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log
#When you see this log entry, the Cloudera Manager Admin Console is ready:
INFO WebServerImpl:com.cloudera.server.cmf.WebServerImpl: Started Jetty server.

#download allkeys.asc
cd /var/www/html/cloudera-repos/cm6/6.2.0
wget https://archive.cloudera.com/cm6/6.2.0/allkeys.asc

#Open your Browser then go to the address as below:
#http://<server_host>:7180