4 cdh 5.12 centos 6.10三节点安装

4 cdh 5.12  centos 6.10 三节点安装

[root@hadoop1 opt]# cat /etc/redhat-release
CentOS release 6.10 (Final)
# service iptables stop
# chkconfig iptables off
# sed -i 's/=enforcing/=disabled/' /etc/selinux/config
# cat >> /etc/profile << EOF
export HISTTIMEFORMAT='%F %T '
EOF
# sed -i '$a vm.swappiness = 0' /etc/sysctl.conf
# sed -i '$a echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag' /etc/rc.local
# sed -i '$a echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled' /etc/rc.local
# vim /etc/ntp.conf ##注释掉所有的 server 开头内容
# Use public servers from the pool.ntp.org project.
# Please consider joining the pool (http://www.pool.ntp.org/join.html).
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst
server cn.pool.ntp.org
# service ntpd start
# chkconfig ntpd on

# rpm -qa | grep java-1
java-1.5.0-gcj-1.5.0.0-29.1.el6.x86_64
java-1.6.0-openjdk-1.6.0.41-1.13.13.1.el6_8.x86_64
java-1.7.0-openjdk-1.7.0.181-2.6.14.10.el6.x86_64
# rpm -e --nodeps ` rpm -qa | grep java-1`
# tar -zxvf jdk-8u191-linux-x64.tar.gz
# ln -s jdk1.8.0_191 jdk
# alternatives --install /usr/bin/java java /opt/jdk/bin/java 100
# alternatives --install /usr/bin/javac javac /opt/jdk/bin/javac 100
# cat >> /etc/profile << EOF
export JAVA_HOME=/opt/jdk
export PATH=$PATH:$JAVA_HOME/bin
EOF
[root@hadoop1 opt]# source /etc/profile
[root@hadoop1 opt]# java -version
java version "1.8.0_191"
Java(TM) SE Runtime Environment (build 1.8.0_191-b12)
Java HotSpot(TM) 64-Bit Server VM (build 25.191-b12, mixed mode)
[root@hadoop1 opt]# javac -version
javac 1.8.0_191
# cat >> /etc/hosts << EOF
* hadoop1
* hadoop2
* hadoop3
EOF
[root@hadoop1 opt]# id hadoop
id: hadoop: No such user
[root@hadoop1 opt]# useradd hadoop
[root@hadoop1 opt]# vim /etc/sudoers
hadoop ALL=NOPASSWD:ALL

chown -R hadoop:hadoop /opt

[root@hadoop1 opt]# ./test_hadoop_env.sh
######### 1. iptables: #####################
 current iptables status: iptables: Firewall is not running.
 chkconfig status： iptables           0:off    1:off    2:off    3:off    4:off    5:off    6:off
######### 2. Selinux: ######################
 current selinux status: Disabled
 config of selinux: SELINUX=disabled
######### 3. THP: ##########################
 defrag status:[always] madvise never
 enabled status:[always] madvise never
######### 4. Swappiness => 0 : ###################
 current swapness setting: 0
######### 5.ntp: ###################
     remote           refid      st t when poll reach   delay   offset  jitter
==============================================================================
*ntp1.flashdance *   2 u  977 1024  375  296.330   15.505  30.880
######### 6. JDK: ###################
 current java version:
javac 1.8.0_191
######### 7. hosts: ###################
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
* hadoop1
* hadoop2
* hadoop3

[root@hadoop1 opt]# chmox +x test_hadoop_env.sh
[root@hadoop1 opt]# pssh -h cluster -i "/opt/test_hadoop_env.sh"
[root@hadoop1 opt]# prsync -h cluster /etc/yum.repos.d/cm512.repo /etc/yum.repos.d/.

[root@hadoop1 opt]# ./mysql_auto_glibc_5724.sh
[mysql@hadoop1 scripts]$ vim cdh521.sql
[mysql@hadoop1 scripts]$ mysql -uroot -p -S  /data/mysqldata/3306/mysql.sock
(root@localhost:mysql.sock) [(none)]> source /data/mysqldata/scripts/cdh521.sql

http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.12.1/RPMS/x86_64/
# cd /var/www/html/
# mkdir cm512
# cd cm512/
[root@hadoop1 cm512]# ll
-rw-r--r-- 1 root root   9676820 Feb 21  2018 cloudera-manager-agent-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm
-rw-r--r-- 1 root root 709818804 May 24 11:49 cloudera-manager-daemons-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm
-rw-r--r-- 1 root root      8692 May 24 11:18 cloudera-manager-server-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm
-rw-r--r-- 1 root root     10600 May 24 11:18 cloudera-manager-server-db-2-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm
-rw-r--r-- 1 root root  31918952 May 24 11:19 enterprise-debuginfo-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm
-rw-r--r-- 1 root root  71204325 May 24 11:20 jdk-6u31-linux-amd64.rpm
-rw-r--r-- 1 root root 142039186 May 24 11:22 oracle-j2sdk1.7-1.7.0+update67-1.x86_64.rpm
# createrepo .
# w3m http://hadoop1/cm512
[root@hadoop1 yum.repos.d]# vim cm512.repo
# yum clean all
# yum repolist
http://archive.cloudera.com/cdh5/parcels/5.12.1/
CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel
CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel.sha1
manifest.json
# mkdir -p /var/www/html/cdh512
[root@hadoop1 cdh512]# ll
-rw-r--r-- 1 root root 1620405492 May 24 11:22 CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel
-rw-r--r-- 1 root root         41 May 24 10:47 CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel.sha1
-rw-r--r-- 1 root root      72312 Feb 23  2018 manifest.json
--通过yum安装ClouderaManager Server
[root@hadoop1 ~]# yum -y install cloudera-manager-server
(1/2): cloudera-manager-daemons-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm                                                                     | 677 MB     00:13
(2/2): cloudera-manager-server-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm                                                                      | 8.5 kB     00:00
---------------------------------------------------------------------------------------------------------------------------------------------------------------
Total                                                                                                                           49 MB/s | 677 MB     00:13
warning: rpmts_HdrFromFdno: Header V4 DSA/SHA1 Signature, key ID e8f86acd: NOKEY

Public key for cloudera-manager-server-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm is not installed

# yum -y install cloudera-manager-server --nogpgcheck
  Installing : cloudera-manager-daemons-5.12.1-1.cm5121.p0.6.el6.x86_64                                                                                    1/2
  Installing : cloudera-manager-server-5.12.1-1.cm5121.p0.6.el6.x86_64                                                                                     2/2
  Verifying  : cloudera-manager-server-5.12.1-1.cm5121.p0.6.el6.x86_64                                                                                     1/2
  Verifying  : cloudera-manager-daemons-5.12.1-1.cm5121.p0.6.el6.x86_64                                                                                    2/2 

Installed:
  cloudera-manager-server.x86_64 0:5.12.1-1.cm5121.p0.6.el6                                                                                                    

Dependency Installed:
  cloudera-manager-daemons.x86_64 0:5.12.1-1.cm5121.p0.6.el6
--初始化CM数据库
[root@hadoop1 ~]# /usr/share/cmf/schema/scm_prepare_database.sh mysql cm cm password
2019-05-24 14:50:31,604 [main] INFO  com.cloudera.enterprise.dbutil.DbCommandExecutor  - Unable to find JDBC driver for database type: MySQL
2019-05-24 14:50:31,606 [main] ERROR com.cloudera.enterprise.dbutil.DbCommandExecutor  - JDBC Driver com.mysql.jdbc.Driver not found.
2019-05-24 14:50:31,606 [main] ERROR com.cloudera.enterprise.dbutil.DbCommandExecutor  - Exiting with exit code 3
--下载mysql驱动
https://dev.mysql.com/downloads/connector/j/
[root@hadoop1 opt]# cp mysql-connector-java-5.1.47-bin.jar /usr/share/cmf/lib/.

[root@hadoop1 opt]# /usr/share/cmf/schema/scm_prepare_database.sh mysql -h * --scm-host * cm cm password
JAVA_HOME=/opt/jdk
Verifying that we can write to /etc/cloudera-scm-server
Creating SCM configuration file in /etc/cloudera-scm-server
Executing:  /opt/jdk/bin/java -cp /usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connector-java.jar:/usr/share/cmf/schema/../lib/* com.cloudera.enterprise.dbutil.DbCommandExecutor /etc/cloudera-scm-server/db.properties com.cloudera.cmf.db.
Fri May 24 15:11:10 CST 2019 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.
2019-05-24 15:11:10,656 [main] INFO  com.cloudera.enterprise.dbutil.DbCommandExecutor  - Successfully connected to database.
2019-05-24 15:11:10,687 [main] ERROR com.cloudera.enterprise.dbutil.DbCommandExecutor  - Unable to create/drop a table.
java.sql.SQLException: Statement violates GTID consistency: CREATE TABLE ... SELECT.
    at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:965)
    at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3978)
    at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3914)
    at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2530)
    at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2683)
    at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2491)
    at com.mysql.jdbc.StatementImpl.executeUpdateInternal(StatementImpl.java:1552)
    at com.mysql.jdbc.StatementImpl.executeLargeUpdate(StatementImpl.java:2607)
    at com.mysql.jdbc.StatementImpl.executeUpdate(StatementImpl.java:1480)
    at com.cloudera.enterprise.dbutil.DbCommandExecutor.runDdlStatementPair(DbCommandExecutor.java:357)
    at com.cloudera.enterprise.dbutil.DbCommandExecutor.testIfUserCanCreateTable(DbCommandExecutor.java:333)
    at com.cloudera.enterprise.dbutil.DbCommandExecutor.testIfUserCanRunDdl(DbCommandExecutor.java:299)
    at com.cloudera.enterprise.dbutil.DbCommandExecutor.testDbConnection(DbCommandExecutor.java:258)
    at com.cloudera.enterprise.dbutil.DbCommandExecutor.main(DbCommandExecutor.java:138)
2019-05-24 15:11:10,696 [main] ERROR com.cloudera.enterprise.dbutil.DbCommandExecutor  - Exiting with exit code 9
--> Error 9, giving up (use --force if you wish to ignore the error)
--注释vim /data/mysqldata/3306/my.cnf文件中的GTID，要关闭gtid模式
[root@hadoop1 opt]# /usr/share/cmf/schema/scm_prepare_database.sh mysql -h * --scm-host * cm cm password
JAVA_HOME=/opt/jdk
Verifying that we can write to /etc/cloudera-scm-server
Creating SCM configuration file in /etc/cloudera-scm-server
Executing:  /opt/jdk/bin/java -cp /usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connector-java.jar:/usr/share/cmf/schema/../lib/* com.cloudera.enterprise.dbutil.DbCommandExecutor /etc/cloudera-scm-server/db.properties com.cloudera.cmf.db.
Fri May 24 15:46:09 CST 2019 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.
2019-05-24 15:46:09,582 [main] INFO  com.cloudera.enterprise.dbutil.DbCommandExecutor  - Successfully connected to database.
All done, your SCM database is configured correctly!

--启动ClouderaManager Server
[root@hadoop1 opt]# service cloudera-scm-server start
Starting cloudera-scm-server:                              [FAILED]

[root@hadoop1 cloudera-scm-server]# cat /var/log/cloudera-scm-server/cloudera-scm-server.out
+======================================================================+
|      Error: JAVA_HOME is not set and Java could not be found
[root@hadoop1 cloudera-scm-server]# echo $JAVA_HOME
/opt/jdk
[root@hadoop1 cloudera-scm-server]# find / -name 'cloudera-scm-server'
[root@hadoop1 cloudera-scm-server]# vim /etc/rc.d/init.d/cloudera-scm-server 

export JAVA_HOME=/opt/jdk

[root@hadoop1 cloudera-scm-server]# service cloudera-scm-server start
Starting cloudera-scm-server:                              [  OK  ]
[root@hadoop1 cloudera-scm-server]# netstat -lnt|grep 7180
tcp        0      0 0.0.0.0:7180                0.0.0.0:*                   LISTEN
[root@hadoop1 cdh512]# mv CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel.sha1 CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel.sha

--访问控制台
http://*:7180/cmf/login