ansible 部署hadoop

规划

ansible 节点
ansible   controller
镜像rhel 8.2
镜像ansible

hadoop 集群
master
slave1
slave2
镜像centos 1810

0.初始化

hadoop集群配网络修改主机名

10.104.44.25   master
10.104.44.49   slave1
10.104.44.36   slave2

一.配置ansible，配置集群hadoop用户免密

1.配置ansible controller etc/hosts

[root@localhost ansible]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6

10.104.44.25   master
10.104.44.49   slave1
10.104.44.36   slave2

2.安装ansible并配置hosts

[root@localhost mnt]# ansible --version
ansible 2.9.11
  config file = /etc/ansible/ansible.cfg
  configured module search path = ['/root/.ansible/plugins/modules', '/usr/share/ansible/plugins/modules']
  ansible python module location = /usr/lib/python3.6/site-packages/ansible
  executable location = /usr/bin/ansible
  python version = 3.6.8 (default, Dec  5 2019, 15:45:45) [GCC 8.3.1 20191121 (Red Hat 8.3.1-5)]

[root@localhost ansible]# cat hosts
master
slave1
slave2

3.集群添加hadoop用户

[root@localhost ansible]# cat user_hadoopcreate
- hosts: all
  vars:
          password: '123'
  tasks:
          - name: create
            user:
                    name: hadoop
                    uid: 1200
                    password: "{{ password | password_hash('sha512') }}"
[root@localhost ansible]# 

[root@localhost ansible]# ansible-playbook user_hadoopcreate -k
PLAY RECAP *********************************************************************
master                     : ok=2    changed=1    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0
slave1                     : ok=2    changed=1    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0
slave2                     : ok=2    changed=1    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0

4.配置hadoop用户提权

ansible all -m  copy -a "dest=/etc/sudoers.d/ansible content='hadoop ALL=\(ALL\)  NOPASSWD:ALL'" -k

5.使root用户免密登录集群，使用Hadoop用户

ansible controller创建密匙

[root@localhost ansible]# ssh-keygen

ansible免密操控hadoop用户

[root@localhost .ssh]#     ansible  all  -m  copy -a 'src=/root/.ssh/id_rsa.pub dest=/home/hadoop/.ssh/authorized_keys   owner=hadoop group=hadoop'  -k

6.配置ansible 配置文件

host_key_checking = False  #不检查公钥

允许提权
  become=True
  become_method=sudo
  become_user=root
  become_ask_pass=False
远程用户为hadoop
remote_user = hadoop

sed -i 's/remote_user = root/remote_user = hadoop/' ansible.cfg.bak
sed -i 's/^#become/become/g' ansible.cfg.bak
sed -i 's/^# host_key_checking/host_key_checking/g' ansible.cfg.bak

7.同步hosts

[root@localhost ansible]#     ansible  all  -m  copy -a 'src=/etc/hosts dest=/etc/hosts'

8.集群之间的配置hadoop用户免密登录

执行脚本前需要controller下载expect

yum reinstall --downloadonly --downloaddir=/etc/ansible/ -y libtcl*
yum reinstall --downloadonly --downloaddir=/etc/ansible/ -y expect

编写脚本，生产密钥，并发送公钥给其他集群

[root@localhost ansible]# cat ask.sh 

 expect <<EOF
    set timeout 10
    spawn  ssh-keygen
    expect {
    "save the key"  { send "\n"; exp_continue }
        "passphrase"   { send "\n"; exp_continue }
        "passphrase again" { send "\n"; exp_continue }

    }
    spawn ssh-copy-id hadoop@slave1
    expect {
        "yes/no" { send "yes\n";exp_continue }
        "password" { send "123\n";exp_continue }
    }
    spawn ssh-copy-id hadoop@slave2
    expect {
        "yes/no" { send "yes\n";exp_continue }
        "password" { send "123\n";exp_continue }
    }

    spawn ssh-copy-id hadoop@master
    expect {
        "yes/no" { send "yes\n";exp_continue }
        "password" { send "123\n";exp_continue }
    }
EOF

编写playbook，hadoop节点安装expect软件，并执行免密脚本

[root@localhost ansible]# cat key_operation.yml
- hosts: slave2
  remote_user: hadoop
  become: false
  vars:
         package: /etc/ansible/tcl-8.6.8-2.el8.x86_64.rpm
         package2: /etc/ansible/expect-5.45.4-5.el8.x86_64.rpm
  tasks:
            - name: deliver
              copy:
                        src: "{{ item }}"
                        dest: /home/hadoop
              loop:
                    - "{{ package }}"
                    - "{{ package2 }}"

            - name: rpm
              yum:
                  name:
                        - /home/hadoop/tcl-8.6.8-2.el8.x86_64.rpm
                        - /home/hadoop/expect-5.45.4-5.el8.x86_64.rpm
                  state:  present

            - name: try script
              script: /etc/ansible/ask.sh

9.将以上操作整理成first脚本，可以跑脚本部署以上操作

    [root@localhost first]# cat first.sh
    #!/bin/bash

    expect  << EOF
        set timeout 10
        spawn ansible-playbook user_hadoopcreate.yml -k
        expect {
            "password" { send "123\n";exp_continue }
        }
        spawn ansible all -m file -a "path=/home/hadoop/.ssh state=directory  owner=hadoop group=hadoop" -k
        expect {
            "password" { send "123\n";exp_continue }
        }
        spawn ansible all -m  copy -a "dest=/etc/sudoers.d/ansible content='hadoop ALL=\(ALL\)  NOPASSWD:ALL'" -k

        expect {
            "password" { send "123\n";exp_continue }
        }
        spawn ansible  all  -m  copy -a "src=/root/.ssh/id_rsa.pub dest=/home/hadoop/.ssh/authorized_keys   owner=hadoop group=hadoop"  -k
        expect {
            "password" { send "123\n";exp_continue }
        }
    EOF

    ansible  all  -m  copy -a 'src=/etc/hosts dest=/etc/hosts'  

    # change configuration
    sed -i 's/remote_user = root/remote_user = hadoop/' ansible.cfg
    sed -i 's/^#become/become/g' ansible.cfg
    sed -i 's/^# host_key_checking/host_key_checking/g' ansible.cfg

    yum reinstall --downloadonly --downloaddir=/etc/ansible/ -y libtcl*
    yum reinstall --downloadonly --downloaddir=/etc/ansible/ -y expect

    ansible-playbook key_operation.yml

二.配置hadoop基础配置

上传jdk与hadoop到ansible节点

四个配置文件内容

[root@localhost second]# cat hdfs.config
<configuration>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>file:/opt/hadoop/dfs/name</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>file:/opt/hadoop/dfs/data</value>
    </property>
    <property>
        <name>dfs.replication</name>
        <value>2</value>
    </property>
</configuration>

[root@localhost second]# cat yarn.config
<configuration>
    <property>
        <name>arn.resourcemanager.address</name>
        <value>master:8032</value>
    </property>
    <property>
        <name>yarn.resourcemanager.scheduler.address</name>
        <value>master:8030</value>
    </property>
    <property>
        <name>yarn.resourcemanager.webapp.address</name>
        <value>master:8088</value>
    </property>
    <property>
        <name>yarn.resourcemanager.resource-tracker.address</name>
        <value>master:8031</value>
    </property>
    <property>
        <name>yarn.resourcemanager.admin.address</name>
        <value>master:8033</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
        <value>org.apache.hadoop.mapred.ShuffleHandler</value>
    </property>
</configuration>

[root@localhost second]# cat core.config
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://master:9000</value>
    </property>
    <property>
        <name>io.file.buffer.size</name>
        <value>131072</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>file:/opt/hadoop/tmp</value>
    </property>
</configuration>

[root@localhost second]# cat mapred.config
<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>master:10020</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>master:19888</value>
    </property>
</configuration>

ansible-playbook 配置

cat clucig.yml 

#关闭防火墙与selinux
- hosts: all
  tasks:
          - name: Disable SELinux
            selinux:
                state: disabled

          - name: stop firewalld
            systemd:
                state: stopped
                name: firewalld
                enabled: no
            ignore_errors: yes

#将包解压到目标主机
- hosts: all
  tasks:

          - name: tar
            unarchive:
                        src: "/opt/{{ item }}"
                        dest: /opt/
            loop:
                        - jdk-8u152-linux-x64.tar.gz
                        - hadoop-2.7.1.tar.gz

          - name: mv
            shell: mv /opt/jdk1.8.0_152 /opt/jdk
          - name: mv2
            shell: mv /opt/hadoop-2.7.1 /opt/hadoop

#更改环境变量
- hosts: all
  tasks:
          - name: copy
            copy:
                    dest: /etc/profile.d/hadoop.sh
                    content: "export JAVA_HOME=/opt/jdk

                                export HADOOP_HOME=/opt/hadoop

                                export PATH=${JAVA_HOME}/bin:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin:$PATH\n"

          - name: echo
            shell: echo  'export JAVA_HOME=/opt/jdk' >> /opt/hadoop/etc/hadoop/hadoop-env.sh           

#修改配置文件
#1
          - name: delete something
            shell: sed  -i '/<configuration>/,/<\/configuration>/d' /opt/hadoop/etc/hadoop/hdfs-site.xml

          - name: copy
            copy:
                    src: hdfs.config
                    dest: /opt/

          - name: add configuration
            shell: cat /opt/hdfs.config >> /opt/hadoop/etc/hadoop/hdfs-site.xml

          - name: shell operate
            shell: mkdir -p /opt/hadoop/dfs/{name,data}

#2
          - name: delete something
            shell: sed  -i '/<configuration>/,/<\/configuration>/d' /opt/hadoop/etc/hadoop/core-site.xml

          - name: copy
            copy:
                        src: core.config
                        dest: /opt/

          - name: add configuration
            shell: cat /opt/core.config >> /opt/hadoop/etc/hadoop/core-site.xml

          - name: shell operate2
            shell: mkdir -p /opt/hadoop/tmp

#3
          - name: copy template
            shell: cp /opt/hadoop/etc/hadoop/mapred-site.xml.template /opt/hadoop/etc/hadoop/mapred-site.xml

          - name: delete something
            shell: sed  -i '/<configuration>/,/<\/configuration>/d' /opt/hadoop/etc/hadoop/mapred-site.xml
          - name: copy
            copy:
                        src: mapred.config
                        dest: /opt/
          - name: add configuration
            shell: cat /opt/mapred.config >> /opt/hadoop/etc/hadoop/mapred-site.xml

#4

          - name: delete something
            shell: sed  -i '/<configuration>/,/<\/configuration>/d' /opt/hadoop/etc/hadoop/yarn-site.xml
          - name: copy
            copy:
                        src: yarn.config
                        dest: /opt/
          - name: add configuration
            shell: cat /opt/yarn.config >> /opt/hadoop/etc/hadoop/yarn-site.xml

#configuration finish

          - name: master or slave
            copy:
                        dest: /opt/hadoop/etc/hadoop/masters
                        content: "10.104.44.25\n"
          - name: master or slave
            copy:
                        dest: /opt/hadoop/etc/hadoop/slaves
                        content: "10.104.44.36\n10.104.44.49\n"

          - name: chmod
            shell: chown -R hadoop.hadoop /opt/

剧本可以完成hadoop第四章配置文件

三.启动hadoop

windwos配置域名解析

c:\windows\system32\drivers\etc\hosts

手动启动hadoop

start-all.sh

hdfs namenode -format

start-dfs.sh

上传文件

hdfs dfs -put rhel-8.2-x86_64-dvd.iso