etcd单节点安装

本篇安装单个etcd，然后进行扩容etcd节点至2个，环境配置如果做了的话就跳过

实验架构
test1: 192.168.0.91   etcd
test2: 192.168.0.92    无
test3: 192.168.0.93    无

、环境配置

# 如下操作在所有节点操作

修改主机名

# 注意修改 各自节点对应的 主机名

sed -i '$a\hostname=test1' /etc/hostname
sed -i '$a\hostname=test1' /etc/sysconfig/network && hostnamectl set-hostname test1

配置hosts解析

cat >>/etc/hosts<<EOF
192.168.0.91 test1
192.168.0.92 test2
192.168.0.93 test3
EOF

禁用selinux

sed -i 's/SELINUX=permissive/SELINUX=disabled/' /etc/sysconfig/selinux

sed -i 's/enforcing/disabled/g' /etc/selinux/config

关闭swap

# 注释/etc/fstab文件里swap相关的行

sed -i 's/\/dev\/mapper\/centos-swap/#\/dev\/mapper\/centos-swap/g' /etc/fstab

关掉防火墙

systemctl stop firewalld && systemctl disable firewalld

退出xshell重新登录，查看主机名

开启forward

iptables -P FORWARD ACCEPT

配置转发相关参数

cat >> /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables =
net.bridge.bridge-nf-call-iptables =
vm.swappiness=
EOF

加载系统参数

sysctl --system

加载ipvs相关内核模块

# 如果重新开机，需要重新加载

modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh
modprobe nf_conntrack_ipv4
lsmod | grep ip_vs

安装etcd

下面几步都在test1 节点操作

下载安装包

useradd etcd

mkdir -p /server/software/k8s

mkdir -p /opt/k8s/bin

cd /server/software/k8s

wget https://github.com/coreos/etcd/releases/download/v3.2.18/etcd-v3.2.18-linux-amd64.tar.gz

tar -xf etcd-v3.2.18-linux-amd64.tar.gz

mv etcd-v3.2.18-linux-amd64/etcd* /opt/k8s/bin

chmod +x /opt/k8s/bin/*

ln -s /opt/k8s/bin/etcd /usr/bin/etcd

ln -s /opt/k8s/bin/etcdctl /usr/bin/etcdctl

etcd --version

2、安装CFSSL证书生成工具

只在test1节点操作

mkdir -pv /server/software/k8s
cd /server/software/k8s

wget下载cfssl工具

wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64

安装cfssl工具

# 只要把安装包改下名字，移动到usr/local/bin/下，加上授权即可

mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo
mv cfssl_linux-amd64 /usr/local/bin/cfssl
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
chmod +x /usr/local/bin/cfssl*

3、创建PKI配置文件

# 只 在test1节点操作

# 作用：生成其他组件ca证书时需要用到(除了根证书)CA 配置文件

mkdir -p $HOME/ssl && cd $HOME/ssl

cat >ca-config.json<<EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "87600h"
      }
    }
  }
}
EOF

注意：PKI配置文件中的profiles中同时定义了 server、clietns，表明使用这个PKI创建的证书既可以作为服务器验证用，也可以作为客户端验证用

这里对PKI安全认证不做过多解释，

PKI安全认证请参照：https://www.cnblogs.com/effortsing/p/10332492.html

4、生成 ca 根证书

# 只在test1节点操作

# ca 证书作用：生成其他组件证书时需要用到根证书

cd $HOME/ssl

cat >ca-csr.json<<EOF
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ],
  "ca": {
     "expiry": "87600h"
  }
}
EOF

生成证书

cfssl gencert -initca ca-csr.json | cfssljson -bare ca

查看生成的证书

[root@test1 ssl]# ls

ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem 

5、添加证书到受信任列表（选做）

# 在 test1 节点操作

# 添加ca证书到linux系统受信任列表,这样在执行命令的时候就不用带上证书路径了。

# 添加信任后： etcdctl cluster-health = etcdctl cluster-health /etc/kubernetes/cert/ca.pem ，就是省了个证书
# 如果没有添加ca证书到linux系统受信任列表，后面执行etcdctl cluster-health 会报如下错误。

cat ca.pem >>  /etc/pki/tls/certs/ca-bundle.crt

6、管理证书

# 把根证书和私钥复制到一个目录里面

mkdir -p /etc/kubernetes/cert/

cp ca*.pem /etc/kubernetes/cert/

chmod 777 /etc/kubernetes/*

5、生成etcd的ca证书和私钥

# 只在test1节点上操作

cd $HOME/ssl

cat >etcd-csr.json<<EOF
{
    "CN": "etcd",
    "hosts": [
      "127.0.0.1",
      "192.168.0.92",
      "192.168.0.93",
      "192.168.0.91"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "BeiJing",
            "L": "BeiJing",
            "O": "etcd",
            "OU": "Etcd Security"
        }
    ]
}
EOF

生成证书

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd

查看生成的证书和私钥

[root@test1 ssl]# ls
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem  etcd.csr  etcd-csr.json  etcd-key.pem  etcd.pem

6、添加证书到受信任列表（选做）

# 在 test1 节点操作

# 添加ca证书到linux系统受信任列表

cat etcd.pem >>  /etc/pki/tls/certs/ca-bundle.crt

7、管理证书

把etcd证书复制到一个目录里面

mkdir -p /etc/etcd/cert/

cp etcd*.pem /etc/etcd/cert/

chmod 777 /etc/etcd/cert/*

8、启动etcd

8.1、 配置etcd启动脚本

# 配置 环境变量

cat >> /etc/profile << EOF
export ETCD_NAME=$(hostname)
export INTERNAL_IP=$(hostname -i | awk '{print $NF}')
export ECTD_CLUSTER='test1=https://192.168.0.91:2380'
EOF
source /etc/profile

8.2、配置启动文件

本文配置文件开启了集群外部服务端、客户端、认证，以及集群内部之间服务端、客户端认证。所以客户端etcdctl访问时候需要带上客户端证书

mkdir -p /data/etcd
cat> /etc/systemd/system/etcd.service<< EOF
[Service]
Type=notify
WorkingDirectory=/data/etcd
EnvironmentFile=-/etc/etcd/etcd.conf
ExecStart=/opt/k8s/bin/etcd \\
--name ${ETCD_NAME} \\
--cert-file=/etc/etcd/cert/etcd.pem \\
--key-file=/etc/etcd/cert/etcd-key.pem \\
--peer-cert-file=/etc/etcd/cert/etcd.pem \\
--peer-key-file=/etc/etcd/cert/etcd-key.pem \\
--trusted-ca-file=/etc/kubernetes/cert/ca.pem \\
--peer-trusted-ca-file=/etc/kubernetes/cert/ca.pem \\
--initial-advertise-peer-urls https://${INTERNAL_IP}:2380 \\
--listen-peer-urls https://${INTERNAL_IP}:2380 \\
--listen-client-urls https://${INTERNAL_IP}:2379,http://127.0.0.1:2379 \\
--advertise-client-urls https://${INTERNAL_IP}:2379 \\
--initial-cluster-token my-etcd-token \\
--initial-cluster $ECTD_CLUSTER \\
--initial-cluster-state new \\
--data-dir=/data/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

8.3、启动etctd

systemctl daemon-reload      #一定要执行，否则报错
systemctl start etcd
systemctl status etcd
systemctl enable etcd

9、查看集群成员和安全状态

必须得带上证书，涉及到服务端、客户端认证

[root@test1 ~]# etcdctl --ca-file /etc/kubernetes/cert/ca.pem --cert-file /etc/etcd/cert/etcd.pem --key-file /etc/etcd/cert/etcd-key.pem member list
42f7141ed6110de1: name=test1 peerURLs=https://192.168.0.91:2380 clientURLs=https://192.168.0.91:2379 isLeader=true

[root@test1 ~]# etcdctl --ca-file /etc/kubernetes/cert/ca.pem --cert-file /etc/etcd/cert/etcd.pem --key-file /etc/etcd/cert/etcd-key.pem cluster-health

member 42f7141ed6110de1 is healthy: got healthy result from https://192.168.0.91:2379

cluster is healthy

可以看到peerURLs已经是https模式了，由于test1节点是新建的集群，所以属于重建集群开启pki安全认证；

这里对pki安全认证不多做解释，具体请参照：https://www.cnblogs.com/effortsing/p/10332492.html

报错解决：

删除etcd数据目录重新启动

参照文档：
http://www.maogx.win/posts/35/
http://www.maogx.win/
https://juejin.im/user/59ffa2836fb9a0451c39c64f/posts
https://blog.csdn.net/fy573060627/article/details/52872740