k8s集群———etcd-三节点部署

etcd集群部署
,创建etcd可执行文件，配置文件，证书文件存放目录
mkdir /opt/etcd/{bin,cfg,ssl} -p

,创建包文件存放目录
mkdir /soft -p

，解压etcd包。并将可执行文件移动到/opt/etcd/bin
tar zxvf etcd-v3.2.12-linux-amd64.tar.gz
mv etcd-v3.2.12-linux-amd64/{etcd,etcdctl} /opt/etcd/bin/

,etcd配置文件
$ cat etcd
#[Member]
ETCD_NAME="etcd01"   #节点名称，如果有多个节点，这里必须要改，etcd02，etcd03
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"    #数据目录
ETCD_LISTEN_PEER_URLS="https://192.168.1.63:2380"   #集群沟通端口2380
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.63:2379"  #客户端沟通端口2379

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.63:2380"  #集群通告地址
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.63:2379"   #客户端通告地址
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.1.63:2380,etcd02=https://192.168.1.65:2380,etcd03=https://192.168.1.66:2380"  #这个集群中所有节点，每个节点都要有
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"   #集群token
ETCD_INITIAL_CLUSTER_STATE="new"   #新创建集群，existing表示加入已有集群
root@k8s-master: /opt/etcd/cfg ::
$ 

，systemd管理etcd
#里面的参数都是需要引用主配置文件的变量，所有如果报错，尝试查看一下主配置文件是否配置出错，/opt/etcd/cfs/etcd
root@k8s-master: /opt/etcd/cfg ::
$ cat /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/opt/etcd/cfg/etcd
ExecStart=/opt/etcd/bin/etcd --name=${ETCD_NAME} --data-dir=${ETCD_DATA_DIR} --listen-peer-urls=${ETCD_LISTEN_PEER_URLS} --listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 --advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} --initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} --initial-cluster=${ETCD_INITIAL_CLUSTER} --initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} --initial-cluster-state=new --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --peer-cert-file=/opt/etcd/ssl/server.pem --peer-key-file=/opt/etcd/ssl/server-key.pem --trusted-ca-file=/opt/etcd/ssl/ca.pem --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem
Restart=on-failure
LimitNOFILE=

[Install]
WantedBy=multi-user.target
root@k8s-master: /opt/etcd/cfg ::
$   

,重新加载配置文件并启动
systemctl daemon-reload
systemctl enable etcd
systemctl restart etcd

,查看启动日志
tail -f /var/log/messages    #会出现与node01和node02无法沟通的状况
#看下边日志，这是因为客户端并没有配置etcd节点文件和ssl，所以会一直报错，systemctl start etcd其实是启动成功，但是沟通不到，所以会启动很长时间
Mar  :: localhost etcd: health check for peer 472edcb0986774fe could not connect: dial tcp 192.168.1.65:: connect: connection refused (prober "ROUND_TRIPPER_RAFT_MESSAGE")
Mar  :: localhost etcd: health check for peer 89e49aedde68fee4 could not connect: dial tcp 192.168.1.66:: connect: connection refused (prober "ROUND_TRIPPER_RAFT_MESSAGE")
Mar  :: localhost etcd: health check for peer 472edcb0986774fe could not connect: dial tcp 192.168.1.65:: connect: connection refused (prober "ROUND_TRIPPER_SNAPSHOT")
Mar  :: localhost etcd: health check for peer 89e49aedde68fee4 could not connect: dial tcp 192.168.1.66:: connect: connection refused (prober "ROUND_TRIPPER_SNAPSHOT")

，node01，node02操作

#将master节点配置文件scp到node01，node02

#将/opt/etcd/下的配置文件文件，文件夹递归传到node01，node02的opt下
scp -r  /opt/etcd/ root@192.168.1.66:/opt
scp -r  /opt/etcd/ root@192.168.1.65:/opt

#将systemctl下的etcd.service传到node01，node02的/usr/lib/systemd/system/
scp /usr/lib/systemd/system/etcd.service root@192.168.1.65:/usr/lib/systemd/system/
scp /usr/lib/systemd/system/etcd.service root@192.168.1.66:/usr/lib/systemd/system/

#这时在tail -f /var/log/messages
ps:
#由于环境是虚拟机环境所以，以下日志是master和node节点时间不同步造成的ntpdate time.windows.com

Mar  :: localhost etcd: the clock difference against peer 472edcb0986774fe is too high [.792944111s > 1s] (prober "ROUND_TRIPPER_RAFT_MESSAGE")
Mar  :: localhost etcd: the clock difference against peer 89e49aedde68fee4 is too high [.861673928s > 1s] (prober "ROUND_TRIPPER_SNAPSHOT")
Mar  :: localhost etcd: the clock difference against peer 89e49aedde68fee4 is too high [.858782669s > 1s] (prober "ROUND_TRIPPER_RAFT_MESSAGE")
Mar  :: localhost etcd: the clock difference against peer 472edcb0986774fe is too high [.793075827s > 1s] (prober "ROUND_TRIPPER_RAFT_MESSAGE")
Mar  :: localhost etcd: the clock difference against peer 472edcb0986774fe is too high [.795990455s > 1s] (prober "ROUND_TRIPPER_SNAPSHOT")
Mar  :: localhost etcd: the clock difference against peer 89e49aedde68fee4 is too high [.858938895s > 1s] (prober "ROUND_TRIPPER_RAFT_MESSAGE")
Mar  :: localhost etcd: the clock difference against peer 89e49aedde68fee4 is too high [.861743791s > 1s] (prober "ROUND_TRIPPER_SNAPSHOT")
Mar  :: localhost etcd: the clock difference against peer 472edcb0986774fe is too high [.796159244s > 1s] (prober "ROUND_TRIPPER_SNAPSHOT")
Mar  :: localhost etcd: the clock difference against peer 472edcb0986774fe is too high [.792476037s > 1s] (prober "ROUND_TRIPPER_RAFT_MESSAGE")

$ crontab -l
 * * * * ntpdate time.windows.com >/dev/null >&

，最后测试一下集群节点状态
（完成）
#如果输出下面信息，就说明集群部署成功。如果有问题第一步先看日志:/var/log/message 或 journalctl -u etcd
root@k8s-master: ~ ::
$ /opt/etcd/bin/etcdctl --ca-file=/opt/etcd/ssl/ca.pem --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --endpoints="https://192.168.1.63:2379,https://192.168.1.65:2379,https://192.168.1.66:2379" cluster-health
member 472edcb0986774fe is healthy: got healthy result from https://192.168.1.65:2379
member 89e49aedde68fee4 is healthy: got healthy result from https://192.168.1.66:2379
member ddaf91a76208ea00 is healthy: got healthy result from https://192.168.1.63:2379
cluster is healthy
root@k8s-master: ~ ::
$