ETCD集群+ TLS认证

目录

• 1、下载etcd cfssl相关命令
• 2、配置文件 /app/etcd/conf/conf.yml
• 3、用系统命令启动，创建 /usr/lib/systemd/system/etcd.service
• 4、用 supervisor启动
• 5、TLS验证
• 6、用户认证
  • • 重点：新集群中节点启动时，尽量快速同时启动所有节点，单独启动会报错！！

1、下载etcd cfssl相关命令

# 下载etcd   cfssl相关命令
[root@1a32vla0168zzzz cfssl]# ll /app/etcd/bin/
cfssl  cfssl-certinfo  cfssljson  etcd  etcdctl  etcdutl
[root@1a32vla0168zzzz appdeploy]# cd /app/etcd/cfssl/
[root@1a32vla0168zzzz cfssl]# ls
ca.csr  ca-csr.json  ca-key.pem  ca.pem  config.json  server.csr  server.json  server-key.pem  server.pem
[root@1a32vla0168zzzz cfssl]# cat config.json
{
  "signing": {
    "default": {
        "usages": [
          "signing",
          "key encipherment",
          "server auth",
          "client auth"
        ],
        "expiry": "87600h"
    }
  }
}
[root@1a32vla0168zzzz cfssl]# cat ca-csr.json
{
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "O": "etcd",
      "OU": "etcd",
      "L": "apisix",
      "ST": "apisix",
      "C": "china"
    }
  ],
  "CN": "etcd"
}
[root@1a32vla0168zzzz cfssl]# cat server.json
{
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "O": "etcd",
      "OU": "etcd",
      "L": "apisix",
      "ST": "apisix",
      "C": "china"
    }
  ],
  "CN": "etcd",
  "hosts": [
    "xxx.xxx.xxx.5",
    "xxx.xxx.xxx.6",
    "xxx.xxx.xxx.7"
  ]
}
[root@1a32vla0168zzzz cfssl]# cfssl gencert --initca=true ca-csr.json | cfssljson --bare ca
[root@1a32vla0168zzzz cfssl]# cfssl gencert --ca ca.pem --ca-key ca-key.pem --config config.json server.json | cfssljson --bare server

 # 复制 .pem文件至其他节点

2、配置文件 /app/etcd/conf/conf.yml

etcd01（etcd02和etcd03配置替换下ip即可），注意https http的配置细节

# vim /app/etcd/conf/conf.yml
name: etcd01
data-dir: /app/etcd/data
initial-advertise-peer-urls: https://xxx.xxx.xxx.5:2380
listen-peer-urls: https://xxx.xxx.xxx.5:2380
listen-client-urls: https://xxx.xxx.xxx.5:2379,http://127.0.0.1:2379
advertise-client-urls: https://xxx.xxx.xxx.5:2379
initial-cluster-token: apisix-etcd-cluster
initial-cluster: etcd01=https://xxx.xxx.xxx.5:2380,etcd02=https://xxx.xxx.xxx.6:2380,etcd03=https://xxx.xxx.xxx.7:2380
initial-cluster-state: new

# [security]
client-transport-security:
  client-cert-auth: true
  trusted-ca-file: /app/etcd/cfssl/ca.pem
  cert-file: /app/etcd/cfssl/server.pem
  key-file: /app/etcd/cfssl/server-key.pem
peer-transport-security:
  client-cert-auth: true
  trusted-ca-file: /app/etcd/cfssl/ca.pem
  cert-file: /app/etcd/cfssl/server.pem
  key-file: /app/etcd/cfssl/server-key.pem

3、用系统命令启动，创建 /usr/lib/systemd/system/etcd.service

[unix_http_server]
file=//app/etcd/supervisor/upervisor.sock   ; the path to the socket file

[supervisord]
logfile=/app/etcd/supervisor/supervisor.log ; main log file; default $CWD/supervisord.log
logfile_maxbytes=50MB
logfile_backups=5
loglevel=info
pidfile=/app/etcd/supervisor/supervisord.pid ; supervisord pidfile; default supervisord.pid
nodaemon=false
silent=false
minfds=1024
minprocs=200

[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

[supervisorctl]
serverurl=unix:///app/etcd/supervisor/supervisor.sock

[program:apisix-etcd]
directory = /app/etcd/
command=/app/etcd/bin/etcd --config-file=/app/etcd/conf/conf.yml
autostart=true
startsecs=5
startretries=3
autorestart=true            ; 程序崩溃时自动重启，重启次数是有限制的，默认为3次
redirect_stderr=true        ; 重定向输出的日志
stdout_syslog=true
stdout_logfile=/app/etcd/supervisor/stdout.log        ; stdout log path, NONE for none; default AUTO
stdout_logfile_backups=5
stderr_logfile_backups=5
stderr_syslog=true

启动服务

systemctl daemon-reload && systemctl start etcd.service

注意：新集群中节点启动时，尽量快速同时启动所有节点，单独启动会报错

4、用 supervisor启动

[root@1a32vla0168zzzz etcd]# cat supervisor/supervisor.conf
[unix_http_server]
file=//app/etcd/supervisor/upervisor.sock   ; the path to the socket file

[supervisord]
logfile=/app/etcd/supervisor/supervisor.log ; main log file; default $CWD/supervisord.log
logfile_maxbytes=50MB
logfile_backups=5
loglevel=info
pidfile=/app/etcd/supervisor/supervisord.pid ; supervisord pidfile; default supervisord.pid
nodaemon=false
silent=false
minfds=1024
minprocs=200

[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

[supervisorctl]
serverurl=unix:///app/etcd/supervisor/supervisor.sock

[program:apisix-etcd]
directory = /app/etcd/
command=/app/etcd/bin/etcd --config-file=/app/etcd/conf/conf.yml
autostart=true
startsecs=5
startretries=3
autorestart=true            ; 程序崩溃时自动重启，重启次数是有限制的，默认为3次
redirect_stderr=true        ; 重定向输出的日志
stdout_syslog=true
stdout_logfile=/app/etcd/supervisor/stdout.log        ; stdout log path, NONE for none; default AUTO
stdout_logfile_backups=5
stderr_logfile_backups=5
stderr_syslog=true

5、TLS验证

注意：https而不是http

[root@1a32vla0168zzzz etcd]# etcdctl endpoint health --endpoints "https://xxx.xxx.xxx.5:2379,https://xxx.xxx.xxx.6:2379,https://xxx.xxx.xxx.7:2379"
{"level":"warn","ts":1659374088.913921,"logger":"client","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000032380/xxx.xxx.xxx.5:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: authentication handshake failed: x509: certificate signed by unknown authority\""}
{"level":"warn","ts":1659374088.9140902,"logger":"client","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00032e700/xxx.xxx.xxx.6:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: authentication handshake failed: x509: certificate signed by unknown authority\""}
{"level":"warn","ts":1659374088.9141088,"logger":"client","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0000c08c0/xxx.xxx.xxx.7:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: authentication handshake failed: x509: certificate signed by unknown authority\""}
https://xxx.xxx.xxx.5:2379 is unhealthy: failed to commit proposal: context deadline exceeded
https://xxx.xxx.xxx.6:2379 is unhealthy: failed to commit proposal: context deadline exceeded
https://xxx.xxx.xxx.7:2379 is unhealthy: failed to commit proposal: context deadline exceeded
Error: unhealthy cluster
# 不带证书验证会发现提示都是unhealthy，必须带上证书
[root@1a32vla0168zzzz etcd]# etcdctl --cacert /app/etcd/cfssl/ca.pem --cert /app/etcd/cfssl/server.pem --key /app/etcd/cfssl/server-key.pem --endpoints "https://xxx.xxx.xxx.5:2379,https://xxx.xxx.xxx.6:2379,https://xxx.xxx.xxx.7:2379"  endpoint health
https://xxx.xxx.xxx.5:2379 is healthy: successfully committed proposal: took = 9.169585ms
https://xxx.xxx.xxx.6:2379 is healthy: successfully committed proposal: took = 10.263672ms
https://xxx.xxx.xxx.7:2379 is healthy: successfully committed proposal: took = 13.415932ms
[root@1a32vla0168zzzz etcd]# 

6、用户认证

[root@1a32vla0169zzzz etcd]# etcdctl auth status
Authentication Status: false
AuthRevision: 1
[root@1a32vla0169zzzz etcd]# etcdctl user list
[root@1a32vla0169zzzz etcd]# etcdctl role list
[root@1a32vla0169zzzz etcd]# etcdctl role add root
Role root created
[root@1a32vla0169zzzz etcd]# etcdctl user add root:123456
User root created
[root@1a32vla0169zzzz etcd]# etcdctl user grant-role root root
Role root is granted to user root

# 创建读写权限和仅读权限的用户和角色
[root@1a32vla0169zzzz etcd]# etcdctl role add read_write
Role read_write created
[root@1a32vla0169zzzz etcd]# etcdctl role add read_only
Role read_only created
[root@1a32vla0169zzzz etcd]# etcdctl role grant-permission read_write --prefix=true readwrite /
Role read_write updated
[root@1a32vla0169zzzz etcd]# etcdctl role grant-permission read_only --prefix=true read /
Role read_only updated
[root@1a32vla0169zzzz etcd]# etcdctl user add admin:admin123
User admin created
[root@1a32vla0169zzzz etcd]# etcdctl user add monitor:monitor123
User monitor created
[root@1a32vla0169zzzz etcd]# etcdctl user grant-role admin read_write
Role read_write is granted to user admin
[root@1a32vla0169zzzz etcd]# etcdctl user grant-role monitor read_only
Role read_only is granted to user monitor
# 开启用户认证
[root@1a32vla0169zzzz etcd]# etcdctl auth enable
Authentication Enabled

此时，验证需要带上用户密码

# 不带用户密码会报错
# etcdctl --cacert /app/etcd/cfssl/ca.pem --cert /app/etcd/cfssl/server.pem --key /app/etcd/cfssl/server-key.pem --endpoints "https://xxx.xxx.xxx.5:2379,https://xxx.xxx.xxx.6:2379,https://xxx.xxx.xxx.7:2379"  endpoint health
{"level":"warn","ts":1659374402.6467996,"logger":"client","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00040ca80/xxx.xxx.xxx.5:2379","attempt":0,"error":"rpc error: code = PermissionDenied desc = etcdserver: permission denied"}
{"level":"warn","ts":1659374402.6468987,"logger":"client","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0000b28c0/xxx.xxx.xxx.7:2379","attempt":0,"error":"rpc error: code = PermissionDenied desc = etcdserver: permission denied"}
{"level":"warn","ts":1659374402.6474524,"logger":"client","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0000321c0/xxx.xxx.xxx.6:2379","attempt":0,"error":"rpc error: code = PermissionDenied desc = etcdserver: permission denied"}
https://xxx.xxx.xxx.7:2379 is healthy: successfully committed proposal: took = 15.426246ms
https://xxx.xxx.xxx.5:2379 is healthy: successfully committed proposal: took = 15.216552ms
https://xxx.xxx.xxx.6:2379 is healthy: successfully committed proposal: took = 15.952318ms

# etcdctl --cacert /app/etcd/cfssl/ca.pem --cert /app/etcd/cfssl/server.pem --key /app/etcd/cfssl/server-key.pem --endpoints "https://xxx.xxx.xxx.5:2379,https://xxx.xxx.xxx.6:2379,https://xxx.xxx.xxx.7:2379"  endpoint health --user=root:123456
https://xxx.xxx.xxx.6:2379 is healthy: successfully committed proposal: took = 933.418µs
https://xxx.xxx.xxx.5:2379 is healthy: successfully committed proposal: took = 915.707µs
https://xxx.xxx.xxx.7:2379 is healthy: successfully committed proposal: took = 1.209063ms

重点：新集群中节点启动时，尽量快速同时启动所有节点，单独启动会报错！！