kubeadm 线上集群部署(二) K8S Master集群安装以及工作节点的部署

PS：所有机器主机名请提前改好好

在上一篇,ETCD集群和nginx我们已经搭建成功了,下面我们需要搭建master相关组件,apiverser需要与etcd通信并操作

1.配置证书

将etcd证书上传到master节点,在etcd01上操作

USER=root
export K8SHOST1=172.16.100.31
for HOST in ${K8SHOST1}
do
ssh ${USER}@${HOST} 'mkdir -p /etc/kubernetes/pki/etcd'
scp -r /etc/kubernetes/pki/etcd/ca.crt ${USER}@${HOST}:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/apiserver-etcd-client.crt ${USER}@${HOST}:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/apiserver-etcd-client.key ${USER}@${HOST}:/etc/kubernetes/pki/
done

在k8s1上操作,在操作之前，我们已预先安装过单点k8s集群拿到了我们想要一的初始化环境,所以，下面有2种部署方式

第一种,环境没有得到初始化

初始化之前,登录etcd01上传etcd证书到master01上
scp -r /etc/kubernetes/pki root@172.16.100.31:/etc/kubernetes/
wget https://gitee.com/hewei8520/File/raw/master/1.13.5/start.sh

vim start.sh

#!/usr/bin/env bash
export HOST1=172.16.100.31
export HOST2=172.16.100.32
export HOST3=172.16.100.33
export HOST4=172.16.100.34
export HOST5=172.16.100.35
# master地址池,因为需要在master01上操作,所以IPS里不需要01的地址
IPS='172.16.100.32 172.16.100.33 172.16.100.34 172.16.100.35'
# 远程访问登录帐号
export USER=root
# 初始化LVM挂载卷
curl -s https://gitee.com/hewei8520/File/raw/master/1.13.5/lvm.sh | bash
# 初始化系统
curl -s https://gitee.com/hewei8520/File/raw/master/1.13.5/systemd.sh |bash
mkdir -p /data/kubelet
ln -s /data/kubelet /var/lib/kubelet
systemctl stop docker
mv /var/lib/docker /data/
ln -s /data/docker /var/lib/docker
systemctl restart docker
# 生成kubeadm 配置
# 脚本包含IP地址,如果IP不一致请手动下载好修改掉,和之前的系统环境所属IP和hostname保持一致
wget https://gitee.com/hewei8520/File/raw/master/1.13.5/base-env-config-multi-node.sh
bash base-env-config-multi-node.sh
# 生成证书 为了不让证书提前过期,已重新编译了kubeadm 证书有效期10年
wget https://github.com/qq676596084/QuickDeploy/raw/master/1.13.5/bin/kubeadm && chmod +x kubeadm
./kubeadm init phase certs ca
./kubeadm init phase certs apiserver --config=kubeadm-config-init.yaml
./kubeadm init phase certs apiserver-kubelet-client --config=kubeadm-config-init.yaml
./kubeadm init phase certs front-proxy-ca --config=kubeadm-config-init.yaml
./kubeadm init phase certs front-proxy-client --config=kubeadm-config-init.yaml

# 初始化mstart01
kubeadm init --config kubeadm-config-init.yaml
count=$(netstat -anpt |grep  |grep LISTEN |wc -l)
if [ $count -gt  ];then
  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config
  kubectl apply -f https://gitee.com/hewei8520/File/raw/master/1.13.4/rbac-kdd.yaml
  kubectl apply -f https://gitee.com/hewei8520/File/raw/master/1.13.4/calico.yaml
  # status
  master_status=$(kubectl get nodes | grep "NotReady" | awk {'print $2'})
  while [[ `kubectl get nodes | grep "NotReady" | awk {'print $2'} |sed -n '1p'` == "NotReady" ]]
  do
    echo "master 节点正在初始化,请稍候"
    sleep
  done
else
  echo "master 初始化失败, 请查看日志"
exit
fi

echo "k8s-01 初始化成功"
sed -i 's/var\/lib/data/g' /lib/systemd/system/kubelet.service.d/-kubeadm.conf
sed -i  's/Environment="KUBELET_KUBECONFIG_ARGS=/Environment="KUBELET_KUBECONFIG_ARGS=--root-dir=\/data\/kubelet  /g' /lib/systemd/system/kubelet.service.d/-kubeadm.conf
systemctl enable --now docker
systemctl enable --now kubelet
kubectl get nodes -o wide
# 更新IPS maser节点
kubeadm_join=`kubeadm token create --print-join-command`
for HOST in ${IPS}
do
  ssh ${USER}@${HOST} curl -s https://gitee.com/hewei8520/File/raw/master/1.13.5/lvm.sh |bash
  ssh ${USER}@${HOST} mkdir -p /etc/kubernetes/
  scp -r /etc/kubernetes/pki ${USER}@${HOST}:/etc/kubernetes/
  ssh ${USER}@${HOST} curl -s https://gitee.com/hewei8520/File/raw/master/1.13.5/base-env-config-multi-node.sh |bash
  ssh ${USER}@${HOST} curl -s https://gitee.com/hewei8520/File/raw/master/1.13.5/systemd.sh |bash
  ssh ${USER}@${HOST} ${kubeadm_join} --experimental-control-plane
  ssh ${USER}@${HOST} sed -i 's/var\/lib/data/g' /usr/lib/systemd/system/kubelet.service.d/-kubeadm.conf
  ssh ${USER}@${HOST} sed -i  's/Environment="KUBELET_KUBECONFIG_ARGS=/Environment="KUBELET_KUBECONFIG_ARGS=--root-dir=\/data\/kubelet  /g' /usr/lib/systemd/system/kubelet.service.d/-kubeadm.conf
  ssh ${USER}@${HOST} systemctl enable --now kubelet
done

start.sh

若地址填写无误执行start.shmaster集群即可安装成功

base-env-config-multi-node.sh 用于生成一些系统环境参数 主机映射以及kubeadm所需要的config配置文件

#!/usr/bin/env bash
export MASTER_IPS='H1=k8s-master-01=172.16.100.31 H2=k8s-master-02=172.16.100.32 H3=k8s-master-03=172.16.100.33 H4=k8s-master-04=172.16.100.34 H5=k8s-master-05=172.16.100.35 V1=apiserver01.1ziton.com=172.16.100.254'
export ETCDHOST0=172.16.100.51
export ETCDHOST1=172.16.100.52
export ETCDHOST2=172.16.100.53
# 网卡名称
export ETH_INT=eth0
# k8s化版本号
export K8SVERSION=v1.13.5
export VIP_PORT=
i=
for IP in $MASTER_IPS
do
i=$(($i+))
export $IP
HOST=`echo $IP |awk -F "=" '{print $1}'`
IPS=`eval echo '$'$HOST`
hostname=`echo $IPS |awk -F "=" '{print $1}'`
hostip=`echo $IPS |awk -F "=" '{print $2}'`
if [[ "$i" -ge "" ]];then
  export VIP=$hostip
  export HOSTVIPNAME=$hostname
  sed -i '$a\'$VIP' '$HOSTVIPNAME'' /etc/hosts
  else
  export HOST$i=$hostip
fi
sed -i '$a\'$hostip' '$hostname'' /etc/hosts
done
hostname=`hostname`
hostip=`ip a |grep ${ETH_INT} |awk '{print $2}' |sed -n '2p' |awk -F "/" '{print $1}'`
sed -i '$a\'$hostip' '$hostname'' /etc/hosts
host=`hostname`
if [[ "$host" != "k8s-master-01" ]];then
    exit
    else
        echo `hostname`
fi
cat <<EOF > /root/kubeadm-config-init.yaml
apiVersion: kubeadm.k8s.io/v1beta1
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
---
apiServer:
  certSANs:
  - "k8s-master-01"
  - "k8s-master-02"
  - "k8s-master-03"
  - "k8s-master-04"
  - "k8s-master-05"
  - "k8s-master-06"
  - "k8s-master-07"
  - "k8s-master-08"
  - "k8s-master-09"
  - "${HOSTVIPNAME}"
  - "127.0.0.1"
  - "$HOST1"
  - "$HOST2"
  - "$HOST3"
  - "$HOST4"
  - "$HOST5"
  - "$VIP"
  extraArgs:
    authorization-mode: Node,RBAC
    advertise-address: 0.0.0.0
    service-node-port-range: "10000-40000"
controlPlaneEndpoint: "${HOSTVIPNAME}:${VIP_PORT}"
controllerManager:
  extraArgs:
    address: 0.0.0.0
scheduler:
  extraArgs:
    address: 0.0.0.0
apiVersion: kubeadm.k8s.io/v1beta1
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
dns:
  type: CoreDNS
etcd:
  external:
    endpoints:
    - https://$ETCDHOST0:2379
    - https://$ETCDHOST1:2379
    - https://$ETCDHOST2:2379
    caFile: /etc/kubernetes/pki/etcd/ca.crt
    certFile: /etc/kubernetes/pki/apiserver-etcd-client.crt
    keyFile: /etc/kubernetes/pki/apiserver-etcd-client.key
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: ${K8SVERSION}
networking:
  dnsDomain: cluster.local
  podSubnet: 10.244.0.0/
  serviceSubnet: 10.96.0.0/
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: "ipvs"
EOF

base-env-config-multi-node.sh

第二种安装方式

初始化之前,登录etcd01上传etcd证书到master01上
scp -r /etc/kubernetes/pki root@172.16.100.31:/etc/kubernetes/
wget https://gitee.com/hewei8520/File/raw/master/1.13.5/init_new.sh
和第一种方式一样,确保地址一致,若不一致,请修改后注释掉wget命令在执行脚本
相当于第一种操作,第二种效率更高一些,我们可以提前把所需要用到的系统参数,变量,rpm包,配置等提前做好,在制作成镜像,包括k8s在初始化的时候,然后我们要做的就是只需要将配置分发到各个节点即可

参考资料：

Installing a pod network add-on(网络组件选择)

Setup tools kubeadm(设置工具 kubeadm)