OCP升级(3.6->3.7)

有个好文档还是靠普很多，感谢同事的文档。升级步骤记录如下

1.检查现有环境

[root@master ~]# etcd --version
etcd Version: 3.2.
Git SHA: 1674e68
Go Version: go1.9.2
Go OS/Arch: linux/amd64
[root@master ~]# etcdctl2 cluster-health
member a4e80e96ea75f7c8 is healthy: got healthy result from https://192.168.0.103:2379
cluster is healthy
[root@master ~]# etcdctl2 member list
a4e80e96ea75f7c8: name=master.example.com peerURLs=https://192.168.0.103:2380 clientURLs=https://192.168.0.103:2379 isLeader=true
[root@master ~]# oc get dc/router -n default
NAME      REVISION   DESIRED   CURRENT   TRIGGERED BY
router                                config
[root@master ~]# oc get dc/docker-registry -n default
NAME              REVISION   DESIRED   CURRENT   TRIGGERED BY
docker-registry                               config
[root@master ~]# oc get node --show-labels
NAME                 STATUS     AGE       VERSION             LABELS
master.example.com   Ready      14h       v1.6.1+5115d708d7   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=master.example.com,openshift-infra=apiserver,region=infra,router=true,zone=default
node1.example.com    Ready      14h       v1.6.1+5115d708d7   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=node1.example.com,region=infra,router=true,zone=default
node2.example.com    NotReady   14h       v1.6.1+5115d708d7   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=node2.example.com,region=infra,zone=default

[root@master ~]# oc get pv
No resources found.
[root@master ~]# oc get pvc
No resources found.

2.备份ansible脚本

cd /usr/share/ansible
cp -r plugins/ plugins-3.6.bak
cp -r openshift-ansible/ openshift-ansible-3.6.bak
ll
total
drwxr-xr-x.  root root  Jul  : openshift-ansible
drwxr-xr-x.  root root  Jul  : openshift-ansible-3.6.bak
drwxr-xr-x.  root root  Jul  : plugins
drwxr-xr-x.  root root  Jul  : plugins-3.6.bak
drwxr-xr-x.  root root   Jan    roles

3.备份etcd数据

cp -r /etc/etcd /etc/etcd.bak
etcdctl2 backup --data-dir /var/lib/etcd --backup-dir /tmp/etcd-data
cp /var/lib/etcd/member/snap/db /tmp/etcd-data
ll /etc/etcd.bak
total
drwx------.  root root   Jul  : ca
-rw-------.  root root  Jul  : ca.crt
-rw-r--r--.  root root  Jul  : etcd.conf
-rw-r--r--.  root root  Jul  : etcd.conf.32617.2018--@::~
drwx------.  root root  Jul  : generated_certs
-rw-------.  root root  Jul  : peer.crt
-rw-r--r--.  root root  Jul  : peer.csr
-rw-------.  root root  Jul  : peer.key
-rw-------.  root root  Jul  : server.crt
-rw-r--r--.  root root  Jul  : server.csr
-rw-------.  root root  Jul  : server.key

ll /var/lib/etcd/member/snap/db /tmp/etcd-data
-rw-------.  etcd etcd  Jul  : /var/lib/etcd/member/snap/db

/tmp/etcd-data:
total
-rw-------.  root root  Jul  : db
drwx------.  root root        Jul  : member

4.备份配置文件

master节点

cp -r /etc/origin/master /etc/origin/master.bak
cp -r /etc/origin/node /etc/origin/node.bak
cp -r /etc/sysconfig/atomic-openshift-master-api /etc/sysconfig/atomic-openshift-master-api.bak
cp -r /etc/sysconfig/atomic-openshift-master-controllers  /etc/sysconfig/atomic-openshift-master-controller.bak
cp -r /etc/sysconfig/atomic-openshift-node /etc/sysconfig/atomic-openshift-node.bak

ll /etc/origin/master.bak
ll /etc/origin/node /etc/origin/node.bak
ll /etc/sysconfig/atomic-openshift-master-api.bak
ll /etc/sysconfig/atomic-openshift-master-controller.bak
ll /etc/sysconfig/atomic-openshift-node.bak

node节点

cp -r /etc/origin/node /etc/origin/node.bak
cp -r /etc/sysconfig/atomic-openshift-node /etc/sysconfig/atomic-openshift-node.bak

ll /etc/origin/node.bak
ll /etc/sysconfig/atomic-openshift-node.bak

备份yum repo文件

cp /etc/yum.repos.d/ocp.repo /etc/yum.repos.d/ocp.repo.bak

5.修改yum repository文件

当然先需要把Package导入

[root@master ~]# cat /etc/yum.repos.d/ocp.repo
[local-rhel--server-rpms]
name=rhel--server-rpms
baseurl=http://192.168.56.103:8080/repo/rhel-7-server-rpms
enabled=
gpgcheck=
[local-rhel--server-extras-rpms]
name=rhel--server-extras-rpms
baseurl=http://192.168.56.103:8080/repo/rhel-7-server-extras-rpms
enabled=
gpgcheck=
[local-rhel--fast-datapath-rpms]
name=rhel--fast-datapath-rpms
baseurl=http://192.168.56.103:8080/repo/rhel-7-fast-datapath-rpms
enabled=
gpgcheck=
[local-rhel--server-ose-3.7-rpms]
name=rhel--server-ose-3.7-rpms
baseurl=http://192.168.56.103:8080/repo/rhel-7-server-ose-3.7-rpms
enabled=
gpgcheck=

yum clean all
yum repolist

yum update -y

6.升级ansible

yum update atomic-openshift-utils ansible
rpm -qa | grep atomic-openshift-utils
atomic-openshift-utils-3.7.-.git..cf01e48.el7.noarch

7.备份模板

mv /usr/share/openshift/examples /usr/share/openshift/examples.3.6
mkdir -p /usr/share/openshift/examples
cp -R /usr/share/ansible/openshift-ansible/roles/openshift_examples/files/examples/v3./* /usr/share/openshift/examples/

8.导入镜像

导入镜像过程略，完成后需要对registry-console进行重新tag和push

docker tag registry.example.com/openshift3/registry-console:v3.7.64 registry.example.com/openshift3/registry-console:v3.

docker push registry.example.com/openshift3/registry-console:v3.

10.修改hosts文件

/etc/ansible/hosts文件

[OSEv3:children]
masters
nodes
etcd
nfs

[OSEv3:vars]
ansible_ssh_user=root
openshift_deployment_type=openshift-enterprise
openshift_release=v3.

osm_use_cockpit=true
osm_cockpit_plugins=['cockpit-kubernetes']
openshift_cockpit_deployer_prefix='openshift3/'
openshift_cockpit_deployer_version='v3.7.64'

osm_cluster_network_cidr=10.128.0.0/
openshift_portal_net=172.30.0.0/
openshift_master_api_port=
openshift_master_console_port=

openshift_enable_service_catalog=true
openshift_service_catalog_image_prefix=registry.example.com/openshift3/ose-
openshift_service_catalog_image_version=v3.7.64
ansible_service_broker_image_prefix=registry.example.com/openshift3/ose-
ansible_service_broker_etcd_image_prefix=registry.example.com/rhel7/
template_service_broker_prefix=registry.example.com/openshift3/
oreg_url=registry.example.com/openshift3/ose-${component}:${version}
openshift_examples_modify_imagestreams=true
openshift_clock_enabled=true

openshift_metrics_storage_kind=nfs
openshift_metrics_install_metrics=true
openshift_metrics_storage_access_modes=['ReadWriteOnce']
openshift_metrics_storage_host=nfs.example.com
openshift_metrics_storage_nfs_directory=/exports
openshift_metrics_storage_volume_name=metrics
openshift_metrics_storage_volume_size=10Gi
openshift_metrics_hawkular_hostname=hawkular-metrics.apps.example.com
openshift_metrics_cassandra_storage_type=emptydir
openshift_metrics_image_prefix=registry.example.com/openshift3/
openshift_hosted_metrics_deploy=true
openshift_hosted_metrics_public_url=https://hawkular-metrics.apps.example.com/hawkular/metrics
openshift_metrics_image_prefix=registry.example.com/openshift3/
openshift_metrics_image_version=v3.7.64

openshift_template_service_broker_namespaces=['openshift']
template_service_broker_selector={"node": "true"}
openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider', 'filename': '/etc/origin/master/htpasswd'}]
# Default login account: admin / handhand
openshift_master_htpasswd_users={'admin': '$apr1$gfaL16Jf$c.5LAvg3xNDVQTkk6HpGB1'}

#openshift_repos_enable_testing=true
openshift_disable_check=docker_image_availability,disk_availability,memory_availability,docker_storage
docker_selinux_enabled=false
openshift_docker_options=" --selinux-enabled --insecure-registry 172.30.0.0/16 --log-driver json-file --log-opt max-size=50M --log-opt max-file=3 --insecure-registry registry.example.com --add-registry registry.example.com"
openshift_docker_additional_registries=registry.example.com
openshift_docker_insecure_registries=registry.example.com
osm_etcd_image=rhel7/etcd
openshift_logging_image_prefix=registry.example.com/openshift3/

openshift_hosted_router_selector='region=infra,router=true'
openshift_master_default_subdomain=app.example.com

# host group for masters
[masters]
master.example.com
# host group for etcd
[etcd]
master.example.com

# host group for nodes, includes region info
[nodes]
master.example.com openshift_node_labels="{'region': 'infra', 'router': 'true', 'zone': 'default'}" openshift_schedulable=true
node1.example.com openshift_node_labels="{'region': 'infra', 'router': 'true', 'zone': 'default'}" openshift_schedulable=true
node2.example.com openshift_node_labels="{'region': 'infra', 'zone': 'default'}" openshift_schedulable=true

[nfs]
nfs.example.com

11.升级前的检查

ansible-playbook --tags pre_upgrade /usr/share/ansible/openshift-ansible/playbooks/byo/openshift-cluster/upgrades/v3_7/upgrade.yml 

遇到一个问题说master,node1,node2的节点的Docker的版本需要1.12以上，但master,node1,node2的docker版本已经是1.13.1

解决办法，修改/etc/yum.conf文件，屏蔽下面这句话，跳过检查

#exclude= docker*1.20*  docker*1.19*  docker*1.18*  docker*1.17*  docker*1.16*  docker*1.15*  docker*1.14*  docker*1.13* 

12.升级

升级master节点

ansible-playbook -vv /usr/share/ansible/openshift-ansible/playbooks/byo/openshift-cluster/upgrades/v3_7/upgrade_control_plane.yml | tee /tmp/upgrade_control_plane_to_3_7.log;

升级完后显示

TASK [Warn if shared-resource-viewer could not be updated] **************************************************************************************************************
task path: /usr/share/ansible/openshift-ansible/playbooks/common/openshift-cluster/upgrades/post_control_plane.yml:
skipping: [master.example.com] => {"changed": false, "skip_reason": "Conditional result was False"}
META: ran handlers

PLAY RECAP **************************************************************************************************************************************************************
localhost                  : ok=   changed=    unreachable=    failed=
master.example.com         : ok=  changed=   unreachable=    failed=
nfs.example.com            : ok=   changed=    unreachable=    failed=
node1.example.com          : ok=   changed=    unreachable=    failed=
node2.example.com          : ok=   changed=    unreachable=    failed=   

检查

[root@master ansible]# oc get nodes
NAME                 STATUS    AGE       VERSION
master.example.com   Ready     3h        v1.7.6+a08f5eeb62
node1.example.com    Ready     3h        v1.6.1+5115d708d7
node2.example.com    Ready     3h        v1.6.1+5115d708d7

[root@master ansible]# oc get pods --all-namespaces
NAMESPACE              NAME                         READY     STATUS             RESTARTS   AGE
default                docker-registry--94zld      /       Running                      6m
default                registry-console--lmhgx     /       Running                      6m
default                router--8hnmz               /       Running                      6m
default                router--g6tlm               /       Running                      5m
kube-service-catalog   apiserver-z6nmz              /       Running                      2h
kube-service-catalog   controller-manager-d2jgc     /       CrashLoopBackOff             2h
openshift-infra        hawkular-cassandra--gp4n9   /       Running                      10m
openshift-infra        hawkular-metrics-4j828       /       Running                      2h
openshift-infra        heapster-rgwrw               /       Running                      3h

[root@master ansible]# oc version
oc v3.7.64
kubernetes v1.7.6+a08f5eeb62
features: Basic-Auth GSSAPI Kerberos SPNEGO

Server https://master.example.com:8443
openshift v3.7.64
kubernetes v1.7.6+a08f5eeb62

升级node节点

ansible-playbook -vv /usr/share/ansible/openshift-ansible/playbooks/byo/openshift-cluster/upgrades/v3_7/upgrade_nodes.yml -e openshift_upgrade_nodes_serial= | tee /tmp/upgrade_node_to_3_7.log;

升级完成后提示

TASK [include] **********************************************************************************************************************************************************
task path: /usr/share/ansible/openshift-ansible/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml:
skipping: [node2.example.com] => {"changed": false, "skip_reason": "Conditional result was False"}
META: ran handlers

PLAY RECAP **************************************************************************************************************************************************************
localhost                  : ok=   changed=    unreachable=    failed=
master.example.com         : ok=   changed=    unreachable=    failed=
nfs.example.com            : ok=   changed=    unreachable=    failed=
node1.example.com          : ok=  changed=   unreachable=    failed=
node2.example.com          : ok=  changed=   unreachable=    failed=   

检查

[root@master ansible]# oc get nodes
NAME                 STATUS    AGE       VERSION
master.example.com   Ready     3h        v1.7.6+a08f5eeb62
node1.example.com    Ready     3h        v1.7.6+a08f5eeb62
node2.example.com    Ready     3h        v1.7.6+a08f5eeb62

[root@master ansible]# oc get pods --all-namespaces
NAMESPACE              NAME                         READY     STATUS    RESTARTS   AGE
default                docker-registry--x7sqf      /       Running             4m
default                registry-console--mslwb     /       Running             4m
default                router--79qjr               /       Running             8m
default                router--8hnmz               /       Running             20m
kube-service-catalog   apiserver-z6nmz              /       Running             3h
kube-service-catalog   controller-manager-d2jgc     /       Error              2h
openshift-infra        hawkular-cassandra--qg2bj   /       Running             8m
openshift-infra        hawkular-metrics-bqqm8       /       Running             4m
openshift-infra        heapster-zt87w               /       Running             8m