OCP升级(3.7 ->3.9)

坑多，搞了好多次。

1./etc/ansible/hosts

[OSEv3:children]
masters
nodes
etcd
nfs

[OSEv3:vars]
ansible_ssh_user=root
openshift_deployment_type=openshift-enterprise
openshift_release=v3.

osm_use_cockpit=true
osm_cockpit_plugins=['cockpit-kubernetes']
openshift_cockpit_deployer_prefix='openshift3/'
openshift_cockpit_deployer_version='v3.9.43'

osm_cluster_network_cidr=10.128.0.0/
openshift_portal_net=172.30.0.0/
openshift_master_api_port=
openshift_master_console_port=

openshift_hosted_registry_storage_kind=nfs
openshift_hosted_registry_storage_access_modes=['ReadWriteMany']
openshift_hosted_registry_storage_nfs_directory=/exports
openshift_hosted_registry_storage_nfs_options='*(rw,root_squash)'
openshift_hosted_registry_storage_volume_name=registry
openshift_hosted_registry_storage_volume_size=10Gi
oreg_url=registry.example.com/openshift3/ose-\${component}:\${version}
openshift_docker_additional_registries=registry.example.com
openshift_docker_insecure_registries=registry.example.com
openshift_docker_blocked_registries=registry.access.redhat.com,docker.io
openshift_image_tag=v3.9.43

openshift_enable_service_catalog=true
openshift_service_catalog_image_prefix=registry.example.com/openshift3/ose-
openshift_service_catalog_image_version=v3.9.43

ansible_service_broker_image_prefix=registry.example.com/openshift3/ose-
ansible_service_broker_etcd_image_prefix=registry.example.com/rhel7/
ansible_service_broker_selector={"region": "infra"}

openshift_template_service_broker_namespaces=['openshift']
template_service_broker_selector={"region": "infra"}
template_service_broker_prefix=registry.example.com/openshift3/ose-

openshift_hosted_manage_registry=false

oreg_url=registry.example.com/openshift3/ose-${component}:${version}
openshift_examples_modify_imagestreams=true
openshift_clock_enabled=true

openshift_metrics_storage_kind=nfs
openshift_metrics_install_metrics=true
openshift_metrics_storage_access_modes=['ReadWriteOnce']
openshift_metrics_storage_host=nfs.example.com
openshift_metrics_storage_nfs_directory=/exports
openshift_metrics_storage_volume_name=metrics
openshift_metrics_storage_volume_size=10Gi
openshift_metrics_hawkular_hostname=hawkular-metrics.apps.example.com
#openshift_metrics_cassandra_storage_type=emptydir
openshift_metrics_image_prefix=registry.example.com/openshift3/
openshift_hosted_metrics_deploy=true
openshift_hosted_metrics_public_url=https://hawkular-metrics.apps.example.com/hawkular/metrics
openshift_metrics_image_version=v3.9.43

openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider', 'filename': '/etc/origin/master/htpasswd'}]
# Default login account: admin / handhand
openshift_master_htpasswd_users={'admin': '$apr1$gfaL16Jf$c.5LAvg3xNDVQTkk6HpGB1'}

#openshift_repos_enable_testing=true
openshift_disable_check=docker_image_availability,disk_availability,memory_availability,docker_storage

docker_selinux_enabled=false
openshift_docker_options=" --selinux-enabled --insecure-registry 172.30.0.0/16 --log-driver json-file --log-opt max-size=50M --log-opt max-file=3 --insecure-registry registry.example.com --add-registry registry.example.com"
osm_etcd_image=rhel7/etcd
openshift_logging_image_prefix=registry.example.com/openshift3/

openshift_hosted_router_selector='region=infra,router=true'
openshift_master_default_subdomain=app.example.com

openshift_web_console_prefix=registry.example.com/openshift3/ose-
openshift_web_console_version=v3.9.43

# host group for masters
[masters]
master.example.com
# host group for etcd
[etcd]
master.example.com

# host group for nodes, includes region info
[nodes]
master.example.com openshift_node_labels="{'region': 'infra', 'router': 'true', 'zone': 'default'}" openshift_schedulable=true
node1.example.com openshift_node_labels="{'region': 'infra', 'router': 'true', 'zone': 'default'}" openshift_schedulable=true
node2.example.com openshift_node_labels="{'region': 'infra', 'zone': 'default', 'node': 'true'}" openshift_schedulable=true

[nfs]
nfs.example.com

2.有几个镜像需要retag

docker pull registry.example.com/openshift3/registry-console:v3.9.43
docker tag registry.example.com/openshift3/registry-console:v3.9.43 registry.example.com/openshift3/registry-console:v3.
docker push registry.example.com/openshift3/registry-console:v3.

docker pull  registry.example.com/openshift3/ose-deployer:v3.9.43
docker tag registry.example.com/openshift3/ose-deployer:v3.9.43 registry.example.com/openshift3/ose-deployer:v3.9.51
docker push registry.example.com/openshift3/ose-deployer:v3.9.51

docker pull  registry.example.com/openshift3/ose-pod:v3.9.43
docker tag registry.example.com/openshift3/ose-pod:v3.9.43 registry.example.com/openshift3/ose-pod:v3.9.51
docker push registry.example.com/openshift3/ose-pod:v3.9.51

更新主节点

ansible-playbook -vv /usr/share/ansible/openshift-ansible/playbooks/byo/openshift-cluster/upgrades/v3_9/upgrade_control_plane.yml | tee /tmp/upgrade_control_plane_to_3_9.log;

完成后状态

TASK [openshift_master : Wait for master API to come back online] *******************************************************************************************************
task path: /usr/share/ansible/openshift-ansible/roles/openshift_master/tasks/restart.yml:
ok: [master.example.com] => {"changed": false, "elapsed": , "failed": false, "path": null, "port": , "search_regex": null, "state": "started"}

TASK [openshift_master : restart master controllers] ********************************************************************************************************************
task path: /usr/share/ansible/openshift-ansible/roles/openshift_master/tasks/restart.yml:
changed: [master.example.com] => {"attempts": , "changed": true, "cmd": ["systemctl", "restart", "atomic-openshift-master-controllers"], "delta": "0:00:00.738269", "end": "2018-11-24 21:47:24.938854", "failed": false, "rc": , "start": "2018-11-24 21:47:24.200585", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}
META: ran handlers

PLAY RECAP **************************************************************************************************************************************************************
localhost                  : ok=   changed=    unreachable=    failed=
master.example.com         : ok=  changed=  unreachable=    failed=
nfs.example.com            : ok=    changed=    unreachable=    failed=   

[root@master ~]# oc get pods --all-namespaces
NAMESPACE                           NAME                          READY     STATUS             RESTARTS   AGE
default                             docker-registry--8kc4s       /       Running                      16m
default                             docker-registry--qh9vq       /       Running                      16m
default                             docker-registry--xdz55       /       Running                      3h
default                             registry-console--qtj4j      /       Running                      16m
default                             router--ctlwd                /       Running                      7m
default                             router--kvbc6                /       Running                      6m
kube-service-catalog                apiserver-bp4j4               /       Running                      3m
kube-service-catalog                controller-manager-m82nr      /       CrashLoopBackOff             3m
openshift-ansible-service-broker    asb--deploy                  /       Error                        2m
openshift-ansible-service-broker    asb-etcd--deploy             /       Error                        2m
openshift-infra                     hawkular-cassandra--6qmm9    /       Running                      3h
openshift-infra                     hawkular-metrics-fmj5n        /       CrashLoopBackOff            3h
openshift-infra                     heapster-8cb76                /       Error                        16m
openshift-template-service-broker   apiserver-7gnvj               /       Error                        2m
openshift-template-service-broker   apiserver-kqqx7               /       Running                      2m
openshift-template-service-broker   apiserver-smzqn               /       Error                        2m
openshift-web-console               webconsole-55d596f44d-n6gf8   /       Running                      9m

[root@master ~]# oc get node
NAME                 STATUS    ROLES     AGE       VERSION
master.example.com   Ready     master    19h       v1.9.1+a0ce1bc657
node1.example.com    Ready     <none>    19h       v1.7.6+a08f5eeb62
node2.example.com    Ready     <none>    19h       v1.7.6+a08f5eeb62

更新node节点

ansible-playbook -vv /usr/share/ansible/openshift-ansible/playbooks/byo/openshift-cluster/upgrades/v3_9/upgrade_nodes.yml -e openshift_upgrade_nodes_serial= | tee /tmp/upgrade_node_to_3_9.log;

任务结束后输出

TASK [openshift_excluder : Enable openshift excluder] *******************************************************************************************************************
task path: /usr/share/ansible/openshift-ansible/roles/openshift_excluder/tasks/exclude.yml:
changed: [node1.example.com] => {"changed": true, "cmd": ["/sbin/atomic-openshift-excluder", "exclude"], "delta": "0:00:00.049623", "end": "2018-11-25 09:04:05.773310", "failed": false, "rc": , "start": "2018-11-25 09:04:05.723687", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}
changed: [node2.example.com] => {"changed": true, "cmd": ["/sbin/atomic-openshift-excluder", "exclude"], "delta": "0:00:00.051837", "end": "2018-11-25 09:04:05.158001", "failed": false, "rc": , "start": "2018-11-25 09:04:05.106164", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}
META: ran handlers
META: ran handlers

PLAY RECAP **************************************************************************************************************************************************************
localhost                  : ok=   changed=    unreachable=    failed=
master.example.com         : ok=   changed=    unreachable=    failed=
nfs.example.com            : ok=   changed=    unreachable=    failed=
node1.example.com          : ok=  changed=   unreachable=    failed=
node2.example.com          : ok=  changed=   unreachable=    failed=   

[root@master ~]# oc get nodes
NAME                 STATUS    ROLES     AGE       VERSION
master.example.com   Ready     master    12h       v1.9.1+a0ce1bc657
node1.example.com    Ready     <none>    12h       v1.9.1+a0ce1bc657
node2.example.com    Ready     <none>    12h       v1.9.1+a0ce1bc657

heapster,metrics为什么不见了，还需要去查

[root@master ~]# oc get pods --all-namespaces
NAMESPACE                           NAME                          READY     STATUS             RESTARTS   AGE
default                             router--kvbc6                /       Running                      18m
kube-service-catalog                apiserver-bp4j4               /       Running                      15m
kube-service-catalog                controller-manager-m82nr      /       CrashLoopBackOff             15m
openshift-ansible-service-broker    asb--deploy                  /       Error                        14m
openshift-ansible-service-broker    asb-etcd--deploy             /       Error                        14m
openshift-template-service-broker   apiserver-7gnvj               /       Running                      14m
openshift-template-service-broker   apiserver-kqqx7               /       Running                      14m
openshift-template-service-broker   apiserver-smzqn               /       Running                      14m
openshift-web-console               webconsole-55d596f44d-n6gf8   /       Running                      21m

更新脚本不要反复执行，遇到的问题包括

• 导入模板失败

TASK [openshift_examples : Import RHEL streams] *************************************************************************************************************************
task path: /usr/share/ansible/openshift-ansible/roles/openshift_examples/tasks/main.yml:
FAILED - RETRYING: Import RHEL streams ( retries left).
FAILED - RETRYING: Import RHEL streams ( retries left).
FAILED - RETRYING: Import RHEL streams ( retries left).

倒入模板失败，目前暂时没理

• RETRYING: Poll for OpenShift pod deployment success

TASK [openshift_hosted : Poll for OpenShift pod deployment success] *****************************************************************************************************
task path: /usr/share/ansible/openshift-ansible/roles/openshift_hosted/tasks/wait_for_pod.yml:
FAILED - RETRYING: Poll for OpenShift pod deployment success ( retries left).
FAILED - RETRYING: Poll for OpenShift pod deployment success ( retries left).

看了一下是在docker-registry部署完后的检查，修改hosts文件，加入

openshift_hosted_manage_registry=false

• 验证TSB是否运行

TASK [template_service_broker : Verify that TSB is running] ********************************************************************************
FAILED - RETRYING: Verify that TSB is running ( retries left).
FAILED - RETRYING: Verify that TSB is running ( retries left).

解决办法，修改service_broker在infra的节点上运行。（之前是node=true节点上)

template_service_broker_selector={"region": "infra"}

• upgrade storage

脚本不能反复执行