部署docker镜像仓库及高可用

 

下载地址： https://github.com/goharbor/harbor/releases

 

安装harbor服务器：

安装harbor

root@harbor-vm1:/usr/local/src# ls

harbor-offline-installer-v1.7.5.tgz

root@harbor-vm1:/usr/local/src# tar -xvf harbor-offline-installer-v1.7.5.tgz  -C /usr/local/src/

root@harbor-vm1:/usr/local/src# cd harbor/

root@harbor-vm1:/usr/local/src/harbor# mkdir certs

root@harbor-vm1:/usr/local/src/harbor# vim harbor.cfg

hostname = harbor1.dexter.com

ui_url_protocol = https

ssl_cert = /usr/local/src/harbor/cert/server.crt

ssl_cert_key = /usr/local/src/harbor/cert/server.key

harbor_admin_password = 123456

 

生成证书

root@harbor-vm1:~# mkdir  /usr/local/src/harbor/cert

root@harbor-vm1:~# cd  /usr/local/src/harbor/cert

root@harbor-vm1:/usr/local/src/harbor/cert# openssl genrsa -out server.key 2048  #生成私有key

root@harbor-vm1:/usr/local/src/harbor/cert# openssl req -x509 -new -nodes -key  server.key  -subj "/CN=harbor1.dexter.com" -days 7120 -out server.crt   #创建有效期时间的自签名证书

root@harbor-vm2:/usr/local/src/harbor/cert# openssl req -x509 -new -nodes -key server.key -subj "/CN=harbor2.dexter.com" -days 7120 -out server.crt   #创建有效期时间的自签名证书

注：如果无法在ubuntu系统上生成server.crt，可以尝试在centos上生成后在复制到ubuntu上。

 

安装docker

使用官方安装脚本自动安装 （仅适用于公网环境）

curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun

 

安装docker ce

ubuntu

apt-get install docker-compose -y

 

centos

yum install -y docker-ce

yum install -y python-pip

yum install -y docker-compose

参考文档： https://yq.aliyun.com/articles/110806

 

#配置Harbor

# ./prepare  

在当前目录下启动harbor实例

# ./install.sh

 

配置客户端使用harbor：

mkdir /etc/docker/certs.d/harbor1.dexter.com -pv

注：客户端主要指master和node

 

[root@k8s-harbor1 harbor]# scp cert/server.crt  172.16.99.121:/etc/docker/certs.d/harbor1.dexter.com/

 

#测试登录

[root@k8s-m1 ~]# docker login harbor1.dexter.com

Username: admin

Password:

Login Succeeded

 

 

修改本机C:\Windows\System32\drivers\etc\hosts文件，添加如下两行

172.16.99.127  harbor1.dexter.com

 

尝试使用浏览器打开harbor,账号：admin，密码：123456。

顺便新建一个基础镜像库

 

测试push镜像到harbor：

root@k8s-m1:~# docker pull alpine

root@k8s-m1:~# docker tag alpine:latest harbor1.dexter.com/baseimages/alpine:latest

root@k8s-m1:~# docker push harbor1.dexter.com/baseimages/alpine:latest

 

测试pull镜像

前提是客户端有证书

root@k8s-n1:~# docker pull harbor1.dexter.com/baseimages/alpine:latest

latest: Pulling from baseimages/alpine

c9b1b535fdd9: Pull complete

Digest: sha256:ddba4d27a7ffc3f86dd6c2f92041af252a1f23a8e742c90e6e1297bfa1bc0c45

Status: Downloaded newer image for harbor1.dexter.com/baseimages/alpine:latest

harbor1.dexter.com/baseimages/alpine:latest

 

设置开机启动仓库

echo 'cd /usr/local/src/harbor && ./install.sh' >>/etc/rc.local

chmod +x /etc/rc.local

 

注：如果只使用http，就简单的多了，不需要证书了。

 

扩展1：使用http配置harbor

修改harbor.cfg文件

hostname = 172.16.99.127

ui_url_protocol = http

harbor_admin_password = 123456

 

启动harbor后，可以使用IP访问

 

 

客户端需要修改服务配置文件/lib/systemd/system/docker.service

ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --insecure-registry 172.16.99.127

# systemctl daemon-reload

# systemctl restart docker

 

# ps -ef | grep dockerd

root      3385     1  0 13:15 ?        00:00:00 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --insecure-registry 172.16.99.127

 

# docker login 172.16.99.127

 

扩展2：使用单独的磁盘（/dev/vdb）存储镜像

 

好像在centos7.6上默认创建的就是ftype=1

# mkfs.xfs -n ftype=1 /dev/vdb  

 

挂载磁盘

# mount /dev/vdb /var/lib/docker/

 

开机挂载

# blkid  /dev/vdb

/dev/vdb: UUID="36d3a4e3-8ff7-4eb3-b75f-6a30c0eaf802" TYPE="xfs"

# echo 'UUID=36d3a4e3-8ff7-4eb3-b75f-6a30c0eaf802 /var/lib/docker/ xfs defaults 0 0' >>/etc/fstab

 

 

挂载完磁盘后再次安装docker

# yum install -y docker-ce

 

扩展3：harbor镜像仓库高可用

重新部署了2台harbor镜像仓库

172.16.99.152

172.16.99.153

 

VIP

172.16.99.148

 

harbor.cfg配置如下

# grep -v '^#\|^$' /usr/local/src/harbor/harbor.cfg

hostname = 172.16.99.152

ui_url_protocol = http

harbor_admin_password = 123456

 

# grep -v '^#\|^$' /usr/local/src/harbor/harbor.cfg

hostname = 172.16.99.153

ui_url_protocol = http

harbor_admin_password = 123456

 

 

 

 

172.16.99.153

 

 

客户端

# cat /lib/systemd/system/docker.service

ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --insecure-registry 172.16.99.152 --insecure-registry 172.16.99.153 --insecure-registry 172.16.99.148

 

# docker login 172.16.99.153

Username: admin

Password:

WARNING! Your password will be stored unencrypted in /root/.docker/config.json.

Configure a credential helper to remove this warning. See

https://docs.docker.com/engine/reference/commandline/login/#credentials-store

 

Login Succeeded

# docker tag nginx:v2 172.16.99.153/baseimages/nginx:v2

# docker push 172.16.99.153/baseimages/nginx:v2

 

 

配置高可用

172.16.99.153

 

 

172.16.99.152

 

查看结果

172.16.99.152拿到了172.16.99.153上的镜像

 

 

 

keepalived+harpoxy(VIP)

 

# cat /etc/keepalived/keepalived.conf

global_defs {

    router_id lb-master-172.16.99.126

}

 

vrrp_script check-haproxy {

    script "killall -0 haproxy"

    interval 5

    weight -60

}

 

vrrp_instance VI-kube-master {

    state MASTER

    priority 120

    unicast_src_ip 172.16.99.126

    unicast_peer {

        172.16.99.125

    }

    dont_track_primary

    interface eth0

    virtual_router_id 111

    advert_int 3

    track_script {

        check-haproxy

    }

    virtual_ipaddress {

        172.16.99.148

    }

}

# cat /etc/keepalived/keepalived.conf

global_defs {

    router_id lb-backup-172.16.99.125

}

 

vrrp_script check-haproxy {

    script "killall -0 haproxy"

    interval 5

    weight -60

}

 

vrrp_instance VI-kube-master {

    state BACKUP

    priority 63

    unicast_src_ip 172.16.99.125

    unicast_peer {

        172.16.99.126

    }

    dont_track_primary

    interface eth0

    virtual_router_id 111

    advert_int 3

    track_script {

        check-haproxy

    }

    virtual_ipaddress {

        172.16.99.148

    }

}

 

# cat /etc/haproxy/haproxy.cfg

global

        log /dev/log    local0

        log /dev/log    local1 notice

        chroot /var/lib/haproxy

        stats socket /run/haproxy/admin.sock mode 660 level admin

        stats timeout 30s

        user haproxy

        group haproxy

        daemon

        nbproc 1

 

defaults

        log     global

        timeout connect 5000

        timeout client  10m

        timeout server  10m

 

listen web

        bind 0.0.0.0:80

        mode tcp

        option tcplog

        balance roundrobin

        server 172.16.99.152 172.16.99.152:80  check inter 2000 fall 2 rise 2 weight 1

        server 172.16.99.153 172.16.99.153:80  check inter 2000 fall 2 rise 2 weight 1

 

客户端直接从VIP地址访问和拉取镜像

root@host-172-16-99-151:~# docker login 172.16.99.148

Username: admin

Password:

WARNING! Your password will be stored unencrypted in /root/.docker/config.json.

Configure a credential helper to remove this warning. See

https://docs.docker.com/engine/reference/commandline/login/#credentials-store

 

Login Succeeded

root@host-172-16-99-151:~# docker pull 172.16.99.148/baseimages/nginx:v2

 

在客户端上传镜像并查看结果

# docker tag mysql:5.6 172.16.99.148/baseimages/mysql:5.6

# docker push 172.16.99.148/baseimages/mysql:5.6

 

我们在harbor的web段发现2台harbor机器都有了mysql:5.6这个镜像，这样我们就完成了docker镜像仓库的高可用