Harbor 共享后端高可用

1. 主机配置

主机地址	主机配置	主机角色	软件版本
192.168.1.60	CPU：4C MEM：4GB Disk： 100GB	Harbor+Keepalived	Harbor 2.1.3 Keepalived 2.2.1 Docker 19.03.9 VIP：192.168.1.156
192.168.1.61	CPU：4C MEM：4GB Disk： 100GB	Harbor+Keepalived	Harbor 2.1.3 Keepalived 2.2.1 Docker 19.03.9 VIP：192.168.1.156
192.168.1.62	CPU：4C MEM：8GB Disk： 500GB	Postgres+Redis+NFS	Docker 19.03.9

2. 基础安装配置

2.1 Docker 安装教程

2.1.1 安装存储驱动

sudo yum install -y yum-utils device-mapper-persistent-data lvm2

2.1.2 添加 Docker-ce 安装仓库

sudo yum-config-manager --add-repo \
https://download.docker.com/linux/centos/docker-ce.repo

2.1.3 查看与安装所需版本

yum list docker-ce --showduplicates | sort -r
yum -y install docker-ce-19.03.9 docker-ce-cli-19.03.9 containerd.io

2.1.4 配置国内仓库

 {
    "registry-mirrors": ["https://docker.mirrors.ustc.edu.cn","https://hub-mirror.c.163.com"],
    "max-concurrent-downloads": 20,
    "live-restore": true,
    "max-concurrent-uploads": 10,
    "debug": true,
    "data-root": "/data/docker_data",
    "exec-root": "/data/docker_exec",
    "log-opts": {
      "max-size": "100m",
      "max-file": "5"
    }
}

2.1.5 启动 Docker

systemctl start docker && systemctl enable docker

2.1.6 安装 docker-compose

sudo wget https://github.com/docker/compose/releases/download/1.28.4/docker-compose-Linux-x86_64
sudo mv docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose

3. 安装配置数据与共享数据(有存储可跳过)

3.1 NFS Server端配置

# 安装软件
yum -y install nfs-utils rpcbind
# 创建共享目录
mkdir /data/harbor-data
# 配置共享目录
echo "/data/harbor-data *(rw,sync,no_root_squash)"  >> /etc/exports
# 启动服务
systemctl enable rpcbind && systemctl restart rpcbind
systemctl enable nfs && systemctl restart nfs
# 检测工目录
showmount -e localhost

3.2 NFS Client端配置

# 安装软件
yum -y install nfs-utils
# 创建挂载目录
mkdir /data/harbor-data
# 配置自动挂载
vi /etc/fstab # 添加如行
192.168.1.62:/data/harbor-data  /data/harbor-data  nfs defaults 0 0
# 进行挂载
mount -a

3.3 postgres+redis服务

# 创建数据存放目录
docker volume create --driver local \
--opt type=none \
--opt device=/data/harbor-relay/postgres-data \
--opt o=bind postgres-data
docker volume create --driver local \
--opt type=none \
--opt device=/data/harbor-relay/redis-data \
--opt o=bind redis-data

3.4 启动服务

version: '3.1'

services:
  db:
    image: postgres
    container_name: harbor-postgres
    restart: always
    environment:
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: password
    volumes:
      - postgres-data:/var/lib/postgresql/data
    ports:
      - 5432:5432
  redis:
    image: redis
    container_name: harbor-redis
    restart: always
    environment:
      TZ: Asia/Shanghai
      LANG: en_US.UTF-8
    command: redis-server /etc/conf/redis.conf
    privileged: true
    volumes:
      - redis-data:/data
      - ./conf:/etc/conf
    ports:
      - 6379:6379
volumes:
  postgres-data:
    external: true
  redis-data:
    external: true

4. Harbor 服务安装配置

4.1 下载离线安装包

wget https://github.com/goharbor/harbor/releases/download/v2.1.3/harbor-offline-installer-v2.1.3.tgz

4.2 创建外部数据库

# 登陆到容器内进行配置
root@0c68861b7df3:/# psql -U postgres
psql (9.6.20)
Type "help" for help.

postgres=# create user harbor with password 'harbor123';
CREATE ROLE
postgres=# CREATE DATABASE harbor;
CREATE DATABASE
postgres=# create database harbor_clair;
CREATE DATABASE
postgres=# create database harbor_notary_server;
CREATE DATABASE
postgres=# create database harbor_notary_signer;
CREATE DATABASE
postgres=# GRANT ALL PRIVILEGES ON DATABASE harbor to harbor;
GRANT
postgres=# GRANT ALL PRIVILEGES ON DATABASE harbor_clair to harbor;
GRANT
postgres=# GRANT ALL PRIVILEGES ON DATABASE harbor_notary_server to harbor;
GRANT
postgres=# GRANT ALL PRIVILEGES ON DATABASE harbor_notary_signer to harbor;
GRANT
postgres=# \q

4.3 生成自签名证书

#!/bin/bash

# 在该目录下操作生成证书，正好供harbor.yml使用
mkdir -p /data/harbor/cert
cd /data/harbor/cert

DOMAIN_NAME=$1

${DOMAIN_NAME:-magic-harbor.magic.com}

openssl genrsa -out ca.key 4096
openssl req -x509 -new -nodes -sha512 -days 3650 -subj "/C=CN/ST=ShenZhen/L=ShenZhen/O=magic/OU=Harbor/CN=${DOMAIN_NAME}" -key ca.key -out ca.crt
openssl genrsa -out ${DOMAIN_NAME}.key 4096
openssl req -sha512 -new -subj "/C=CN/ST=ShenZhen/L=ShenZhen/O=magic/OU=Harbor/CN=${DOMAIN_NAME}" -key ${DOMAIN_NAME}.key -out ${DOMAIN_NAME}.csr

cat > v3.ext <<-EOF
authorityKeyIdentifier=keyid,issuer
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names

[alt_names]
DNS.1=${DOMAIN_NAME}
EOF

openssl x509 -req -sha512 -days 3650 -extfile v3.ext -CA ca.crt -CAkey ca.key -CAcreateserial -in ${DOMAIN_NAME}.csr -out ${DOMAIN_NAME}.crt

openssl x509 -inform PEM -in ${DOMAIN_NAME}.crt -out ${DOMAIN_NAME}.cert

cp ${DOMAIN_NAME}.crt /etc/pki/ca-trust/source/anchors/${DOMAIN_NAME}.crt

4.4 重新配置 Docker

# 把这三个复制到docke下
mkdir -p /etc/docker/certs.d/magic-harbor.magic.com/
cp magic-harbor.magic.com.cert /etc/docker/certs.d/magic-harbor.magic.com/
cp magic-harbor.magic.com.key /etc/docker/certs.d/ymagic-harbor.magic.com/
cp ca.crt /etc/docker/certs.d/magic-harbor.magic.com/

最终docker目录结构：
/etc/docker/certs.d/
    └── magic-harbor.magic.com
       ├── magic-harbor.magic.com.cert  <-- Server certificate signed by CA
       ├── magic-harbor.magic.com.key   <-- Server key signed by CA
       └── ca.crt               <-- Certificate authority that signed the registry certificate
# 重启docker
systemctl restart docker.service

# 停止
docker-compose down -v

# 重新生成配置文件
./prepare --with-notary --with-clair --with-chartmuseum

# 启动
docker-compose up -d

4.5 更改 harbor.yml

hostname: magic-harbor.magic.com

https:
  port: 443
  certificate: /data/harbor/cert/magic-harbor.magic.com.crt
  private_key: /data/harbor/cert/magic-harbor.magic.com.key
harbor_admin_password: Harbor12345

# The default data volume
data_volume: /data/harbor-data

trivy:
  # ignoreUnfixed The flag to display only fixed vulnerabilities
  ignore_unfixed: false
  skip_update: false
  insecure: false

jobservice:
  # Maximum number of job workers in job service
  max_job_workers: 10

notification:
  # Maximum retry count for webhook job
  webhook_job_max_retry: 10

chart:
  # Change the value of absolute_url to enabled can enable absolute url in chart
  absolute_url: disabled

# Log configurations
log:
  level: info
  local:
    rotate_count: 50
    rotate_size: 200M
    location: /var/log/harbor
_version: 2.0.0
external_database:
  harbor:
    host: 192.168.1.62
    port: 5432
    db_name: harbor
    username: harbor
    password: harbor123
    ssl_mode: disable
    max_idle_conns: 2
    max_open_conns: 0
  clair:
    host: 192.168.1.62
    port: 5432
    db_name: harbor_clair
    username: harbor
    password: harbor123
    ssl_mode: disable
  notary_signer:
    host: 192.168.1.62
    port: 5432
    db_name: harbor_notary_signer
    username: harbor
    password: harbor123
    ssl_mode: disable
  notary_server:
    host: 192.168.1.62
    port: 5432
    db_name: harbor_notary_server
    username: harbor
    password: harbor123
    ssl_mode: disable
external_redis:
  host: 192.168.1.62:6379
  password:
  registry_db_index: 1
  jobservice_db_index: 2
  chartmuseum_db_index: 3
  clair_db_index: 4
proxy:
  http_proxy:
  https_proxy:
  no_proxy:
  components:
    - core
    - jobservice
    - clair
    - trivy

4.6 启动 Harbor

# 开启 chart 仓库服务、开启静态分析容器漏洞服务、内容信任插件
./install.sh --with-chartmuseum --with-trivy --with-clair --with-notary

4.7 验证 Harbor

# docker-compose ps
chartmuseum         ./docker-entrypoint.sh           Up (healthy)
clair               ./docker-entrypoint.sh           Up (healthy)
clair-adapter       /home/clair-adapter/entryp ...   Up (healthy)
harbor-core         /harbor/entrypoint.sh            Up (healthy)
harbor-jobservice   /harbor/entrypoint.sh            Up (healthy)
harbor-log          /bin/sh -c /usr/local/bin/ ...   Up (healthy)   127.0.0.1:1514->10514/tcp
harbor-portal       nginx -g daemon off;             Up (healthy)
nginx               nginx -g daemon off;             Up (healthy)   0.0.0.0:4443->4443/tcp, 0.0.0.0:80->8080/tcp, 0.0.0.0:443->8443/tcp
notary-server       /bin/sh -c migrate-patch - ...   Up
notary-signer       /bin/sh -c migrate-patch - ...   Up
registry            /home/harbor/entrypoint.sh       Up (healthy)
registryctl         /home/harbor/start.sh            Up (healthy)
trivy-adapter       /home/scanner/entrypoint.sh      Up (healthy)

5. Harbor 高可用配置

5.1 下载 Keepalived

wget https://www.keepalived.org/software/keepalived-2.2.1.tar.gz

5.2 编译安装 Keepalived

5.2.1 安装依赖

yum -y install gcc gcc-c++ openssl openssl-devel

5.2.2 安装 keepalived

./configure --sysconf=/etc --prefix=/usr/local/keepalived && \
make && \
make install

5.2.3 准备 keepalived 配置文件 与 check.sh 文件

keepalived.conf

global_defs {
  router_id haborlb
}
vrrp_sync_groups VG1 {
  group {
    VI_1
  }
}
#Please change "ens160" to the interface name on you loadbalancer hosts.
#In some case it will be eth0, ens16xxx etc.
vrrp_instance VI_1 {
  interface eth0

  track_interface {
    eth0
  }

  state MASTER
  virtual_router_id 51
  priority 10

  virtual_ipaddress {
    192.168.1.156/24
  }
  advert_int 1
  authentication {
    auth_type PASS
    auth_pass d0cker
  }

}

##########################HTTPS#################################
#Please uncomment the follow when harbor running under https
virtual_server 192.168.1.156 443 {
  delay_loop 15
  lb_algo rr
  lb_kind DR
  protocol TCP
  nat_mask 255.255.255.0
  persistence_timeout 10

  real_server 192.168.1.60 443 {
    weight 10
    MISC_CHECK {
       misc_path "/usr/local/bin/check.sh 192.168.1.60"
       misc_timeout 5
    }
  }

  real_server 192.168.1.61 443 {
    weight 10
    MISC_CHECK {
       misc_path "/usr/local/bin/check.sh 192.168.1.61"
       misc_timeout 5
    }
  }
}
#########################End of HTTPS Section#################

check.sh

#!/bin/bash

set -e
#get protocol

LOG=/var/log/keepalived_check.log
nodeip=$1
nodeaddress="http://${nodeip}"
http_code=`curl -s -o /dev/null -w "%{http_code}" ${nodeaddress}`

if [ $http_code == 200 ] ; then
  protocol="http"
elif [ $http_code == 308 ]
then
  protocol="https"
else
  echo "`date +"%Y-%m-%d %H:%M:%S"` $1, CHECK_CODE=$http_code" >> $LOG
  exit 1
fi

systeminfo=`curl -k -o - -s ${protocol}://${nodeip}/api/v2.0/systeminfo`

echo $systeminfo | grep "registry_url"
if [ $? != 0 ] ; then
  exit 1
fi

5.3 同步配置文件到另外一台 harbor

scp keepalived.conf check.sh root@192.168.1.61:/etc/keepalived/keepalived.conf
scp /usr/local/bin/check.sh root@192.168.1.61:/usr/local/bin/check.sh

5.4 2 台 Harbor 启动 keepalived

systemctl start keepalived && systemctl enable keepalived

5.5 验证 VIP

# VIP 为 192.168.1.156
ip addr | grep 192.168.1.156
inet 192.168.1.156/24 scope global secondary eth0

6. Harbor 验证

6.1 配置私有仓库

cat /etc/docker/daemon.json
{
    "registry-mirrors": ["https://docker.mirrors.ustc.edu.cn","https://hub-mirror.c.164.com","https://magic-harbor.magic.com"],
    "insecure-registries": ["https://magic-harbor.magic.com"],
    "max-concurrent-downloads": 20,
    "live-restore": true,
    "max-concurrent-uploads": 10,
    "debug": true,
    "data-root": "/data/docker_data",
    "exec-root": "/data/docker_exec",
    "log-opts": {
      "max-size": "100m",
      "max-file": "5"
    }
}

6.2 验证上传下载

# docker tag hello-world magic-harbor.magic.com/library/hello-world:latest
# docker push magic-harbor.magic.com/library/hello-world:latest
    The push refers to repository [magic-harbor.magic.com/library/hello-world]
    9c27e219663c: Preparing
    unauthorized: unauthorized to access repository: library/hello-world, action: push: unauthorized to access repository: library/hello-world, action: push
# docker login  magic-harbor.magic.com
    Username: admin
    Password:
    WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
    Configure a credential helper to remove this warning. See
    https://docs.docker.com/engine/reference/commandline/login/#credentials-store

    Login Succeeded
# docker push magic-harbor.magic.com/library/hello-world:latest
    The push refers to repository [magic-harbor.magic.com/library/hello-world]
    9c27e219663c: Pushed
    latest: digest: sha256:90659bf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cbc042 size: 525