K8S集群集成harbor(1.9.3)服务并配置HTTPS

[root@nginx-keepalived-155-227 ~]# wget https://github.com/goharbor/harbor/releases/download/v1.9.3/harbor-offline-installer-v1.9.3.tgz
[root@nginx-keepalived-155-227 ~]# tar xf harbor-offline-installer-v1.9.3.tgz -C /usr/local/
[root@nginx-keepalived-155-227 ~]# cd /usr/local/harbor/

[root@nginx-keepalived-155-227 harbor]# egrep -v '#|^$' harbor.yml
hostname: mydocker.register.com   #访问的地址，可以使用IP。由于后面使用HTTPS协议，这里直接使用域名避免后面再修改
http:                             #使用的协议
  port: 80                        #使用协议的端口
harbor_admin_password: Harbor12345  #harbor管理员的密码 管理账号为 admin
database:  #数据相关配置
  password: root123
  max_idle_conns: 50
  max_open_conns: 100
data_volume: /data  #harbor仓库镜像存储的路径
clair:  #设置Clair更新的时间间隔（以小时为单位）。设置为0以禁用更新。默认值为12小时。
  updaters_interval: 12
jobservice:
  max_job_workers: 10 #作业服务中复制工作者的最大数量。
notification:
  webhook_job_max_retry: 10  #设置Web挂钩作业的最大重试次数。预设值为10
chart:
  absolute_url: disabled  #设置enabled Chart使用绝对URL。设置disabled Chart以使用相对URL
log:  #配置日志记录
  level: info  #设置日志级别debug，info，warning，error，或fatal。默认值为info。
  local:  #设置日志保留参数
    rotate_count: 50  #切割保留的日志个数
    rotate_size: 200M  #日志大于该值后切割一个新日志文件
    location: /var/log/harbor
_version: 1.9.0
proxy:  #配置Clair，复制作业服务和Harbor使用的代理。如果不需要代理，请保留空白。
  http_proxy:
  https_proxy:
  no_proxy: 127.0.0.1,localhost,.local,.internal,log,db,redis,nginx,core,portal,postgresql,jobservice,registry,registryctl,clair
  components:
    - core
    - jobservice
    - clair

[root@nginx-keepalived-155-227 harbor]# ./install.sh 

[Step 0]: checking installation environment ...

Note: docker version: 19.03.5

Note: docker-compose version: 1.25.0

......

[Step 3]: starting Harbor ...
Creating network "harbor_harbor" with the default driver
Creating harbor-log ... done
Creating registryctl   ... done
Creating registry      ... done
Creating redis         ... done
Creating harbor-portal     ... done
Creating harbor-db     ... done
Creating harbor-core   ... done
Creating harbor-jobservice ... done
Creating nginx             ... done

✔ ----Harbor has been installed and started successfully.----

Now you should be able to visit the admin portal at http://mydocker.register.com.   #提示我们访问的URL地址，注意访问配置好hosts或者DNS服务
For more details, please visit https://github.com/goharbor/harbor .

[root@nginx-keepalived-155-227 harbor]# docker-compose ps
      Name                     Command                  State                 Ports          
---------------------------------------------------------------------------------------------
harbor-core         /harbor/harbor_core              Up (healthy)                            
harbor-db           /docker-entrypoint.sh            Up (healthy)   5432/tcp                 
harbor-jobservice   /harbor/harbor_jobservice  ...   Up (healthy)                            
harbor-log          /bin/sh -c /usr/local/bin/ ...   Up (healthy)   127.0.0.1:1514->10514/tcp
harbor-portal       nginx -g daemon off;             Up (healthy)   8080/tcp                 
nginx               nginx -g daemon off;             Up (healthy)   0.0.0.0:80->8080/tcp     
redis               redis-server /etc/redis.conf     Up (healthy)   6379/tcp                 
registry            /entrypoint.sh /etc/regist ...   Up (healthy)   5000/tcp                 
registryctl         /harbor/start.sh                 Up (healthy)   

[root@k8s-master-155-221 cert]#cat > harbor-csr.json <<EOF
{
  "CN": "harbor",
  "hosts": [
    "127.0.0.1",
    "172.16.155.227",   #227是我们harbor master服务器
    "172.16.155.228",   #228是我们以后准备创建的harbor slave服务器，现在没有，但是提前写入证书避免后面重新生成
    "mydocker.register.com"  #指定harbor的域名
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "4Paradigm"
    }
  ]
}
EOF

[root@k8s-master-155-221 cert]# cfssl gencert -ca=/mnt/k8s/cert/ca.pem -ca-key=/mnt/k8s/cert/ca-key.pem -config=/mnt/k8s/cert/ca-config.json -profile=kubernetes harbor-csr.json  | cfssljson -bare harbor   #签署证书  
[root@k8s-master-155-221 cert]# ls harbor*
harbor.csr harbor-csr.json harbor-key.pem harbor.pem
[root@k8s-master-155-221 cert]# scp harbor*.pem 172.16.155.227:/usr/local/harbor  #拷贝证书到harbor服务器

[root@nginx-keepalived-155-227 harbor]#docker-compose down -v  #关闭harbor服务
[root@nginx-keepalived-155-227 harbor]# mkdir ssl
[root@nginx-keepalived-155-227 harbor]# mv harbor-key.pem harbor.pem ssl/
[root@nginx-keepalived-155-227 harbor]# vim harbor.yml 
......
#http:  #注释HTTP协议
  # port for http, d efault is 80. If https enabled, this port will redirect to https port
#  port: 80 #注释HTTP端口

# https related config
https:  #启用HTTPS协议，注意行首不可以有空格，否则报错
#   # https port for harbor, default is 443
   port: 443  #HTTPS端口
#   # The path of cert and key files for nginx
   certificate: /usr/local/harbor/ssl/harbor.pem  #证书
   private_key: /usr/local/harbor/ssl/harbor-key.pem #私钥
......
[root@nginx-keepalived-155-227 harbor]# ./prepare
[root@nginx-keepalived-155-227 harbor]# docker-compose up -d
[root@nginx-keepalived-155-227 harbor]# docker-compose ps

mkdir /etc/docker/certs.d/mydocker.register.com

scp ca.pem ca-key.pem 172.16.155.xxx:/etc/docker/certs.d/mydocker.register.com/

vim /etc/docker/daemon.json  #除insecure-registries配置外，其他可以不配置
{
    "registry-mirrors": ["https://rljaavx7.mirror.aliyuncs.com","https://docker.mirrors.ustc.edu.cn","https://hub-mirror.c.163.com/"], 
    "insecure-registries": ["mydocker.register.com"],  #设置insecure-registries，注意填写的值
    "max-concurrent-downloads": 20,
    "live-restore": true,
    "max-concurrent-uploads": 10,
    "debug": true,
    "data-root": "/Data/k8s/docker/data", 
    "exec-root": "/Data/k8s/docker/exec",
    "log-opts": {
      "max-size": "100m",
      "max-file": "5"
    }
}

systemctl daemon-reload && systemctl enable docker && systemctl restart docker

vim /etc/hosts
172.16.155.227  mydocker.register.com

[root@k8s-master-155-221 cert]# docker  login  -u  admin  -p  Harbor12345 mydocker.register.com
WARNING! Using --password via the CLI is insecure. Use --password-stdin.
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded

[root@k8s-master-155-221 mydocker.register.com]# docker pull busybox
[root@k8s-master-155-221 mydocker.register.com]# docker images|grep busybox
mydocker.register.com/library/busybox                                         v1                  6d5fcfe5ff17        8 weeks ago         1.22MB
busybox                                                                       latest              6d5fcfe5ff17        8 weeks ago         1.22MB
busybox                                                                       <none>              b534869c81f0        2 months ago        1.22MB
[root@k8s-master-155-221 mydocker.register.com]# docker push mydocker.register.com/library/busybox:v1
The push refers to repository [mydocker.register.com/library/busybox]
195be5f8be1d: Pushed
v1: digest: sha256:edafc0a0fb057813850d1ba44014914ca02d671ae247107ca70c94db686e7de6 size: 527