K8S - Jenkins在K8S下的持续集成

准备nfs网络存储

提前安装好nfs服务

[root@master ~]# yum -y install nfs-utils rpcbind
[root@master ~]# systemctl start rpcbind && systemctl enable rpcbind && systemctl status rpcbind
[root@master lv]# mkdir -p /nfsdata/data/jenkins
[root@master ~]# vim /etc/exports/data/jenkins
/nfsdata/data/jenkins *(rw,sync,no_root_squash)
[root@master ~]# systemctl start nfs-server && systemctl enable nfs-server && systemctl status nfs-server
[root@master ~]# showmount -e
Export list for master:
/nfsdata/data/jenkins *

安装Jenkins服务到K8S集群

使用Dockerfile制作Jenkins镜像

下载war包放到Dockerfile文件同一个目录下，Dockerfile如下，war包下载地址：https://mirrors.tuna.tsinghua.edu.cn/jenkins/

yaml文件资源下载地址：

https://gitee.com/fdd-39969/cicd.git

git clone https://gitee.com/fdd-39969/cicd.git
cd cicd/jenkins

ls cicd/jenkins/
Dockerfile jenkins-pv.yaml jenkins-svc.yaml rbac.yaml
jenkins-deploy.yaml jenkins-slave jenkins.war

创建Dockerfile文件

FROM java:8
RUN echo 'hello docker, start build image'

RUN mkdir -p /app
WORKDIR /app

RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
RUN echo "Asia/Shanghai" > /etc/timezone

COPY jenkins.war .

CMD ["java" ,"-Xms1024m","-Xmx1024m", "-jar","/app/jenkins.war"]

制作镜像

docker build -t 192.168.166.132/library/jenkins-2.273 .
docker push 192.168.166.132/library/jenkins-2.273

K8S安装Jenkins应用

在k8s集群内创建Jenkins工作的namespace，我这边统一放在devops这个ns底下；

# 创建命名空间
kubectl create ns devops

我这里把Jenkins工作目录单独挂载到PVC，需要先创建pv-pvc,挂载点是使用的nfs服务，请先创建好服务，jenkins-pv-pvc.yaml如下：

kind: PersistentVolume
apiVersion: v1
metadata:
  name: jenkins-home-pv
  namespace: devops
spec:
  accessModes:
    - ReadWriteOnce        #访问模式定义为只能以读写的方式挂载到单个节点
  capacity:
    storage: 10Gi
  persistentVolumeReclaimPolicy: Retain
  storageClassName: nfs
  nfs:
    path: /nfsdata/data/jenkins
    server: 192.168.166.128

---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: jenkins-home-pvc
  namespace: devops
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
  storageClassName: nfs      #这里指定关联的PV名称

为Jenkins创建单独的ServiceAccount,这里的ClusterRole直接使用的cluster-admin,jenkins-serveraccount.yaml如下；

apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    app: jenkins
  name: jenkins-admin
  namespace: devops
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: jenkins-admin
  labels:
    app: jenkins
subjects:
  - kind: ServiceAccount
    name: jenkins-admin
    namespace: devops
roleRef:
  kind: ClusterRole
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io

jenkins-deployment.yaml如下：

注：如果需要指定在某个节点上运行，需要自行打污点设置 （格式： kubectl label nodes k8s-node1 app.k8s.icj1/devops=）

apiVersion: apps/v1
kind: Deployment
metadata:
  name: jenkins
  namespace: devops
  labels:
    app: jenkins
spec:
  replicas: 1
  selector:
    matchLabels:
      app: jenkins
  template:
    metadata:
      labels:
        app: jenkins
    spec:
      serviceAccountName: jenkins-admin
      imagePullSecrets:
        - name: ram-secret
      #affinity:
        #nodeAffinity:
        #  requiredDuringSchedulingIgnoredDuringExecution:
            #nodeSelectorTerms:
            #- matchExpressions:
              #- key: apps.k8s.icjl/devops
       #         operator: Exists
      containers:
      - name: jenkins
        image: 192.168.166.132/library/jenkins-2.273
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - name: jenkins-home
          mountPath: /root/.jenkins
          readOnly: false
        ports:
        - containerPort: 8080
        - containerPort: 50000
      volumes:
      - name: jenkins-home
        persistentVolumeClaim:
          claimName: jenkins-home-pvc     # 关联PVC卷

创建service，这边使用了NodePort,jenkins-service.yaml如下；

apiVersion: v1
kind: Service
metadata:
  labels:
    app: jenkins
  name: jenkins
  namespace: devops
  annotations:
    prometheus.io/scrape: 'true'
spec:
  type: NodePort
  ports:
  - name: jenkins-web
    port: 8080
    targetPort: 8080
    nodePort: 30008          # jenkins 地址端口
  - name: jenkins-agent
    port: 50000
    targetPort: 50000
    nodePort: 30005          # jenkins 通道端口
  selector:
    app: jenkins

也可以使用ingress暴露的方式,jenkins-ingress.yaml如下：（我这里没有使用ingress）

apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: jenkins
  labels:
    name: jenkins
  namespace: devops
spec:
  rules:
  - host: jenkins.hiningmeng.cn
    http:
      paths:
      - path: /
        backend:
          serviceName: jenkins
          servicePort: 8080

执行yaml

kubectl apply -f jenkins-pv-pvc.yaml
kubectl apply -f jenkins-serveraccount.yaml
kubectl apply -f jenkins-deployment.yaml
kubectl apply -f jenkins-service.yaml
kubectl apply -f jenkins-ingress.yaml

查看pod状态

[root@k8s-master jenkins]# kubectl get pv,pvc,svc -n devops
NAME                               CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                     STORAGECLASS   REASON   AGE
persistentvolume/jenkins-home-pv   10Gi       RWO            Retain           Bound    devops/jenkins-home-pvc   nfs                     44m

NAME                                     STATUS   VOLUME            CAPACITY   ACCESS MODES   STORAGECLASS   AGE
persistentvolumeclaim/jenkins-home-pvc   Bound    jenkins-home-pv   10Gi       RWO            nfs            44m

NAME              TYPE       CLUSTER-IP    EXTERNAL-IP   PORT(S)                          AGE
service/jenkins   NodePort   10.111.96.4   <none>        8080:30008/TCP,50000:30005/TCP   44m
[root@k8s-master jenkins]# kubectl get ep -n devops
NAME      ENDPOINTS                                  AGE
jenkins   10.244.169.132:50000,10.244.169.132:8080   44m
[root@k8s-master jenkins]# kubectl get pods -n devops
NAME                      READY   STATUS    RESTARTS   AGE
jenkins-74f44f658-rjfb2   1/1     Running   0          30m
[root@k8s-master jenkins]#
[root@k8s-master jenkins]# kubectl describe pods jenkins-74f44f658-rjfb2  -n devops
---
Events:
  Type    Reason     Age   From                Message
  ----    ------     ----  ----                -------
  Normal  Scheduled  31m   default-scheduler   Successfully assigned devops/jenkins-74f44f658-rjfb2 to k8s-node2
  Normal  Pulled     31m   kubelet, k8s-node2  Container image "192.168.166.132/library/jenkins-2.273" already present on machine
  Normal  Created    31m   kubelet, k8s-node2  Created container jenkins
  Normal  Started    31m   kubelet, k8s-node2  Started container jenkins

# 看到以上这些信息说明启动成功

查看pod日志

kubectl logs jenkins-74f44f658-rjfb2  -n devops
---

# 复制管理员密码

访问Jenkins，如果没有做ingress 使用NodePort+暴露的端口

访问地址：http://NodePort:30008

第一次部署会进行初始化：

点无，不安装任何插件

创建管理员账号 。。。。

 

安装插件

默认从国外网络下载插件，会比较慢，建议修改国内源：

# 进入到nfs共享目录
cd /nfsdata/data/jenkins/updates/
sed -i 's/https:\/\/updates.jenkins.io\/download/https:\/\/mirrors.tuna.tsinghua.edu.cn\/jenkins/g' default.json
sed -i 's/http:\/\/www.google.com/https:\/\/www.baidu.com/g' default.json

# 删除pod重建，pod名称改成你实际的
kubectl delete pod jenkins-dccd449c7-vx6sj -n devops

# 或者 xxx.yaml 是启动Jenkins pod的yaml文件
kubectl delete -f xxx.yaml
kubectl apply -f xxx.yaml

http://mirrors.tuna.tsinghua.edu.cn/jenkins/updates/update-center.json

管理Jenkins->系统配置-->管理插件-->分别搜索Git Parameter/Git/Pipeline/kubernetes/Config

File Provider，选中点击安装。

Git Parameter：Git参数化构建

Git：拉取代码

Pipeline：流水线

kubernetes：连接Kubernetes动态创建Slave代理

Config File Provider：存储kubectl用于连接k8s集群的kubeconfig配置文件

 

配置Kubernetes云信息

在系统管理 --> 系统设置 ，最后面有个Cloud设置，Add a new cloud

 

添加具体的Kubernetes信息，K8S服务器可以是Jenkins本身所在的服务器，也可以是其他集群(需要配置证书)，这里以本身所在集群为例。

• 名称 ：用于pipeline调用云名称
• Kubernetes地址：可以通过kubectl cluster-info命令获取
• Kubernetes 服务证书 key：本身所在的集群因为我们通过sa所以不需要
• Kubernetes 命名空间：Jenkins的nodePod节点启动的namespace
• Jenkins 地址：主节点8080端口通过nodeport暴露出来的，地址:端口
• Jenkins 通道：主节点50000端口通过nodeport暴露出来的，地址:端口
  

  构建Slave镜像

• 课件目录里涉及四个文件：
  Dockerfile：构建镜像
  jenkins-slave：shell脚本启动slave.jar
  settings.xml：修改maven官方源为阿里云源
  slave.jar：agent程序，接受master下发的任务

slave镜像资源下载地址：

https://gitee.com/fdd-39969/cicd.git

git clone https://gitee.com/fdd-39969/cicd.git

Dockerfile文件内容如下：

[root@k8s-master jenkins]# cd jenkins-slave/
[root@k8s-master jenkins-slave]# ls
Dockerfile  jenkins-slave  kubectl  settings.xml  slave.jar
[root@k8s-master jenkins-slave]#
[root@k8s-master jenkins-slave]#
[root@k8s-master jenkins-slave]# cat Dockerfile
FROM centos:7
LABEL maintainer fengyuanfei

RUN yum install -y java-1.8.0-openjdk maven curl git libtool-ltdl-devel && \
    yum clean all && \
    rm -rf /var/cache/yum/* && \
    mkdir -p /usr/share/jenkins

COPY slave.jar /usr/share/jenkins/slave.jar
COPY jenkins-slave /usr/bin/jenkins-slave
COPY settings.xml /etc/maven/settings.xml
RUN chmod +x /usr/bin/jenkins-slave
COPY kubectl /usr/bin/

ENTRYPOINT ["jenkins-slave"]

构建并推送到镜像仓库：

docker build -t 192.168.166.132/library/jenkins-slave-jdk .
docker push 192.168.166.132/library/enkins-slave-jdk

war包可以通过添加 new新节点 获取下载

编写Pipeline脚本

创建项目->流水线->Pipeline脚本案例如下： 

// 公共
def registry = "reg.ctnrs.com"
// 项目
def project = "dev"
def app_name = "java-demo"
def image_name = "${registry}/${project}/${app_name}:${BUILD_NUMBER}"
def git_address = "http://192.168.31.70:9999/root/java-demo.git"
// 认证
def secret_name = "registry-pull-secret"
def docker_registry_auth = "c333c561-73af-43a7-bad8-83c9b9433916"
def git_auth = "d946c462-bec9-4b5b-9b77-9af538dd1776"
def k8s_auth = "d2374738-3c6f-4077-aa55-186b74c50002"

pipeline {
  agent {
    kubernetes {
        label "jenkins-slave"
        yaml """
kind: Pod
metadata:
  name: jenkins-slave
spec:
  containers:
  - name: jnlp
    image: "${registry}/library/jenkins-slave-jdk:1.8"
    imagePullPolicy: Always
    volumeMounts:
      - name: docker-cmd
        mountPath: /usr/bin/docker
      - name: docker-sock
        mountPath: /var/run/docker.sock
      - name: maven-cache
        mountPath: /root/.m2
  volumes:
    - name: docker-cmd
      hostPath:
        path: /usr/bin/docker
    - name: docker-sock
      hostPath:
        path: /var/run/docker.sock
    - name: maven-cache
      hostPath:
        path: /tmp/m2
"""
        }

      }
    parameters {
        gitParameter branch: '', branchFilter: '.*', defaultValue: 'master', description: '选择发布的分支', name: 'Branch', quickFilterEnabled: false, selectedValue: 'NONE', sortMode: 'NONE', tagFilter: '*', type: 'PT_BRANCH'
        choice (choices: ['1', '3', '5', '7'], description: '副本数', name: 'ReplicaCount')
        choice (choices: ['dev','test','prod'], description: '命名空间', name: 'Namespace')
    }
    stages {
        stage('拉取代码'){
            steps {
                checkout([$class: 'GitSCM',
                branches: [[name: "${params.Branch}"]],
                doGenerateSubmoduleConfigurations: false,
                extensions: [], submoduleCfg: [],
                userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address}"]]
                ])
            }
        }

        stage('代码编译'){
           steps {
             sh """
                mvn clean package -Dmaven.test.skip=true
                """
           }
        }

        stage('构建镜像'){
           steps {
                withCredentials([usernamePassword(credentialsId: "${docker_registry_auth}", passwordVariable: 'password', usernameVariable: 'username')]) {
                sh """
                  echo '
                    FROM ${registry}/library/tomcat:v1
                    LABEL maitainer lizhenliang
                    RUN rm -rf /usr/local/tomcat/webapps/*
                    ADD target/*.war /usr/local/tomcat/webapps/ROOT.war
                  ' > Dockerfile
                  docker build -t ${image_name} .
                  docker login -u ${username} -p '${password}' ${registry}
                  docker push ${image_name}
                """
                }
           }
        }
        stage('部署到K8S平台'){
          steps {
              configFileProvider([configFile(fileId: "${k8s_auth}", targetLocation: "admin.kubeconfig")]){
                sh """
                  sed -i 's#IMAGE_NAME#${image_name}#' deploy.yaml
                  sed -i 's#SECRET_NAME#${secret_name}#' deploy.yaml
                  sed -i 's#REPLICAS#${ReplicaCount}#' deploy.yaml
                  kubectl apply -f deploy.yaml -n ${Namespace} --kubeconfig=admin.kubeconfig
                """
              }
          }
        }
    }
}

上述脚本中，registry变量值改成你的镜像仓库地址。

将之前部署该项目涉及的yaml合并到一个名为deploy.yaml文件中，并提交到该项目代码仓库，与

pom.xml同级目录，用于上述脚本自动部署使用。

1、将harbor认证和gitlab认证保存到Jenkins凭据

管理Jenkins->安全-->管理凭据->Jnekins->添加凭据->Username with password

Username：用户名

Password：密码

ID：留空

Description：描述

分别添加连接git和harbor凭据，并修改上面脚本docker_registry_auth 和git_auth变量的值为Jenkins

凭据ID。

2、将kubeconfig存储在Jenkins，用于slave镜像里kubectl连接k8s集群

管理Jenkins-> Managed files->Add->Custom file ->Content字段内容是kubeconfig（默认路径在

master节点/root/.kube/config），然后复制ID替换上述脚本中k8s_auth变量的值。

构建测试

第一步测试：

pipeline {
    agent {
        kubernetes {
            label "jenkins-slave"
            yaml '''
apiVersion: v1
kind: Pod
metadata:
  name: jenkins-slave
spec:
  containers:
  - name: jnlp
    image: 192.168.166.132/library/jenkins-slave-jdk:1.8
'''
        }
    }
    stages {
        stage('第一步测试') {
            steps {
                sh 'hostname'
            }
        }
    }
}

\未完。。。待续。。。