2.Jenkins结合k8s完成Jenkins slave功能

1.构建镜像

下载基础镜像，这里使用openvz的包，下载地址为：https://wiki.openvz.org/Download/template/precreated，下载centos7的镜像

下载镜像后导入到本地

docker import centos--x86_64.tar.gz openvz-centos7

从官方下载最新版的Jenkins的war包和适配的jdk，这里使用jdk8。构建Jenkins的dockerfile文件dockerfile-jenkins

[root@dataserver jenkins]#cat dockerfile-jenkins
FROM openvz-centos7
ADD jdk-8u241-linux-x64.tar.gz  /home
ADD jenkins2..war /home/jenkins.war
ENV JAVA_HOME=/home/jdk1..0_241
ENV PATH=$PATH:/home/jdk1..0_241/bin
ENV JENKINS_HOME=/var/jenkins_home
WORKDIR /home
CMD java -jar jenkins.war --httpPort=

构建后推送到本地仓库

docker build -t 192.168.31.9:/jenkins-self:2.222 -f dockerfile-jenkins .
docker push 192.168.31.9:/jenkins-self:2.222

构建Jenkins-agent的dockerfile-agent

[root@dataserver jenkins]# cat dockerfile-agent
FROM openvz-centos7
ADD jdk-8u241-linux-x64.tar.gz  /home
ADD agent.jar /home
ENV JAVA_HOME=/home/jdk1..0_241
ENV PATH=$PATH:/home/jdk1..0_241/bin
RUN yum -y install docker kubernetes-client kubernetes*
WORKDIR /home
CMD exec /home/jdk1..0_241/bin/java -Dorg.jenkinsci.remoting.engine.JnlpProtocol3.disabled=true -cp /home/agent.jar hudson.remoting.jnlp.Main -headless -url ${JENKINS_URL} -workDir ${JENKINS_AGENT_WORKDIR} ${JENKINS_SECRET} ${JENKINS_AGENT_NAME}

构建后推送到本地仓库

docker build -t 192.168.31.9:/jenkins-agent-self:v2.222.11 -f dockerfile-agent .
docker push 192.168.31.9:/jenkins-agent-self:v2.222.11

agent.jar可以从Jenkins中下载。manager Jenkins --> 节点管理 --> 新建节点 --> 输入test，选择固定节点 --> 远程工作目录输入 /home --> 启动方式选择通过web web启动代理后点击保存。
这里这个节点是不在线的，点击后提示如下：

点击下面界面显示的agent.jar后就可以下载对应的agent.jar包

2.部署Jenkins到k8s中

namespace.yaml

[root@env11 cicd]# cat namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: kube-ops

pvc.yaml这里使用nfs服务提供外置存储

[root@env11 cicd]# cat pvc.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
  name: opspv
spec:
  capacity:
    storage: 20Gi
  accessModes:
  - ReadWriteMany
  persistentVolumeReclaimPolicy: Delete
  nfs:
    server: 192.168.31.9
    path: /data/nfsData/jenkins

---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: opspvc
  namespace: kube-ops
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 20Gi

rbac.yaml

[root@env11 cicd]# cat rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins2
  namespace: kube-ops

---

kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: jenkins2
  namespace: kube-ops
rules:
  - apiGroups: ["extensions", "apps"]
    resources: ["deployments"]
    verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
  - apiGroups: [""]
    resources: ["services"]
    verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["create","delete","get","list","patch","update","watch"]
  - apiGroups: [""]
    resources: ["pods/exec"]
    verbs: ["create","delete","get","list","patch","update","watch"]
  - apiGroups: [""]
    resources: ["pods/log"]
    verbs: ["get","list","watch"]
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get"]

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: jenkins2
  namespace: kube-ops
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: jenkins2
subjects:
  - kind: ServiceAccount
    name: jenkins2
    namespace: kube-ops

jenkins2.yaml

[root@env11 cicd]# cat jenkins2.yaml
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: jenkins2
  namespace: kube-ops
spec:
  template:
    metadata:
      labels:
        app: jenkins2
    spec:
      terminationGracePeriodSeconds:
      serviceAccount: jenkins2
      containers:
      - name: jenkins
        image: 192.168.31.9:/jenkins-self:v2.
        imagePullPolicy: IfNotPresent
        command: ["/bin/sh","-c","java -jar jenkins.war --httpPort=8080"]
        ports:
        - containerPort:
          name: web
          protocol: TCP
        - containerPort:
          name: agent
          protocol: TCP
        resources:
          limits:
            cpu: 2000m
            memory: 2Gi
          requests:
            cpu: 500m
            memory: 512Mi
        #livenessProbe:
        #  httpGet:
        #    path: /login
        #    port:
        #  initialDelaySeconds:
        #  timeoutSeconds:
        #  failureThreshold:
        #readinessProbe:
        #  httpGet:
        #    path: /login
        #    port:
        #  initialDelaySeconds:
        #  timeoutSeconds:
        #  failureThreshold:
        volumeMounts:
        - name: jenkinshome
          subPath: jenkinsenv
          mountPath: /var/jenkins_home
        env:
        - name: LIMITS_MEMORY
          valueFrom:
            resourceFieldRef:
              resource: limits.memory
              divisor: 1Mi
        - name: JAVA_OPTS
          value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay= -Dhudson.slaves.NodeProvisioner.MARGIN= -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85 -Duser.timezone=Asia/Shanghai
      securityContext:
        fsGroup:
      volumes:
      - name: jenkinshome
        persistentVolumeClaim:
          claimName: opspvc

---
apiVersion: v1
kind: Service
metadata:
  name: jenkins2
  namespace: kube-ops
  labels:
    app: jenkins2
spec:
  selector:
    app: jenkins2
  type: NodePort
  ports:
  - name: web
    port:
    targetPort: web
    nodePort:
  - name: agent
    port:
    targetPort: agent

部署完成后，就可以通过集群IP:3002后登陆Jenkins

3.需要安装插件：

Jenkins需要安装插件Kubernetes plugin插件才可以动态调用k8s的api接口完成pod的创建

4.配置Jenkins调用k8s配置

配置Jenkins代理为固定端口50000，因为service定义是对外暴露的端口是50000。

点击Jenkins的节点管理

点击节点管理

点击configure clouds，添加一个远端服务

配置Kubernetes地址为http://kubernetes.default:443，命名空间为kube-ops，和Jenkins是同一个命名空间，然后点击连接测试。因为Jenkins的pod在提交是配置了rbac授权，所以可以访问k8s。

配置Jenkins地址为http://jenkins2.kube-ops.svc.cluster.local:8080，如果上面的service配置的名字为jenkins，这里就配置为jenkins而不是jenkins2。

配置pod模版，名字配置为jenkins-slave-001，命名空间为kube-ops，标签列表为haimaxy-jnlp，这个标签列表很重要，后面需要用到。容器列表配置jnlp，镜像就是自己构建的192.168.31.9:5000/jenkins-agent-self:v2.222.11，工作目录写/home/jenkins

下面的运行的命令和参数都不要写，否则会覆盖掉镜像中定义的启动命令。

这里把/var/run/docker.sock和/home/jenkins/.kube挂载到容器中，这样可以使用docker和kubectl命令。注意kubectl必须在每个节点都可以执行。

5.测试

添加一个项目来测试。

这里选择节点标签就是前面的定义的haimaxy-jnlp。

写入shell命令。

添加完成后开始构建

构建过程，就是在k8s中创建了一个pod来运行。

查看构建日志

此时在k8s中查看命名空间kube-ops下有自动创建的pod。

执行完成后查看日志，发现同样获取了命名空间下的pod信息。