7.3 自定义镜像-运行nginx与tomcat并结合PV/PVC/NFS以实现动静分离示例

1、在NFS SERVER上为tomcat、nginx创建相关目录

NFS SERVER的部署配置参考：https://www.cnblogs.com/yanql/p/15410308.html 1.3.1 部署NFS server

# tomcat
mkdir /data/k8s-data/tomcat/webapps/apps1 -p
echo 'tomcat index' > /data/k8s-data/tomcat/webapps/apps1/index.html

# nginx
mkdir /data/k8s-data/nginx/html
echo 'nginx html index' > /data/k8s-data/nginx/html/index.html

2、制作tomcat镜像

2.1、准备tomcat优化配置文件

# catalina.sh 增加内存优化参数
JAVA_OPTS="-Dfile.encoding=UTF-8-server -Xms1024m -Xmx2048m -XX:NewSize=512m -XX:MaxNewSize=1024m -XX:MaxTenuringThreshold=10 -XX:NewRatio=2 -XX:+DisableExplicitGC"

# server.xml 开启Nio2线程模式并设置最大最新连接数
<Executor name="tomcatThreadPool" namePrefix="catalina-exec-"
            maxThreads="500" minSpareThreads="50" prestartminSpareThreads="true" maxQueueSize="100"/>

<Connector executor="tomcatThreadPool"
           port="8080" protocol="org.apache.coyote.http11.Http11Nio2Protocol"
           connectionTimeout="20000"
           redirectPort="8443" />

# tomcat-users.xml 在tomcat-users标签增加如下内容，开启tomcat页面管理
<role rolename="manager"/>
<role rolename="manager-gui"/>
<role rolename="admin"/>
<role rolename="admin-gui"/>
<user username="tomcat" password="FggOueSastmtlQzB" roles="admin-gui,admin,manager-gui,manager"/>

# 开启tomcat页面管理，还需要修改webapps/manager/META-INF/context.xml，将一下行注释掉。
  <!--<Valve className="org.apache.catalina.valves.RemoteAddrValve"
  allow="127\.\d+\.\d+\.\d+|::1|0:0:0:0:0:0:0:1" /> -->

2.2、编写tomcat镜像Dockerfile

# 目录中的文件如下：
-rw-r--r-- 1 root root  10564666 8月  10 03:52 apache-tomcat-8.5.70.tar.gz
-rwxr-x--- 1 root root     25458 10月 20 09:31 catalina.sh*
-rw-r--r-- 1 root root      1057 10月 20 10:14 Dockerfile
-rw-r--r-- 1 root root   2054960 9月  10 19:58 glibc-2.29-r0.apk
-rw-r--r-- 1 root root 191757099 9月  10 19:23 jdk-8u192-linux-x64.tar.gz
-rw------- 1 root root      7647 10月 20 09:31 server.xml
-rw------- 1 root root      3024 10月 20 10:13 tomcat-users.xml
-rw-r--r-- 1 root root   1494714 10月 20 09:53 webapps.tgz

# Dockerfile
FROM 192.168.1.110/base/alpine:v20211010-1704
LABEL maintainer="yanql<yanqianling@faxuan.net>"

ADD jdk-8u192-linux-x64.tar.gz /usr/local/
ADD apache-tomcat-8.5.70.tar.gz /usr/local/
ADD webapps.tgz /opt/
COPY catalina.sh /usr/local/apache-tomcat-8.5.70/bin/catalina.sh
COPY glibc-2.29-r0.apk /
COPY server.xml /usr/local/apache-tomcat-8.5.70/conf/
COPY tomcat-users.xml /usr/local/apache-tomcat-8.5.70/conf/

RUN set -x \
    && addgroup -g 101 -S nginx \
    && adduser -S -D -H -u 101 -h /var/cache/nginx -s /sbin/nologin -G nginx -g nginx nginx \
    && chown -R nginx:nginx /opt/webapps /usr/local/apache-tomcat-8.5.70 \
    && sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories \
    && wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub \
    && apk add glibc-2.29-r0.apk \
    && rm -rf /usr/local/apache-tomcat-8.5.70/webapps

ENV JAVA_HOME /usr/local/jdk1.8.0_192
ENV PATH ${PATH}:${JAVA_HOME}/bin 

EXPOSE 8080

CMD ["/usr/local/apache-tomcat-8.5.70/bin/catalina.sh", "run"]

2.3 构建镜像并长传到本地harbor

# docker build -t 192.168.1.110/web/alpine-jdk-8u192-tomcat-8.5.70:v20211020-1014 .

# docker push 192.168.1.110/web/alpine-jdk-8u192-tomcat-8.5.70:v20211020-1014

3、部署tomcat服务

3.1、编写tomcat ymal文件

root@k8-deploy:~/k8s-yaml/web/tomcat# cat tomcat.yml
apiVersion: v1
kind: Service
metadata:
  name: tomcat-svc
  namespace: yun
spec:
  type: NodePort
  ports:
    - name: http
      port: 8080
      protocol: TCP
      targetPort: 8080
      nodePort: 38080
  selector:
    app: tomcat-app
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tomcat-deploy
  namespace: yun
spec:
  replicas: 1
  selector:
    matchLabels:
      app: tomcat-app
  template:
    metadata:
      labels:
        app: tomcat-app
    spec:
      containers:
        - name: tomcat-ct
          image: 192.168.1.110/web/alpine-jdk-8u192-tomcat-8.5.70:v20211020-1014
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 8080
              protocol: TCP
              name: http-8080
          resources:
            limits:
              cpu: 2
              memory: 2Gi
            requests:
              cpu: 500m
              memory: 1Gi
          volumeMounts:
          - name: tomcat-webapps
            mountPath: "/opt/webapps"
      volumes:
        - name: tomcat-webapps
          nfs:
            server: 192.168.2.10
            path: /data/k8s-data/tomcat/webapps

3.2 启动并检查tomcat服务

root@k8-deploy:~/k8s-yaml/web/tomcat# kubectl apply -f tomcat.yml
service/tomcat-svc created

root@k8-deploy:~/k8s-yaml/web/tomcat# kubectl get pod -n yun
NAME                             READY   STATUS    RESTARTS   AGE
tomcat-deploy-74564db6d6-tbh98   1/1     Running   0          15s

root@k8-deploy:~/k8s-yaml/web/tomcat# kubectl get svc -n yun
NAME         TYPE       CLUSTER-IP    EXTERNAL-IP   PORT(S)          AGE
tomcat-svc   NodePort   10.0.106.99   <none>        8080:38080/TCP   2m17s  

# curl 192.168.2.18:38080/app1/index.html
tomcat index

# kubectl exec tomcat-deploy-74564db6d6-tbh98 -n yun -it -- sh
/ # df -h
Filesystem                Size      Used Available Use% Mounted on
...
192.168.2.10:/data/k8s-data/tomcat/webapps
                         97.9G     18.0G     74.9G  19% /opt/webapps
...

4、制作nginx镜像

4.1、准备nginx配置文件

# nginx.conf
root@k8-deploy:~/k8s-yaml/web/nginx/dockerfile# cat nginx.conf
user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log notice;
pid        /var/run/nginx.pid;

events {
    worker_connections  4096;
}

http {
    include       mime.types;
    default_type  application/octet-stream;
    client_max_body_size 200M;
    sendfile       on;
    keepalive_timeout 65;
    add_header Access-Control-Allow-Origin *;

    log_format main '$time_local|$remote_addr|$server_name|$request|'
                    '$status|$body_bytes_sent|$request_time|$bytes_sent|$request_length|'
                    '$upstream_response_time|$http_referer|$http_user_agent';
    access_log  /var/log/nginx/access.log  main;

    gzip on;
    gzip_min_length 100;
    gzip_buffers    16 8k;
    gzip_http_version 1.1;
    gzip_comp_level 2;
    gzip_types       text/plain application/x-javascript text/css application/xml text/javascript application/javascript application/octet-stream;
    gzip_vary on;

    upstream tomcat_webserver {
        server tomcat_svc:8080;
    }

    include conf.d/*.conf;
}

# default.conf
cat default.conf
server {
    listen       80;
    server_name  localhost;

    location / {
        root   /usr/share/nginx/html;
        index  index.html index.htm;
    }

    location /app1 {
        proxy_pass  http://tomcat_webserver;
        proxy_set_header    Host    $host;
        proxy_set_header    X-Real-IP                    $remote_addr;
        proxy_set_header    X-Forwarded-For              $proxy_add_x_forwarded_for;
        proxy_set_header    HTTP_X_FORWARDED_FOR      $remote_addr;
    }

    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   /usr/share/nginx/html;
    }
}

4.2、编写nginx镜像Dockerfile

 vim Dockerfile
FROM nginx:1.21.1
COPY nginx.conf /etc/nginx/
COPY default.conf /etc/nginx/conf.d/

4.3、构建nginx镜像并上传本地harbor

docker build -t 192.168.1.110/web/nginx:1.21.1-v3 .

docker push 192.168.1.110/web/nginx:1.21.1-v3

5、部署nginx服务

5.1、编写nginx ymal文件

root@k8-deploy:~/k8s-yaml/web/nginx/yaml# cat nginx.yml
apiVersion: v1
kind: Service
metadata:
  name: nginx-svc
  namespace: yun
spec:
  type: NodePort
  ports:
    - name: http
      port: 80
      protocol: TCP
      targetPort: 80
      nodePort: 30080
  selector:
    app: nginx-app
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deploy
  namespace: yun
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx-app
  template:
    metadata:
      labels:
        app: nginx-app
    spec:
      containers:
        - name: nginx-ct
          image: 192.168.1.110/web/nginx:1.21.1-v3
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 80
              protocol: TCP
              name: http
          resources:
            limits:
              cpu: 2
              memory: 2Gi
            requests:
              cpu: 500m
              memory: 1Gi
          volumeMounts:
          - name: nginx-html
            mountPath: "/usr/share/nginx/html"
      volumes:
        - name: nginx-html
          nfs:
            server: 192.168.2.10
            path: /data/k8s-data/nginx/html

5.2 启动并检查nginx服务

# kubectl apply -f nginx.yml

# kubectl get pod -n yun
NAME                             READY   STATUS    RESTARTS   AGE
nginx-deploy-7b9475c5b-2kzg9     1/1     Running   0          6s
tomcat-deploy-74564db6d6-tbh98   1/1     Running   0          88m

# 进入nginx pod 查看nfs挂载
# kubectl exec nginx-deploy-7b9475c5b-2kzg9 -n yun -it -- bash
root@nginx-deploy-7b9475c5b-2kzg9:/# df -h
Filesystem                              Size  Used Avail Use% Mounted on
...
192.168.2.10:/data/k8s-data/nginx/html   98G   18G   75G  20% /usr/share/nginx/html
...

# nginx index.html
# curl 192.168.2.17:30080
nginx html index

# 通过nginx代理的tomcat index.html
# curl 192.168.2.17:30080/app1/index.html
tomcat index