物理ceph集群+K8s

前提条件

在Ceph为k8s创建一个pool

ceph osd pool create k8s 128

创建admin用户

ceph auth get-or-create client.admin mon 'allow r' osd 'allow rwx pool=k8s'

将admin用户的key进行base64编码

[root@node21 my-cluster]# ceph auth get-key client.admin | base64
QVFCbCtHTmQwdEN4TmhBQUdXcElhTkI1QXg0M2dDOWlNemM1dlE9PQ==

使用ceph的filesystem

在k8s集群中，创建secret

apiVersion: v1
kind: Secret
metadata:
  name: ceph-secret
data:
  key: QVFCbCtHTmQwdEN4TmhBQUdXcElhTkI1QXg0M2dDOWlNemM1dlE9PQ==

注：secret是namespace资源，需要在使用的namespace下创建

创建对应的资源

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: demoapp-redis
  namespace: isphere-dev
spec:
  replicas: 1
  serviceName: demoapp-redis
  selector:
    matchLabels:
      app: demoapp-redis
  template:
    metadata:
      labels:
        app: demoapp-redis
    spec:
      containers:
      - name: demoapp-redis
        image: hub.geovis.io/dockerhub/redis
        imagePullPolicy: Always
        ports:
        - name: demoapp-redis
          containerPort: 6379
        volumeMounts:
        - name: demoapp-redis-path
          mountPath: /var/lib/redis
      volumes:
      - name: demoapp-redis-path
        cephfs:
          monitors:
          - 192.168.4.21:6789
          - 192.168.4.22:6789
          - 192.168.4.29:6789
          user: admin
          secretRef:
            name: ceph-secret
          path: /k8svolume/isphere-dev/demoapp-redis
          readOnly: false

k8s下使用ceph的动态扩展，storageclass

创建sceret

[root@t31 ceph_sc]# vi ceph-secret.yaml 

apiVersion: v1
kind: Secret
metadata:
  name: ceph-admin-secret
type: "kubernetes.io/rbd"  #重点
data:
  key: QVFCbCtHTmQwdEN4TmhBQUdXcElhTkI1QXg0M2dDOWlNemM1dlE9PQ==

创建storageclass

[root@t31 ceph_sc]# vim storage-class-ceph.yaml 

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
   name: ceph-rbd
provisioner: kubernetes.io/rbd
parameters:
  monitors: 192.168.4.21:6789,192.168.4.22:6789,192.168.4.29:6789
  adminId: admin
  adminSecretName: ceph-admin-secret
  adminSecretNamespace: kube-system
  pool: k8s
  userId: admin
  userSecretName: ceph-admin-secret
allowVolumeExpansion: true
reclaimPolicy: Delete

创建pvc

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: nginx-test-vol1-claim
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: ceph-rbd
  resources:
    requests:
      storage: 10Gi

创建资源

apiVersion: v1
kind: Pod
metadata:
  name: nginx-test
spec:
  containers:
  - name: nginx
    image: nginx:latest
    volumeMounts:
      - name: nginx-test-vol1
        mountPath: /data/
        readOnly: false
  volumes:
  - name: nginx-test-vol1
    persistentVolumeClaim:
      claimName: nginx-test-vol1-claim