一、环境介绍

[root@k8s-m ~]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6

172.31.250.152 k8s-m
172.31.250.153 node1
172.31.250.154 node2

[root@k8s-m ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-m Ready master 41m v1.13.1
node1 Ready <none> 27m v1.13.1
node2 Ready <none> 28m v1.13.1

#删除k8s集群master的污点(我这边就三台服务器,需要使用master,生产环境千万别这么干)
[root@k8s-m ~]# kubectl taint node k8s-m node-role.kubernetes.io/master-
node/k8s-m untainted

所有存储服务器下载

yum install centos-release-gluster -y

yum install glusterfs-server -y

二、Heketi安装

所有版本:https://github.com/heketi/heketi/releases

#下载安装
wget https://github.com/heketi/heketi/releases/download/v8.0.0/heketi-v8.0.0.linux.amd64.tar.gz
tar xf heketi-v8.0.0.linux.amd64.tar.gz
mkdir -p /data/heketi/{bin,conf,data}
cp heketi/heketi.json /data/heketi/conf/
cp heketi/{heketi,heketi-cli} /data/heketi/bin/ #我们glusterFS部署在k8s集群外,所以heketi通过ssh管理glusterFS。需要创建免秘钥登陆到所有glusterFS节点。
ssh-keygen -f /data/heketi/conf/heketi_key -t rsa -N ''
#将公钥放到所有GlusterFS节点
ssh-copy-id -i /data/heketi/conf/heketi_key.pub root@k8s-m
ssh-copy-id -i /data/heketi/conf/heketi_key.pub root@node1
ssh-copy-id -i /data/heketi/conf/heketi_key.pub root@node2

2、heketi.json配置文件

[root@k8s-m ~]# cat /data/heketi/conf/heketi.json
{
"_port_comment": "Heketi Server Port Number",
"port": "18080", "_enable_tls_comment": "Enable TLS in Heketi Server",
"enable_tls": false, "_cert_file_comment": "Path to a valid certificate file",
"cert_file": "", "_key_file_comment": "Path to a valid private key file",
"key_file": "", "_use_auth": "Enable JWT authorization. Please enable for deployment",
"use_auth": true, "_jwt": "Private keys for access",
"jwt": {
"_admin": "Admin has access to all APIs",
"admin": {
"key": "adminkey"
},
"_user": "User only has access to /volumes endpoint",
"user": {
"key": "userkey"
}
}, "_backup_db_to_kube_secret": "Backup the heketi database to a Kubernetes secret when running in Kubernetes. Default is off.",
"backup_db_to_kube_secret": false, "_glusterfs_comment": "GlusterFS Configuration",
"glusterfs": {
"_executor_comment": [
"Execute plugin. Possible choices: mock, ssh",
"mock: This setting is used for testing and development.",
" It will not send commands to any node.",
"ssh: This setting will notify Heketi to ssh to the nodes.",
" It will need the values in sshexec to be configured.",
"kubernetes: Communicate with GlusterFS containers over",
" Kubernetes exec api."
],
"executor": "ssh", "_sshexec_comment": "SSH username and private key file information",
"sshexec": {
"keyfile": "/data/heketi/conf/heketi_key",
"user": "root",
"port": "22",
"fstab": "/etc/fstab",
"backup_lvm_metadata": false
}, "_kubeexec_comment": "Kubernetes configuration",
"kubeexec": {
"host" :"https://kubernetes.host:8443",
"cert" : "/path/to/crt.file",
"insecure": false,
"user": "kubernetes username",
"password": "password for kubernetes user",
"namespace": "OpenShift project or Kubernetes namespace",
"fstab": "Optional: Specify fstab file on node. Default is /etc/fstab",
"backup_lvm_metadata": false
}, "_db_comment": "Database file name",
"db": "/data/heketi/data/heketi.db", "_refresh_time_monitor_gluster_nodes": "Refresh time in seconds to monitor Gluster nodes",
"refresh_time_monitor_gluster_nodes": 120, "_start_time_monitor_gluster_nodes": "Start time in seconds to monitor Gluster nodes when the heketi comes up",
"start_time_monitor_gluster_nodes": 10, "_loglevel_comment": [
"Set log level. Choices are:",
" none, critical, error, warning, info, debug",
"Default is warning"
],
"loglevel" : "debug", "_auto_create_block_hosting_volume": "Creates Block Hosting volumes automatically if not found or exsisting volume exhausted",
"auto_create_block_hosting_volume": true, "_block_hosting_volume_size": "New block hosting volume will be created in size mentioned, This is considered only if auto-create is enabled.",
"block_hosting_volume_size": 500, "_block_hosting_volume_options": "New block hosting volume will be created with the following set of options. Removing the group gluster-block option is NOT recommended. Additional options can be added next to it separated by a comma.",
"block_hosting_volume_options": "group gluster-block"
}
}

  

3、创建heketi启动脚本

[root@k8s-m ~]# cat /usr/lib/systemd/system/heketi.service
[Unit]
Description=RESTful based volume management framework for GlusterFS
Before=network-online.target
After=network-online.target
Documentation=https://github.com/heketi/heketi
[Service]
Type=simple
LimitNOFILE=
ExecStart=/data/heketi/bin/heketi --config=/data/heketi/conf/heketi.json
KillMode=process
Restart=on-failure
RestartSec=
SuccessExitStatus=
StandardOutput=syslog
StandardError=syslog
[Install]
WantedBy=multi-user.target #启动heketi
systemctl start heketi
systemctl enable heketi
systemctl status heketi

4、Heketi添加cluster

[root@k8s-m ~]# /data/heketi/bin/heketi-cli --user admin --server http://k8s-m:18080 --secret adminkey --json  cluster create
{"id":"aa378f94299c29bbd0224fc902c0cbd8","nodes":[],"volumes":[],"block":true,"file":true,"blockvolumes":[]}

5、#将3个glusterfs节点作为node添加到cluster

[root@k8s-m ~]# /data/heketi/bin/heketi-cli --server http://k8s-m:18080   --user "admin" --secret "adminkey"   node add --cluster "aa378f94299c29bbd0224fc902c0cbd8"  --management-host-name  k8s-m --storage-host-name 172.31.250.152 --zone 1
Node information:
Id: 9280cde1c2640bf3fef483509a04bff2
State: online
Cluster Id: aa378f94299c29bbd0224fc902c0cbd8
Zone:
Management Hostname k8s-m
Storage Hostname 172.31.250.152 [root@k8s-m ~]# /data/heketi/bin/heketi-cli --server http://k8s-m:18080 --user "admin" --secret "adminkey" node add --cluster "aa378f94299c29bbd0224fc902c0cbd8" --management-host-name node1 --storage-host-name 172.31.250.153 --zone 1
Node information:
Id: 19838aea8104aee077b0ea67cd6ea012
State: online
Cluster Id: aa378f94299c29bbd0224fc902c0cbd8
Zone:
Management Hostname node1
Storage Hostname 172.31.250.153

[root@k8s-m ~]# /data/heketi/bin/heketi-cli --server http://k8s-m:18080 --user "admin" --secret "adminkey" node add --cluster "aa378f94299c29bbd0224fc902c0cbd8" --management-host-name node2 --storage-host-name 172.31.250.154 --zone 1
Node information:
Id: 171aeb876968bf9ef1fab28e6a31e919
State: online
Cluster Id: aa378f94299c29bbd0224fc902c0cbd8
Zone:
Management Hostname node2
Storage Hostname 172.31.250.154

6、添加device

#机器只是作为gluster的运行单元,volume是基于device创建的。同时需要特别说明的是,目前heketi仅支持使用裸分区或裸磁盘(未格式化)添加为device,不支持文件系统

[root@k8s-m ~]# /data/heketi/bin/heketi-cli --server http://k8s-m:18080   --user "admin" --secret "adminkey"    --json device add --name="/dev/vdb" --node "9280cde1c2640bf3fef483509a04bff2"
Device added successfully [root@k8s-m ~]# /data/heketi/bin/heketi-cli --server http://k8s-m:18080 --user "admin" --secret "adminkey" --json device add --name="/dev/vdb" --node "19838aea8104aee077b0ea67cd6ea012"
Device added successfully [root@k8s-m ~]# /data/heketi/bin/heketi-cli --server http://k8s-m:18080 --user "admin" --secret "adminkey" --json device add --name="/dev/vdb" --node "171aeb876968bf9ef1fab28e6a31e919"
Device added successfully ##查看
[root@k8s-m ~]# /data/heketi/bin/heketi-cli --server http://k8s-m:18080 --user "admin" --secret "adminkey" node list
Id:171aeb876968bf9ef1fab28e6a31e919 Cluster:aa378f94299c29bbd0224fc902c0cbd8
Id:19838aea8104aee077b0ea67cd6ea012 Cluster:aa378f94299c29bbd0224fc902c0cbd8
Id:9280cde1c2640bf3fef483509a04bff2 Cluster:aa378f94299c29bbd0224fc902c0cbd8

7、添加volume

创建一个大小为3G,副本为3的volume
[root@k8s-m ~]# /data/heketi/bin/heketi-cli --server http://k8s-m:18080 --user "admin" --secret "adminkey" volume create --size 3 --replica 3
Name: vol_2aab01a26b3a2dfc1286eed16c8fcb62
Size: 3
Volume Id: 2aab01a26b3a2dfc1286eed16c8fcb62
Cluster Id: aa378f94299c29bbd0224fc902c0cbd8
Mount: 172.31.250.154:vol_2aab01a26b3a2dfc1286eed16c8fcb62
Mount Options: backup-volfile-servers=172.31.250.153,172.31.250.152
Block: false
Free Size: 0
Reserved Size: 0
Block Hosting Restriction: (none)
Block Volumes: []
Durability Type: replicate
Distributed+Replica: 3

  

8、创建storageclass

[root@k8s-m ~]# cat   storageclass-glusterfs.yaml
apiVersion: v1
kind: Secret
metadata:
name: heketi-secret
namespace: default
data:
# base64 encoded password. E.g.: echo -n "mypassword" | base64
key: YWRtaW5rZXk=
type: kubernetes.io/glusterfs ---
apiVersion: storage.k8s.io/v1beta1
kind: StorageClass
metadata:
name: glusterfs
provisioner: kubernetes.io/glusterfs
allowVolumeExpansion: true
parameters:
resturl: "http://172.31.250.152:18080"
clusterid: "aa378f94299c29bbd0224fc902c0cbd8"
restauthenabled: "true"
restuser: "admin"
#secretNamespace: "default"
#secretName: "heketi-secret"
restuserkey: "adminkey"
gidMin: ""
gidMax: ""
volumetype: "replicate:3" #导入
[root@k8s-m ~]# kubectl apply -f storageclass-glusterfs.yaml
secret/heketi-secret created
storageclass.storage.k8s.io/glusterfs created

9、创建一个 statefulset测试是否会自动生成pv

#查看pv和pvc
[root@k8s-m ~]# kubectl get pv --all-namespaces
No resources found.
[root@k8s-m ~]# kubectl get pvc --all-namespaces
No resources found. #statefulset文件
[root@k8s-m ~]# cat mystatefulset.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx
labels:
app: nginx
spec:
ports:
- port:
name: web
clusterIP: None
selector:
app: nginx
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: nginx
spec:
selector:
matchLabels:
app: nginx # has to match .spec.template.metadata.labels
serviceName: "nginx"
replicas: # by default is
template:
metadata:
labels:
app: nginx # has to match .spec.selector.matchLabels
spec:
terminationGracePeriodSeconds:
containers:
- name: nginx
image: nginx
ports:
- containerPort:
name: web
volumeMounts:
- name: www
mountPath: /usr/share/nginx/html
volumeClaimTemplates:
- metadata:
name: www
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: "glusterfs"
resources:
requests:
storage: 2Gi #查看pv和pvc(根据pvc自动创建了pv)
[root@k8s-m ~]# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvc-0afdc43a--11e9-b8f6-00163e024454 2Gi RWO Delete Bound default/www-nginx- glusterfs 6s
[root@k8s-m ~]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
www-nginx- Bound pvc-0afdc43a--11e9-b8f6-00163e024454 2Gi RWO glusterfs 16s

k8s使用Glusterfs动态生成pv的更多相关文章

  1. 4.k8s存储之Volume、PV、PVC和StatefulSet

    3.Volume 容器磁盘上的文件的生命周期是短暂的,这就使得在容器中运行重要应用时会出现一些问题.首先,当容器崩溃时,kubelet 会重启它,但是容器中的文件将丢失--容器以干净的状态(镜像最初的 ...

  2. Aop动态生成代理类时支持带参数构造函数

    一.背景 在某些情况下,我们需要植入AOP代码的类并没有默认构造函数.那么此时动态生成的代理类也需要相同签名的构造函数,并且内部调用原始类的构造函数.自己折腾了1晚上没搞定,现在搞定了发出来供大家一起 ...

  3. dynamic-css 动态 CSS 库,使得你可以借助 MVVM 模式动态生成和更新 css,从 js 事件和 css 选择器的苦海中脱离出来

    dynamic-css 使得你可以借助 MVVM 模式动态生成和更新 css,从而将本插件到来之前,打散.嵌套在 js 中的修改样式的代码剥离出来.比如你要做元素跟随鼠标移动,或者根据滚动条位置的变化 ...

  4. ABP(现代ASP.NET样板开发框架)系列之20、ABP展现层——动态生成WebApi

    点这里进入ABP系列文章总目录 ABP(现代ASP.NET样板开发框架)系列之20.ABP展现层——动态生成WebApi ABP是“ASP.NET Boilerplate Project (ASP.N ...

  5. 【.NET深呼吸】Zip文件操作(2):动态生成Zip文档

    通过前面一篇烂文的介绍,大伙儿知道,ZipArchive类表示一个zip文档实例,除了用上一篇文章中所列的方法来读写zip文件外,还可以直接通过ZipArchive类,动态生成zip文件. 文件流操作 ...

  6. jquery动态生成的元素添加事件的方法

    动态生成的元素如果要添加事件,要写成 $(document).on("click", "#txtName", function() { alert(this.v ...

  7. 利用Java动态生成 PDF 文档

    利用Java动态生成 PDF 文档,则需要开源的API.首先我们先想象需求,在企业应用中,客户会提出一些复杂的需求,比如会针对具体的业务,构建比较典型的具备文档性质的内容,一般会导出PDF进行存档.那 ...

  8. 用C#从数据库动态生成AdminLTE菜单的一种方法

    当前的应用设计风格趋于Flat扁平化,很多基于BootStrap实现了很多UI非常漂亮的管理界面(Bootstrap admin template). 此核心文件开源在Github:https://g ...

  9. 动态生成二维码插件 jquery.qrcode.js

    前段时间做项目,需要动态生成一个二维码,于是就在网上找了一下发现一个jquery插件jquery.qrcode.js,所以今天就简单说一下这个插件的使用: jquery.qrcode.js是依赖jqu ...

随机推荐

  1. [matlab] 22.matlab图论实例 最短路问题与最小生成树 (转载)

    最短路问题之 Floyd 某公司在六个城市 c1c1,c2c2,….,c6c6 中有分公司,从 cici 到 cjcj 的直接航程票价记在下述矩阵的 (ii,jj) 位置上. (∞∞表示无直接航路), ...

  2. 【vue】vue +element 搭建项目,实现实时输入效果时停止输入后发送请求

    1.实现的效果 输入关键字后,根据输入的关键字实时显示搜索的结果,按回车键时也进行搜索 2.原理: 通过时间戳+定时器+一个全局变量实现.代码量很少比较易懂. 用户输入时触发keyup事件,并调用事件 ...

  3. 【vue】vue +element 搭建项目,el-input 常用的验证

    1.el-input 常用布局 <el-input class="filter-item dialog-search" size="small" @key ...

  4. 三、Oracle 查询+where条件

    一.查询1.distinct:查询去除重复的行,是所有的列都重复才满足条件2.列别名:as或者空格 select name as 姓名 from student3.查询字段可以做数学运算,也可以做字符 ...

  5. ORA-27154: post/wait create failed ORA-27300 ORA-27301 ORA-27302

    今天刚装了Oracle 11g,配制好了之后启动数据库时遇到下面的错误:SQL> startupORA-27154: post/wait create failedORA-27300: OS s ...

  6. 性能调优9:根据WaitType诊断性能

    SQL Server数据库接收到查询请求,从生成计划到执行计划的过程,等待次数和等待时间在一定程度上揭示了系统性能的压力,如果资源严重不足,就会成为性能的瓶颈.因此,对等待的监控非常有助于对系统性能进 ...

  7. .net core实践系列之SSO-同域实现

    前言 SSO的系列还是以.Net Core作为实践例子与大家分享,SSO在Web方面复杂度分同域与跨域.本篇先分享同域的设计与实现,跨域将在下篇与大家分享. 如有需要调试demo的,可把SSO项目部署 ...

  8. 重装mysql后导致Navicat连接失败

    今天重装了mysql数据库,然后再使用navicat去连接数据库的时候,一直报错 1251 Client does not support authentication protocol reques ...

  9. 使用mysql,sql语言删除冗余信息

    这是表,我们需要操作的就是删除除了学号不同,其它信息都相同的冗余信息 思路:删除表格class3中的冗余的stu_id信息,那么接下来我们应该去筛选哪些stu_id信息是冗余的, 此时我们想到的就是利 ...

  10. html总结:固定表格中单元格宽度

    当然要提前设置好table的width值,然后再写这个,使得每列宽度都相等. <style> table { table-layout: fixed; } </style>