ceph安装对象网关

1、概述

安装3个网关节点分别是：controller-03、controller-04和controller-05，使用ceph gw自带的Civetweb提供服务，前端使用nginx作为前端代理。

2、分别在三各节点安装ceph gw

在ceph-deploy节点执行安装

ceph-deploy install --rgw controller-03 controller-04 controller-05

ceph-deploy --overwrite rgw create controller-03 controller-04 controller-05

执行上面的命令不成功

 

分别在每个gw节点上执行：yum install ceph-radosgw,然后再执行：ceph-deploy --overwrite rgw create controller-03 controller-04 controller-05成功。

可以看到网关已经运行。

ceph gw 默认使用自带Civetweb提供服务，在浏览器输入host:7480,可以看到正确运行

 　

3、配置各节点网关

在controller-03执行：ceph auth get client.rgw.controller-03 > /etc/ceph/ceph.client.radosgw.keyring

在controller-04执行：ceph auth get client.rgw.controller-04 > /etc/ceph/ceph.client.radosgw.keyring

在controller-05执行：ceph auth get client.rgw.controller-05 > /etc/ceph/ceph.client.radosgw.keyring

 

配置ceph.conf文件：

[client.rgw.controller-03]

rgw_frontends = "civetweb port=8899"

host = controller-03

keyring = /etc/ceph/ceph.client.radosgw.keyring

rgw dns name = xiangcloudos.com

rgw socket path = /var/run/ceph/ceph-client.rgw.controller-03.asok

rgw content length compat = true

[client.rgw.controller-04]

rgw_frontends = "civetweb port=8899"

host = controller-04

keyring = /etc/ceph/ceph.client.radosgw.keyring

rgw dns name = xiangcloudos.com

rgw socket path = /var/run/ceph/ceph-client.rgw.controller-04.asok

rgw content length compat = true

[client.rgw.controller-05]

rgw_frontends = "civetweb port=8899"

host = controller-05

keyring = /etc/ceph/ceph.client.radosgw.keyring

rgw dns name = xiangcloudos.com

rgw socket path = /var/run/ceph/ceph-client.rgw.controller-05.asok

rgw content length compat = true

 

分发配置然后重新启动每个节点的cephgw,执行 service radosgw restart.

4、nginx配置

当前在controller-01上安装nginx，配置nginx作为cephgw的前端代理。

http {

log_format main '$remote_addr - $remote_user [$time_local] "$request" '

'$status $body_bytes_sent "$http_referer" '

'"$http_user_agent" "$http_x_forwarded_for"';

access_log /var/log/nginx/access.log main;

sendfile on;

tcp_nopush on;

tcp_nodelay on;

keepalive_timeout 65;

types_hash_max_size 2048;

include /etc/nginx/mime.types;

default_type application/octet-stream;

# Load modular configuration files from the /etc/nginx/conf.d directory.

# See http://nginx.org/en/docs/ngx_core_module.html#include

# for more information.

 

include /etc/nginx/conf.d/*.conf;

upstream ceph_radosgw_zone {

　　server controller-03:8899 weight=1 max_fails=2 fail_timeout=5;

　　server controller-04:8899 weight=1 max_fails=2 fail_timeout=5;

　　server controller-05:8899 weight=1 max_fails=2 fail_timeout=5;

}

server {

listen 81 default_server;

listen [::]:81 default_server;

server_name xiangcloudos.com;

 

# Load configuration files for the default server block.

include /etc/nginx/default.d/*.conf;

if ($host ~* (.*)\.xiangcloudos\.com$)

{

set $sub_dom '$1';

rewrite ^(.*)$ /$sub_dom$1 last;

}

location /

{

proxy_pass http://ceph_radosgw_zone;

}

}

}

5、创建cephgw存储池

ceph osd pool create .rgw 128 128 replicated_sata_ruleset

ceph osd pool create .rgw.root 128 128 replicated_sata_ruleset

ceph osd pool create .rgw.control 128 128 replicated_sata_ruleset

ceph osd pool create .rgw.gc 128 128 replicated_sata_ruleset

ceph osd pool create .rgw.buckets 128 128 replicated_sata_ruleset

ceph osd pool create .rgw.buckets.index 128 128 replicated_sata_ruleset

ceph osd pool create .rgw.buckets.extra 128 128 replicated_sata_ruleset

ceph osd pool create .log 128 128 replicated_sata_ruleset

ceph osd pool create .intent-log 128 128 replicated_sata_ruleset

ceph osd pool create .usage 128 128 replicated_sata_ruleset

ceph osd pool create .users 128 128 replicated_sata_ruleset

ceph osd pool create .users.email 128 128 replicated_sata_ruleset

ceph osd pool create .users.swift 128 128 replicated_sata_ruleset

ceph osd pool create .users.uid 128 128 replicated_sata_ruleset

6、创建radosgw用户进行访问

radosgw-admin user create --uid="admin" --display-name="admin"

{

　　"user_id": "admin",

　　"display_name": "admin",

　　"email": "",

　　"suspended": 0,

　　"max_buckets": 1000,

　　"auid": 0,

　　"subusers": [],

　　"keys": [

　　{

　　　　"user": "admin",

　　　　"access_key": "8NCG9CGJ1XHAUUXSJ5IS",

　　　　"secret_key": "bC4POA9I8tmC1FgP1DCFx5YpB7m6kExzFkoaO0Hm"

　　}

　　],

　　"swift_keys": [],

　　"caps": [],

　　"op_mask": "read, write, delete",

　　"default_placement": "",

　　"placement_tags": [],

　　"bucket_quota": {

　　　　"enabled": false,

　　　　"max_size_kb": -1,

　　　　"max_objects": -1

},

"user_quota": {

　　"enabled": false,

　　"max_size_kb": -1,

　　"max_objects": -1

},

"temp_url_keys": []

}

创建swift用户：

radosgw-admin subuser create --uid=admin --subuser=admin --access=full

{

　　"user_id": "admin",

　　"display_name": "admin",

　　"email": "",

　　"suspended": 0,

　　"max_buckets": 1000,

　　"auid": 0,

"subusers": [

{

　　"id": "admin:admin",

　　"permissions": "full-control"

}

],

"keys": [

{

　　"user": "admin",

　　"access_key": "8NCG9CGJ1XHAUUXSJ5IS",

　　"secret_key": "bC4POA9I8tmC1FgP1DCFx5YpB7m6kExzFkoaO0Hm"

},

{

　　"user": "admin:admin",

　　"access_key": "FI05ZE4AEV3H8Y0BJP6D",

　　"secret_key": ""

}

],

"swift_keys": [],

"caps": [],

"op_mask": "read, write, delete",

"default_placement": "",

"placement_tags": [],

"bucket_quota": {

　　"enabled": false,

　　"max_size_kb": -1,

　　"max_objects": -1

},

"user_quota": {

　　"enabled": false,

　　"max_size_kb": -1,

　　"max_objects": -1

},

"temp_url_keys": []

}

radosgw-admin key create --subuser=admin:swift --key-type=swift --gen-secret

{

　　"user_id": "admin",

　　"display_name": "admin",

　　"email": "",

　　"suspended": 0,

　　"max_buckets": 1000,

　　"auid": 0,

　　"subusers": [

　　{

　　　　"id": "admin:admin",

　　　　"permissions": "full-control"

　　}

　　],

　　"keys": [

　　{

　　"user": "admin",

　　"access_key": "8NCG9CGJ1XHAUUXSJ5IS",

　　"secret_key": "bC4POA9I8tmC1FgP1DCFx5YpB7m6kExzFkoaO0Hm"

　　},

　　{

　　"user": "admin:admin",

　　"access_key": "FI05ZE4AEV3H8Y0BJP6D",

　　"secret_key": ""

　　}

　　],

　　"swift_keys": [

　　{

　　"user": "admin:swift",

　　"secret_key": "2BQyi6uMi1lKfq1fDcrYiA2ApcUimtYzXMuiRoC8"

　　}

　　],

　　"caps": [],

　　"op_mask": "read, write, delete",

　　"default_placement": "",

　　"placement_tags": [],

　　"bucket_quota": {

　　"enabled": false,

　　"max_size_kb": -1,

　　"max_objects": -1

　　},

　　"user_quota": {

　　"enabled": false,

　　"max_size_kb": -1,

　　"max_objects": -1

　　},

"temp_url_keys": []

}

7、测试

import boto
import boto.s3.connection

access_key = '8NCG9CGJ1XHAUUXSJ5IS'
secret_key = 'bC4POA9I8tmC1FgP1DCFx5YpB7m6kExzFkoaO0Hm'
conn = boto.connect_s3(
aws_access_key_id = access_key,
aws_secret_access_key = secret_key,
host = 'controller-01', port = 81,
is_secure=False, calling_format = boto.s3.connection.OrdinaryCallingFormat(),
)
bucket = conn.create_bucket('polly')
for bucket in conn.get_all_buckets():
print "{name}".format(name = bucket.name,)
print "{created}".format(created = bucket.creation_date,)
for key in bucket.list():
print "{name}\t{size}\t{modified}".format(name = key.name,size = key.size,modified = key.last_modified,)