ceph-对象存储

ceph对象存储

作为文件系统的磁盘，操作系统不能直接访问对象存储。相反，它只能通过应用程序级别的API访问。ceph是一种分布式对象存储系统，通过ceph对象网关提供对象存储接口，也称为RADOS网关（RGW）接口，它构建在ceph RADOS层之上。RGW使用librgw（RADOS Gateway library）和librados，允许应用程序与ceph对象存储建立连接。RGW为应用程序提供了一个RESTful S3/swift兼容的接口，用于在ceph集群中以对象的形式存储数据。ceph还支持多租户对象存储，可以通过RESTful API访问。此外，RGW还支持ceph管理API，可以使用本机API调用来管理ceph存储集群。

librados软件库非常灵活，允许用户应用程序通过C、C++、java、python和php绑定直接访问ceph存储集群。ceph对象存储还具有多站点功能，即灾难恢复提供解决方案。

部署对象存储

安装ceph-radosgw

[ceph-admin@ceph-node1 my-cluster]$ sudo yum install ceph-radosgw

部署rgw

[ceph-admin@ceph-node1 my-cluster]$ ceph-deploy rgw create ceph-node1 ceph-node2 ceph-node3

[ceph-admin@ceph-node1 my-cluster]$ sudo netstat -tnlp |grep

tcp               0.0.0.0:            0.0.0.0:*               LISTEN      /radosgw

如果要修改为80端口，可修改配置文件重启

vim /etc/ceph/ceph.conf

[client.rgw.ceph-node1]

rgw_frontends = "civetweb port=80"

sudo systemctl restart ceph-radosgw@rgw.ceph-node1.service

创建池

[ceph-admin@ceph-node1 my-cluster]$ wget https://raw.githubusercontent.com/aishangwei/ceph-demo/master/ceph-deploy/rgw/pool

[ceph-admin@ceph-node1 my-cluster]$ wget  https://raw.githubusercontent.com/aishangwei/ceph-demo/master/ceph-deploy/rgw/create_pool.sh

[ceph-admin@ceph-node1 my-cluster]$ cat create_pool.sh

#!/bin/bash

PG_NUM=

PGP_NUM=

SIZE=

for i in `cat /home/ceph-admin/my-cluster/pool`

        do

        ceph osd pool create $i $PG_NUM

        ceph osd pool set $i size $SIZE

        done

for i in `cat /home/ceph-admin/my-cluster/pool`

        do

        ceph osd pool set $i pgp_num $PGP_NUM

        done

[ceph-admin@ceph-node1 my-cluster]$ chmod +x create_pool.sh

[ceph-admin@ceph-node1 my-cluster]$ ./create_pool.sh

测试是否能访问ceph集群

[ceph-admin@ceph-node1 my-cluster]$ sudo ls -l /var/lib/ceph/

total

drwxr-x---  ceph ceph   Jan  : bootstrap-mds

drwxr-x---  ceph ceph  Feb  : bootstrap-mgr

drwxr-x---  ceph ceph  Feb  : bootstrap-osd

drwxr-x---  ceph ceph   Jan  : bootstrap-rbd

drwxr-x---  ceph ceph  Feb  : bootstrap-rgw

drwxr-x---  ceph ceph   Jan  : mds

drwxr-x---  ceph ceph  Feb  : mgr

drwxr-x---  ceph ceph  Feb  : mon

drwxr-x---  ceph ceph  Feb  : osd

drwxr-xr-x  root root  Feb  : radosgw

[ceph-admin@ceph-node1 my-cluster]$ sudo cp /var/lib/ceph/radosgw/ceph-rgw.ceph-node1/keyring ./

[ceph-admin@ceph-node1 my-cluster]$ ceph -s -k keyring --name client.rgw.ceph-node1

  cluster:

    id:     cde2c9f7-009e-4bb4-a206-95afa4c43495

    health: HEALTH_OK

  services:

    mon:  daemons, quorum ceph-node1,ceph-node2,ceph-node3

    mgr: ceph-node1(active), standbys: ceph-node2, ceph-node3

    osd:  osds:  up,  in

    rgw:  daemons active

  data:

    pools:    pools,  pgs

    objects:  objects, 114MiB

    usage:   .45GiB used, 171GiB / 180GiB avail

    pgs:      active+clean

  io:

    client:   0B/s rd, 0op/s rd, 0op/s wr

使用S3 API访问ceph对象存储

创建radosgw用户

[ceph-admin@ceph-node1 my-cluster]$ radosgw-admin user create --uid=radosgw --display-name="radosgw"

{

    "user_id": "radosgw",

    "display_name": "radosgw",

    "email": "",

    "suspended": ,

    "max_buckets": ,

    "auid": ,

    "subusers": [],

    "keys": [

        {

            "user": "radosgw",

            "access_key": "DKOORDOMS6YHR2OW5M23",

            "secret_key": "OOBNCO0d03oiBaLCtYePPQ7gIeUR2Y7UuB24pBW4"

        }

    ],

    "swift_keys": [],

    "caps": [],

    "op_mask": "read, write, delete",

    "default_placement": "",

    "placement_tags": [],

    "bucket_quota": {

        "enabled": false,

        "check_on_raw": false,

        "max_size": -,

        "max_size_kb": ,

        "max_objects": -

    },

    "user_quota": {

        "enabled": false,

        "check_on_raw": false,

        "max_size": -,

        "max_size_kb": ,

        "max_objects": -

    },

    "temp_url_keys": [],

    "type": "rgw"

}

安装s3cmd客户端

[root@localhost ~]# yum install -y s3cmd

[root@localhost ~]# s3cmd --configure

Enter new values or accept defaults in brackets with Enter.

Refer to user manual for detailed description of all options.

Access key and Secret key are your identifiers for Amazon S3. Leave them empty for using the env variables.

Access Key: DKOORDOMS6YHR2OW5M23

Secret Key: OOBNCO0d03oiBaLCtYePPQ7gIeUR2Y7UuB24pBW4

Default Region [US]: ZH

Use "s3.amazonaws.com" for S3 Endpoint and not modify it to the target Amazon S3.

S3 Endpoint [s3.amazonaws.com]: 

Use "%(bucket)s.s3.amazonaws.com" to the target Amazon S3. "%(bucket)s" and "%(location)s" vars can be used

if the target S3 system supports dns based buckets.

DNS-style bucket+hostname:port template for accessing a bucket [%(bucket)s.s3.amazonaws.com]: 

Encryption password is used to protect your files from reading

by unauthorized persons while in transfer to S3

Encryption password:

Path to GPG program [/usr/bin/gpg]: 

When using secure HTTPS protocol all communication with Amazon S3

servers is protected from 3rd party eavesdropping. This method is

slower than plain HTTP, and can only be proxied with Python 2.7 or newer

Use HTTPS protocol [Yes]: no

On some networks all internet access must go through a HTTP proxy.

Try setting it here if you can't connect to S3 directly

HTTP Proxy server name: 

New settings:

  Access Key: DKOORDOMS6YHR2OW5M23

  Secret Key: OOBNCO0d03oiBaLCtYePPQ7gIeUR2Y7UuB24pBW4

  Default Region: ZH

  S3 Endpoint: s3.amazonaws.com

  DNS-style bucket+hostname:port template for accessing a bucket: %(bucket)s.s3.amazonaws.com

  Encryption password:

  Path to GPG program: /usr/bin/gpg

  Use HTTPS protocol: False

  HTTP Proxy server name:

  HTTP Proxy server port: 

Test access with supplied credentials? [Y/n] n

Save settings? [y/N] y

Configuration saved to '/root/.s3cfg'

编辑s3配置文件

[root@localhost ~]# cat .s3cfg

[default]

access_key = DKOORDOMS6YHR2OW5M23

access_token =

add_encoding_exts =

add_headers =

bucket_location = US

ca_certs_file =

cache_file =

check_ssl_certificate = True

check_ssl_hostname = True

cloudfront_host = cloudfront.amazonaws.com

content_disposition =

content_type =

default_mime_type = binary/octet-stream

delay_updates = False

delete_after = False

delete_after_fetch = False

delete_removed = False

dry_run = False

enable_multipart = True

encrypt = False

expiry_date =

expiry_days =

expiry_prefix =

follow_symlinks = False

force = False

get_continue = False

gpg_command = /usr/bin/gpg

gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s

gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s

gpg_passphrase =

guess_mime_type = True

host_base = ceph-node1:

host_bucket = %(bucket).ceph-node1:

human_readable_sizes = False

invalidate_default_index_on_cf = False

invalidate_default_index_root_on_cf = True

invalidate_on_cf = False

kms_key =

limit = -

limitrate =

list_md5 = False

log_target_prefix =

long_listing = False

max_delete = -

mime_type =

multipart_chunk_size_mb =

multipart_max_chunks =

preserve_attrs = True

progress_meter = True

proxy_host =

proxy_port =

put_continue = False

recursive = False

recv_chunk =

reduced_redundancy = False

requester_pays = False

restore_days =

restore_priority = Standard

secret_key = OOBNCO0d03oiBaLCtYePPQ7gIeUR2Y7UuB24pBW4

send_chunk =

server_side_encryption = False

signature_v2 = False

signurl_use_https = False

simpledb_host = sdb.amazonaws.com

skip_existing = False

socket_timeout =

stats = False

stop_on_error = False

storage_class =

throttle_max =

upload_id =

urlencoding_mode = normal

use_http_expect = False

use_https = False

use_mime_magic = True

verbosity = WARNING

website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/

website_error =

website_index = index.html

创建桶并放入文件

[root@localhost ~]# s3cmd mb s3://first-bucket

Bucket 's3://first-bucket/' created

[root@localhost ~]# s3cmd ls

-- :  s3://first-bucket

[root@localhost ~]# s3cmd put /etc/hosts s3://first-bucket

upload: '/etc/hosts' -> 's3://first-bucket/hosts'  [ of ]

  of    % in    1s   175.80 B/s  done

[root@localhost ~]# s3cmd ls s3://first-bucket

-- :          s3://first-bucket/hosts

使用Swift API访问ceph对象存储

创建swift api子用户

[ceph-admin@ceph-node1 my-cluster]$ radosgw-admin subuser create --uid=radosgw --subuser=radosgw:swift --access=full
{
"user_id": "radosgw",
"display_name": "radosgw",
"email": "",
"suspended": 0,
"max_buckets": 1000,
"auid": 0,
"subusers": [
{
"id": "radosgw:swift",
"permissions": "full-control"
}
],
"keys": [
{
"user": "radosgw",
"access_key": "DKOORDOMS6YHR2OW5M23",
"secret_key": "OOBNCO0d03oiBaLCtYePPQ7gIeUR2Y7UuB24pBW4"
}
],
"swift_keys": [
{
"user": "radosgw:swift",
"secret_key": "bAL11KzCYE1GThPWY70tUo6dVIhvuIbSFEBP06yD"
}
],
"caps": [],
"op_mask": "read, write, delete",
"default_placement": "",
"placement_tags": [],
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"temp_url_keys": [],
"type": "rgw"
}

安装swift api客户端

[root@localhost ~]# yum install python-pip -y

[root@localhost ~]# pip install --upgrade python-swiftclient

测试

[root@localhost ~]# swift -A http://ceph-node1:7480/auth/1.0 -U radosgw:swift -K bAL11KzCYE1GThPWY70tUo6dVIhvuIbSFEBP06yD list

first-bucket

[root@localhost ~]# swift -A http://ceph-node1:7480/auth/1.0 -U radosgw:swift -K bAL11KzCYE1GThPWY70tUo6dVIhvuIbSFEBP06yD list

[root@localhost ~]# swift -A http://ceph-node1:7480/auth/1.0 -U radosgw:swift -K bAL11KzCYE1GThPWY70tUo6dVIhvuIbSFEBP06yD post second-bucket

[root@localhost ~]# swift -A http://ceph-node1:7480/auth/1.0 -U radosgw:swift -K bAL11KzCYE1GThPWY70tUo6dVIhvuIbSFEBP06yD list

first-bucket

second-bucket

[root@localhost ~]# s3cmd ls

-- :  s3://first-bucket

-- :  s3://second-bucket