mogilefs分布式文件存储

MogileFS是一个开源的分布式文件存储系统，由LiveJournal旗下的Danga Interactive公司开发。Danga团队开发了包括 Memcached、MogileFS、Perlbal 等多个知名的开源项目。目前使用MogileFS 的公司非常多，如日本排名先前的几个互联公司及国内的yupoo(又拍)、digg、豆瓣、1号店、大众点评、搜狗和安居客等，分别为所在的组织或公司管理着海量的图片。

MogileFS由3个部分组成：
(1) server：主要包括mogilefsd和mogstored两个应用程序。mogilefsd实现的是tracker，它通过数据库来保存元数据信息，包括站点domain、class、host等；mogstored是存储节点(store node)，它其实是个WebDAV服务，默认监听在7500端口，接受客户端的文件存储请求。在MogileFS安装完后，要运行mogadm工具将所有的store node注册到mogilefsd的数据库里，mogilefsd会对这些节点进行管理和监控。
(2) utils（工具集）：主要是MogileFS的一些管理工具，例如mogadm等。
(3) 客户端API：MogileFS的客户端API很多，例如Perl、PHP、Java、Python等，用这个模块可以编写客户端程序，实现文件的备份管理功能等。

hosts
a----192.168.20.6(Tnginx,mariadb)
a1-----192.168.20.2(tracker,mogstore)
a2-----192.168.20.3(mogstore)
a3-----192.168.20.4(mogstore)

系统要求：centos6或7
主机a1,a2,a3：
    .解决依赖关系
    yum -y install epel-release
    yum install perl-Sys-Syslog perl-Net-Netmask perl-IO-AIO
    .安装 mongilefs
    .
    ├── MogileFS-Server-2.46-.el6.noarch.rpm
    ├── MogileFS-Server-mogilefsd-2.46-.el6.noarch.rpm
    ├── MogileFS-Server-mogstored-2.46-.el6.noarch.rpm
    ├── MogileFS-Utils-2.19-.el6.noarch.rpm
    ├── Perlbal-1.78-.el6.noarch.rpm
    ├── Perlbal-doc-1.78-.el6.noarch.rpm
    ├── perl-Danga-Socket-1.61-.el6.rf.noarch.rpm
    ├── perl-MogileFS-Client-1.14-.el6.noarch.rpm
    ├── perl-Net-Netmask-1.9015-.el6.noarch.rpm
    └── perl-Perlbal-1.78-.el6.noarch.rpm

    yum install *.rpm
.数据库用户授权
主机a:
    grant all on mogilefs.* to mogile@'192.168.20.%' indentified by 'yunshipei';
    grant all on *.* to mogile@'192.168.20.%' indentified by 'yunshipei';
.设定数据库
主机a1:
    mogdbsetup --dbhost=192.168.20.6 --dbname=mogilefs --dbuser=mogile --dbpass=yunshipei

.修改主配置文件
主机a1:
vim mogilefsd.conf
        # Enable daemon mode to work in background and use syslog
        daemonize =
        # Where to store the pid of the daemon (must be the same in the init script)
        pidfile = /var/run/mogilefsd/mogilefsd.pid
        # Database connection information
        db_dsn = DBI:mysql:mogilefs:host=192.168.20.6
        db_user = mogile
        db_pass = yunshipei
        # IP:PORT to listen on for mogilefs client requests
        listen = 0.0.0.0:
        # Optional, if you don't define the port above.
        conf_port =
        # Number of query workers to start by default.
        query_jobs =
        # Number of delete workers to start by default.
        delete_jobs =
        # Number of replicate workers to start by default.
        replicate_jobs =
        # Number of reaper workers to start by default.
        # (you don't usually need to increase this)
        reaper_jobs =
        # Number of fsck workers to start by default.
        # (these can cause a lot of load when fsck'ing)
        #fsck_jobs =
        # Minimum amount of space to reserve in megabytes
        # default:
        # Consider setting this to be larger than the largest file you
        # would normally be uploading.
        #min_free_space =
        # Number of seconds to wait for a storage node to respond.
        # default:
        # Keep this low, so busy storage nodes are quickly ignored.
        #node_timeout =
        # Number of seconds to wait to connect to a storage node.
        # default:
        # Keep this low so overloaded nodes get skipped.
        #conn_timeout =
        # Allow replication to use the secondary node get port,
        # if you have apache or similar configured for GET's
        #repl_use_get_port =
.下创建数据存放位置，并把属主，属组设好
主机a1,a2,a3:
           mkdir /mogfs/dev1    //设备必须唯一,a2为dev2,a3为dev3
           chown -R mogilefs.mogilefs /mogfs

.编辑store配置文件
主机a1,a2,a3:
        maxconns =
        httplisten = 0.0.0.0:
        mgmtlisten = 0.0.0.0:
        docroot = /mogfs           //目录级别
        pidfile = /var/run/mogilefsd/mogstored.pid  

脚本文件需修改,详见文档最后
.mogile check(tracker,hosts)
         mogadm --trackers=192.168.20.2: check
.添加主机
        mogadm --trackers=192.168.20.2: host add 192.168.20.2 --status=alive
        mogadm --trackers=192.168.20.2: host add 192.168.20.3 --status=alive
        mogadm --trackers=192.168.20.2: host add 192.168.20.4 --status=alive
.添加设备
        mogadm --trackers=192.168.20.2: device add 192.168.20.2
        mogadm --trackers=192.168.20.2: device add 192.168.20.3
        mogadm --trackers=192.168.20.2: device add 192.168.20.4
 创建domain
        mogadm domain add  files
        mogadm domain add  images
 创建class
         mogadm class add images jpeg
          mogadm class add files plaintext --replpolicy='MultipleHosts(3)'

.查看主机，设备列表,domain
mogadm --trackers=192.168.20.2: host list
mogadm --trackers=192.168.20.2: device list
mogadm --trackers=192.168.20.2: domain list

.管理
mogadm --tracker=192.168.20.2: host mark 192.168.20.3 alive     //切换主机状态
mogupload --trackers=192.168.20.2 --domain=images --key='hello.png' --file='./QQ图片20170313153251.png' --class=png //上传文件
mogfileinfo --tracker=192.168.20.2: --domain=images --key='hello.png'  //查看keys信息
mogdelete --trackers=192.168.20.2 --domain=images --key='a.png'  //删除key
moglistkeys 查看keys 

.tengine反向代理trackers
   )tengine安装
    、解决依赖关系
    # nginx 的mogilefs模块的官方文档地址：
    http://www.grid.net.ru/nginx/mogilefs.en.html

    # yum groupinstall "Development Tools" "Server Platform Deveopment"
    # yum install openssl-devel pcre-devel

    首先添加用户nginx，实现以之运行nginx服务进程：
    # groupadd -r nginx
    # useradd -r -g nginx nginx

    接着开始编译和安装：
    # ./configure \
      --prefix=/usr \
      --sbin-path=/usr/sbin/nginx \
      --conf-path=/etc/nginx/nginx.conf \
      --error-log-path=/var/log/nginx/error.log \
      --http-log-path=/var/log/nginx/access.log \
      --pid-path=/var/run/nginx/nginx.pid  \
      --lock-path=/var/lock/nginx.lock \
      --user=nginx \
      --group=nginx \
      --with-http_ssl_module \
      --with-http_flv_module \
      --with-http_stub_status_module \
      --with-http_gzip_static_module \
      --http-client-body-temp-path=/var/tmp/nginx/client/ \
      --http-proxy-temp-path=/var/tmp/nginx/proxy/ \
      --http-fastcgi-temp-path=/var/tmp/nginx/fcgi/ \
      --http-uwsgi-temp-path=/var/tmp/nginx/uwsgi \
      --http-scgi-temp-path=/var/tmp/nginx/scgi \
      --with-pcre \
      --with-debug \
      --add-module=
    # make && make install

   )反向代理mogile_trackers
      upstream trackers{
      server 192.168.20.2:;
      server 192.168.20.3:;
      check interval= rise= fall= timeout= ;
      }
server {
        listen       ;
       server_name  localhost;

       location  /images/ {
        mogilefs_tracker trackers;
         mogilefs_domain images;
         mogilefs_methods GET;
         mogilefs_noverify on;
          mogilefs_pass {
                     proxy_pass $mogilefs_path;
                     proxy_hide_header Content-Type;
                     proxy_buffering off;
                      }
                      }
        location /status {
                          check_status;
                          access_log off;
                         }
                         }
                         }

 mogstored脚本
#!/bin/bash
# Author: MageEdu <linuxedu@foxmail.com>
# mogstored - Startup script for the MogileFS storage
#
# chkconfig: -
# description: MogileFS storage
# processname: mogstored
# config: /etc/mogilefs/mogstored.conf
# pidfile: /var/run/mogilefsd/mogstored.pid

# Source function library.
. /etc/rc.d/init.d/functions

# Path to the apachectl script, server binary, and short-form for messages.
lockfile=${LOCKFILE-/var/lock/subsys/mogstored}
pilfile=/var/run/mogilefs/mogstored.pid
RETVAL=

configfile='/etc/mogilefs/mogstored.conf'

prog=$(which mogstored)

start() {
        ulimit -n
        echo -n $"Starting mogstored"
        su - mogilefs -c "$prog -c $configfile --daemon"  &> /dev/null
        RETVAL=$?
        [ $RETVAL =  ] && success && touch ${lockfile} && echo $(pid of mogstored) >> $[pidfile] || failure
        echo
        return $RETVAL
}

stop() {
        echo -n $"Stopping mogstored"
        netstat -nlp|grep "mogstored"|grep -v grep|awk '{print $7}'|awk -F"/" '{print $1}'|xargs kill -
        RETVAL=$?
        [ $RETVAL =  ] && success && rm -f ${lockfile} || failure
        echo
}

reload() {
        echo -n $"Reloading mogstored: "
        killall mogstored -HUP
        RETVAL=$?
        [ $RETVAL =  ] && success || failure
        echo
}

case "$1" in
        start)
                start
                ;;
        stop)
                stop
                ;;
        status)
                status mogstored
                RETVAL=$?
                ;;
        restart)
                stop
                sleep