分布式文件系统的实现

1 配置数据库节点node5(172.16.21.5)

Mysql的cmake编译与安装，详细步步骤参照我的"Mysql的cmake编译与安装博文"

[root@node5 ~]# mysql

MariaDB [(none)]> create database mogilefs;

Query OK, 1 row affected (0.02 sec)

MariaDB [(none)]> use mogilefs

Database changed

#授权root用户所有权限

MariaDB [mogilefs]> grant all on *.* to 'root'@'172.16.21.5';

Query OK, 0 rows affected (0.00 sec)

#授权mogilefs的tracker连接数据库的用户

MariaDB [mogilefs]> grant all on *.* to 'moguser'@'172.16.%.%' identified by 'mogpass' with grant option;

Query OK, 0 rows affected (0.00 sec)

MariaDB [mogilefs]> flush privileges;

Query OK, 0 rows affected (0.00 sec)

2、安装配置mogilefs（三台主机: node2:172.16.21.2, node3:172.16.21.3 ,node4:172.16.21.4）

（1）安装

[root@node2 ~]# ls

anaconda-ks.cfg copyvbox.sh install.log.syslog mariadb-10.0.13.tar.gz mysql.sh

centos6.6.repo install.log mariadb-10.0.13 mogilefs percona-toolkit-2.2.4-1.noarch.rpm

[root@node2 ~]# cd mogilefs/

[root@node2 mogilefs]# ls

MogileFS-Server-2.46-2.el6.noarch.rpm Perlbal-doc-1.78-1.el6.noarch.rpm

MogileFS-Server-mogilefsd-2.46-2.el6.noarch.rpm perl-MogileFS-Client-1.14-1.el6.noarch.rpm

MogileFS-Server-mogstored-2.46-2.el6.noarch.rpm perl-Net-Netmask-1.9015-8.el6.noarch.rpm

MogileFS-Utils-2.19-1.el6.noarch.rpm perl-Perlbal-1.78-1.el6.noarch.rpm

Perlbal-1.78-1.el6.noarch.rpm

[root@node2 mogilefs]# yum -y install *.rpm perl-IO-AIO

（2）配置tracker

节点node2(172.16.21.2)

①、修改mogilefsd的配置文件

[root@node2 ~]#vim /etc/mogilefs/mogilefsd.conf

# Enable daemon mode to work in background and use syslog

daemonize = 1

# Where to store the pid of the daemon (must be the same in the init script)

pidfile = /var/run/mogilefsd/mogilefsd.pid

# Database connection information

db_dsn = DBI:mysql:mogilefs:host=172.16.21.5   #数据库

db_user = moguser        #数据库授权的用户

db_pass = mogpass        #用户密码

# IP:PORT to listen on for mogilefs client requests

listen = 172.16.21.2:7001    #监听的端口及IP

# Optional, if you don't define the port above.

conf_port = 7001

# Number of query workers to start by default.

query_jobs = 10

# Number of delete workers to start by default.

delete_jobs = 1

# Number of replicate workers to start by default.

replicate_jobs = 5

# Number of reaper workers to start by default.

# (you don't usually need to increase this)

reaper_jobs = 1

②、为mogilefsd设定数据库

[root@node2 ~]# mogdbsetup --dbhost=172.16.21.5 --dbname=mogilefs --dbrootuser=root --dbuser=moguser --dbpass=mogpass

This will attempt to setup or upgrade your MogileFS database.

It won't destroy existing data.

Run with --help for more information. Run with --yes to shut up these prompts.

Continue? [N/y]: y

[root@node2 ~]#

[root@node2 ~]# service mogilefsd start

Starting mogilefsd [ OK ]

③、查看7001端口是否成功监听

[root@node2 ~]# ss –tnlp

④、验证数据库(在节点node5:172.16.21.5)

[root@node5 ~]# mysql

MariaDB [(none)]> use mogilefs

Database changed

MariaDB [mogilefs]> show tables;

+----------------------+

| Tables_in_mogilefs |

+----------------------+

| checksum |

| class |

| device |

| domain |

| file |

| file_on |

| file_on_corrupt |

| file_to_delete |

| file_to_delete2 |

| file_to_delete_later |

| file_to_queue |

| file_to_replicate |

| fsck_log |

| host |

| server_settings |

| tempfile |

| unreachable_fids |

+----------------------+

17 rows in set (0.00 sec)

MariaDB [mogilefs]>

说明我在节点二上的设置的mogilefs已经开启

为了完整性，接下来继续在节点node2上操作：

创建一个磁盘分区：

fdisk /dev/sda

n p 3 +30G w

[root@node2 ~]# partx -a /dev/sda

磁盘初始化

[root@node2 ~]# mke2fs -t ext4 /dev/sda3

[root@node2 ~]# mkdir /mogilefs

接下来就是把磁盘/dev/sda3挂载 到 /mogilefs

[root@node2 ~]# vim /etc/fstab

/dev/sda3 /mogilefs ext4 defaults 0 0

[root@node2 ~]# mount –a

[root@node2 ~]# mount

[root@node2 ~]# chown -R mogilefs.mogilefs /mogilefs/

#########配置mogstored#########

[root@node2 ~]# vim /etc/mogilefs/mogstored.conf

maxconns = 10000

httplisten = 0.0.0.0:7500

mgmtlisten = 0.0.0.0:7501

docroot = /mogilefs

（4）查看7500端口监听状态

[root@node2 ~]# service mogstored restart

（5）添加主机

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 host add 172.16.21.2 --ip=172.16.21.2 --status=alive

###查看添加主机命令

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 host list

172.16.21.2 [1]: alive

IP: 172.16.21.2:7500

[root@node2 ~]#

（6）添加设备

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 device add 172.16.21.2 1

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 device list

172.16.21.2 [1]: alive

used(G) free(G) total(G) weight(%)

dev1: alive 0.042 27.866 27.908 100

[root@node2 ~]#

（7）添加domain

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 domain add images

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 domain add files

###列出domain###

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 domain list

domain class mindevcount replpolicy hashtype

-------------------- -------------------- ------------- ------------ -------

files default 2 MultipleHosts() NONE

images default 2 MultipleHosts() NONE

（8）添加class

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 class add images class1

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 class add images class2

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 class list

domain class mindevcount replpolicy hashtype

-------------------- -------------------- ------------- ------------ -------

files default 2 MultipleHosts() NONE

images class1 2 MultipleHosts() NONE

images class2 2 MultipleHosts() NONE

images default 2 MultipleHosts() NONE

（9）配置其它两台主机的mogilefsd和mogstored

因为这三台主机的tracker区用一个数据库，所以在安装配置时，不需再重新设定数据库，只配置相应的mogilefsd和mogstored配置文件就可以了

（10）在节点node2（172.16.21.2）查看最后的状态信息

①、列出所有主机

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 host list

172.16.21.2 [1]: alive

IP: 172.16.21.2:7500

172.16.21.3 [2]: alive

IP: 172.16.21.3:7500

172.16.21.4 [3]: alive

IP: 172.16.21.4:7500

②、列出所有设备

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 device list

172.16.21.2 [1]: alive

used(G) free(G) total(G) weight(%)

dev1: alive 0.043 27.865 27.908 100

172.16.21.3 [2]: alive

used(G) free(G) total(G) weight(%)

dev2: alive 0.043 27.865 27.908 100

172.16.21.4 [3]: alive

used(G) free(G) total(G) weight(%)

dev3: alive 0.043 27.865 27.908 100

③、列出所有domain

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 domain list

domain class mindevcount replpolicy hashtype

-------------------- -------------------- ------------- ------------ -------

files default 2 MultipleHosts() NONE

images class1 2 MultipleHosts() NONE

images class2 2 MultipleHosts() NONE

images class3 2 MultipleHosts() NONE

images class4 2 MultipleHosts() NONE

images default 2 MultipleHosts() NONE

music default 2 MultipleHosts() NONE

note default 2 MultipleHosts() NONE

（11）上传图片资源到mogilefs

[root@node2 ~]# mogupload --trackers=172.16.21.2:7001 --domain=images --key='1.jpg' --file='/root/pictures/1.jpg'

（12）使用mogfileinfo获取上传图片的访问路径

[root@node2 ~]# moglistkeys --trackers=172.16.21.2:7001 --domain=images

1.jpg

[root@node2 ~]# mogfileinfo --trackers=172.16.21.2:7001 --domain=images --key='1.jpg'

- file: 1.jpg

class: default

devcount: 2

domain: images

fid: 5

key: 1.jpg

length: 104372

- http://172.16.21.2:7500/dev1/0/000/000/0000000005.fid

- http://172.16.21.3:7500/dev2/0/000/000/0000000005.fid

五、配置nginx服务器(172.16.21.1)

1、解压nginx_mogilefs_module-1.0.4.zip模块

[root@localhost ~]# unzip nginx_mogilefs_module-1.0.4.zip

lftp 172.16.0.1:/pub/Sources/sources/nginx> get nginx-1.6.2.tar.gz

[root@localhost ~]# tar xf nginx-1.6.2.tar.gz

一、安装Nginx：

1、解决依赖关系

# yum groupinstall "Development Tools" "Server Platform Deveopment"

# yum install openssl-devel pcre-devel

2、安装

首先添加用户nginx，实现以之运行nginx服务进程：

# groupadd -r nginx

# useradd -r -g nginx nginx

3开始编译：

[root@localhost ~]#cd nginx-1.6.2

接着开始编译和安装：

# ./configure \

--prefix=/usr \

--sbin-path=/usr/sbin/nginx \

--conf-path=/etc/nginx/nginx.conf \

--error-log-path=/var/log/nginx/error.log \

--http-log-path=/var/log/nginx/access.log \

--pid-path=/var/run/nginx/nginx.pid \

--lock-path=/var/lock/nginx.lock \

--user=nginx \

--group=nginx \

--with-http_ssl_module \

--with-http_flv_module \

--with-http_stub_status_module \

--with-http_gzip_static_module \

--http-client-body-temp-path=/var/tmp/nginx/client/ \

--http-proxy-temp-path=/var/tmp/nginx/proxy/ \

--http-fastcgi-temp-path=/var/tmp/nginx/fcgi/ \

--http-uwsgi-temp-path=/var/tmp/nginx/uwsgi \

--http-scgi-temp-path=/var/tmp/nginx/scgi \

--with-pcre \

--with-debug \

--add-module=/root/nginx_mogilefs_module-1.0.4

# make && make install

4.为nginx提供SysV init脚本:

新建文件/etc/rc.d/init.d/nginx，内容如下：

#!/bin/sh

#

# nginx - this script starts and stops the nginx daemon

#

# chkconfig: - 85 15

# description: Nginx is an HTTP(S) server, HTTP(S) reverse \

# proxy and IMAP/POP3 proxy server

# processname: nginx

# config: /etc/nginx/nginx.conf

# config: /etc/sysconfig/nginx

# pidfile: /var/run/nginx.pid

# Source function library.

. /etc/rc.d/init.d/functions

# Source networking configuration.

. /etc/sysconfig/network

# Check that networking is up.

[ "$NETWORKING" = "no" ] && exit 0

nginx="/usr/sbin/nginx"

prog=$(basename $nginx)

NGINX_CONF_FILE="/etc/nginx/nginx.conf"

[ -f /etc/sysconfig/nginx ] && . /etc/sysconfig/nginx

lockfile=/var/lock/subsys/nginx

make_dirs() {

# make required directories

user=`nginx -V 2>&1 | grep "configure arguments:" | sed 's/[^*]*--user=\([^ ]*\).*/\1/g' -`

options=`$nginx -V 2>&1 | grep 'configure arguments:'`

for opt in $options; do

if [ `echo $opt | grep '.*-temp-path'` ]; then

value=`echo $opt | cut -d "=" -f 2`

if [ ! -d "$value" ]; then

# echo "creating" $value

mkdir -p $value && chown -R $user $value

fi

fi

done

}

start() {

[ -x $nginx ] || exit 5

[ -f $NGINX_CONF_FILE ] || exit 6

make_dirs

echo -n $"Starting $prog: "

daemon $nginx -c $NGINX_CONF_FILE

retval=$?

echo

[ $retval -eq 0 ] && touch $lockfile

return $retval

}

stop() {

echo -n $"Stopping $prog: "

killproc $prog -QUIT

retval=$?

echo

[ $retval -eq 0 ] && rm -f $lockfile

return $retval

}

restart() {

configtest || return $?

stop

sleep 1

start

}

reload() {

configtest || return $?

echo -n $"Reloading $prog: "

killproc $nginx -HUP

RETVAL=$?

echo

}

force_reload() {

restart

}

configtest() {

$nginx -t -c $NGINX_CONF_FILE

}

rh_status() {

status $prog

}

rh_status_q() {

rh_status >/dev/null 2>&1

}

case "$1" in

start)

rh_status_q && exit 0

$1

;;

stop)

rh_status_q || exit 0

$1

;;

restart|configtest)

$1

;;

reload)

rh_status_q || exit 7

$1

;;

force-reload)

force_reload

;;

status)

rh_status

;;

condrestart|try-restart)

rh_status_q || exit 0

;;

*)

echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload|configtest}"

exit 2

esac

5.而后为此脚本赋予执行权限：

# chmod +x /etc/rc.d/init.d/nginx

添加至服务管理列表，并让其开机自动启动：

# chkconfig --add nginx

# chkconfig nginx on

6.而后就可以启动服务并测试了：

# service nginx start

7、修改nginx配置文件

只需在location段中增加如下几行代码

location / {

mogilefs_tracker 172.16.21.3:7001;

mogilefs_domain images;

mogilefs_methods get;

mogilefs_pass {

proxy_pass $mogilefs_path;

proxy_hide_header Content-Type;

proxy_buffering off;

}

}

使用curl上传文件。

curl -X PUT -T '/etc/passwd' http://172.16.100.15/upload/passwd.html