centos7.1中安装calamari

ceph集群安装

确定版本

[root@node1050 calamari]# ceph --version
ceph version 10.2.9 (2ee413f77150c0f375ff6f10edd6c8f9c7d060d0)
[root@node1050 calamari]#

配置无密访问

准备脚本

[root@node1050 17-ceph-install]# cat reinstall.sh
ceph-deploy purge node1017 node1018 node1019
ceph-deploy purgedata node1017 node1018 node1019
ceph-deploy forgetkeys
rm ceph.*
cp -a /root/yum.repos_bak/* /etc/yum.repos.d/
scp -r /etc/yum.repos.d/* 192.168.80.18:/etc/yum.repos.d/
scp -r /etc/yum.repos.d/* 192.168.80.19:/etc/yum.repos.d/
ssh 192.168.80.17 "yum -y install epel-release ceph ceph-radosgw" &
ssh 192.168.80.18 "yum -y install epel-release ceph ceph-radosgw" &
ssh 192.168.80.19 "yum -y install epel-release ceph ceph-radosgw" &
wait
ceph-deploy new node1017echo public network = 10.10.10.0/24 >> ceph.conf
echo osd pool default size = 2 >> ceph.conf
echo osd pool default size = 2 >> ceph.conf
echo osd pool default pg num = 64 >> ceph.conf
echo osd pool default pgp num = 64 >> ceph.conf
wait
ceph-deploy install node1017 node1018 node1019
ceph-deploy mon create-initial
ceph-deploy admin node1017 node1018 node1019
sh zap.sh
sh zap2.sh
sh prepare.sh
sh prepare2.sh
sh active.sh
sh active2.sh

[root@node1050 17-ceph-install]# cat active.sh
i=1
#for dev in "sdd" "sde" "sdf" "sdg" "sdh" "sdi"
#do
#   ceph-deploy osd activate  node1017:/dev/$dev"1":/dev/sdb$i
#   ceph-deploy osd activate  node1018:/dev/$dev"1":/dev/sdb$i
#   ceph-deploy osd activate  node1019:/dev/$dev"1":/dev/sdb$i
#   i=$(($i+1))
#done
ceph-deploy osd activate  node1017:/dev/sdb1
ceph-deploy osd activate  node1018:/dev/sdc1
ceph-deploy osd activate  node1019:/dev/sdc1

[root@node1050 17-ceph-install]# cat active2.sh
i=1for dev in "sdd" "sde" "sdf" "sdg" "sdh" "sdi"  "sdj" "sdk" "sdl" "sdm" "sdn" "sdo" "sdp" "sdq" "sdr" "sds" "sdt" "sdu" "sdv" "sdw" "sdx" "sdy"
do
ceph-deploy osd activate  node1017:"/dev/$dev"1
ceph-deploy osd activate  node1018:"/dev/$dev"1
ceph-deploy osd activate  node1019:"/dev/$dev"1
done

[root@node1050 17-ceph-install]# cat create_fs.sh
ceph osd pool create cephfs_data 128
ceph osd pool create cephfs_metadata 64
ceph fs new ceph_fs cephfs_metadata cephfs_data
cat cat ceph.client.admin.keyring > /etc/admin.sec

[root@node1050 17-ceph-install]# cat zap.sh
for dev in "sdb" "sdc"
doecho -------------------------$devceph-deploy disk zap node1017:$devceph-deploy disk zap node1018:$devceph-deploy disk zap node1019:$dev
done

[root@node1050 17-ceph-install]# cat zap2.sh
for dev in "sdd" "sde" "sdf" "sdg" "sdh" "sdi"  "sdj" "sdk" "sdl" "sdm" "sdn" "sdo" "sdp" "sdq" "sdr" "sds" "sdt" "sdu" "sdv" "sdw" "sdx" "sdy"
doecho -------------------------$devceph-deploy disk zap node1017:$devceph-deploy disk zap node1018:$devceph-deploy disk zap node1019:$dev
done

[root@node1050 17-ceph-install]# cat prepare.sh ceph-deploy osd prepare node1017:sdbceph-deploy osd prepare node1018:sdcceph-deploy osd prepare node1019:sdc

[root@node1050 17-ceph-install]# cat prepare2.sh for dev in "sdd" "sde" "sdf" "sdg" "sdh" "sdi"  "sdj" "sdk" "sdl" "sdm" "sdn" "sdo" "sdp" "sdq" "sdr" "sds" "sdt" "sdu" "sdv" "sdw" "sdx" "sdy"
do
ceph-deploy osd prepare node1017:$dev
ceph-deploy osd prepare node1018:$dev
ceph-deploy osd prepare node1019:$dev
done

calamari服务端安装

git clone https://github.com/ceph/calamari.git
yum install gcc gcc-c++ postgresql-libs python-virtualenv postgresql-devel httpd checkpolicy selinux-policy-devel selinux-policy-doc selinux-policy-mls rpm-build redhat-lsb-core cairo-devel.x86_64
cd calamari/
git checkout v1.3.1.1
./build-rpm.sh
cd ../rpmbuild/RPMS/x86_64/
yum install calamari-server-1.5.2-13_g768f37d.el7.centos.x86_64.rpm
calamari-ctl initialize
calamari-ctl add_user --password 111111 --email admin@163.com root
calamari-ctl assign_role --role superuser root

安装过程中遇到的问题

编译过程提示/etc/calamari/calamari.conf is a directory

修改如下内容

[root@node1019 calamari]# git diff
diff --git a/Makefile b/Makefile
index 5e51859..ae1526a 100644
--- a/Makefile
+++ b/Makefile
@@ -166,9 +166,8 @@ install-conf: $(CONFFILES)@$(INSTALL) -d $(DESTDIR)/var/lib/graphite/whisper@$(INSTALL) -d $(DESTDIR)/var/lib/calamari@$(INSTALL) -d $(DESTDIR)/var/lib/cthulhu
-@$(INSTALL) -d $(DESTDIR)/etc/calamari
-       @$(INSTALL) -D -m 0644 conf/calamari/$(FLAVOR)/calamari.conf \
+       @$(INSTALL) -D -m 0644 conf/calamari/el6/calamari.conf \$(DESTDIR)/etc/calamari/calamari.conf@$(INSTALL) -D -m 0644 conf/alembic.ini \$(DESTDIR)/etc/calamari/alembic.ini

编译过程中提示语法错误

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-R57qBkSe-1592052236921)(/images/calamari_build_err.png)]
这是因为使用的gevent1.2.2版本中的_socket3.py文件中使用了python3的语法

diff --git a/requirements/2.7/requirements.production.txt b/requirements/2.7/requirements.production.txt
index 41c214c..8e4acb4 100644
--- a/requirements/2.7/requirements.production.txt
+++ b/requirements/2.7/requirements.production.txt
@@ -12,7 +12,7 @@ wsgiref==0.1.2sqlalchemy==0.8.3alembic==0.6.3cython==0.19.2
-gevent>=1.0
+gevent==1.0greenlet>=0.3.2psycogreen==1.0psycopg2==2.5.2
diff --git a/requirements/2.7/requirements.txt b/requirements/2.7/requirements.txt
index 146b772..f02476d 100644
--- a/requirements/2.7/requirements.txt
+++ b/requirements/2.7/requirements.txt
@@ -25,7 +25,7 @@ psutil==2.0.0# Gevent 1.0.x requires cython but doesn't do a good job of depending on itcython==0.19.2
-gevent>=1.0
+gevent==1.0greenlet>=0.3.2

/var/log/calamari/calamari.log中有错误信息

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-4dRoAPyj-1592052236923)(/images/calamari_log_err.png)]
需要安装cairo包yum install cairo-devel.x86_64

calamari客户端安装

1 下载源码

git clone https://github.com/ceph/romana

2 安装依赖包

yum install npm ruby ruby-devel rubygems libpng-devel libpng-devel pngquant optipng libjpeg

3 修改npm仓库地址并下载依赖包

npm config set registry https://registry.npm.taobao.org
npm config get registry
npm install -g bower grunt grunt-cli grunt-contrib-compass
npm install -g n
npm install --global jpegtran-bin
npm install --global gifsiclegem sources --remove https://rubygems.org/
gem sources --remove gems.ruby-china.org
gem sources -a https://ruby.taobao.orggem update --system
gem install compasscd romana/
git checkout v1.2.2
[root@calamari ~]# vim romana/manage/package.json
"grunt-connect-proxy": "~0.2.0"
cp /usr/local/bin/compass /sbin/

4 修改grunt-contrib-imagemin版本号

[root@node1050 romana]# grep "grunt-contrib-imagemin" -R *
admin/package.json:    "grunt-contrib-imagemin": "~0.1.4",
dashboard/package.json:    "grunt-contrib-imagemin": "0.1.4",
login/package.json:    "grunt-contrib-imagemin": "~0.1.3",
manage/package.json:    "grunt-contrib-imagemin": "~0.3.0",
[root@node1050 romana]# vim admin/package.json
[root@node1050 romana]# vim dashboard/package.json
[root@node1050 romana]# vim login/package.json
[root@node1050 romana]# vim manage/package.json
[root@node1050 romana]# grep "grunt-contrib-imagemin" -R *
admin/package.json:    "grunt-contrib-imagemin": "0.4.1",
dashboard/package.json:    "grunt-contrib-imagemin": "0.4.1",
login/package.json:    "grunt-contrib-imagemin": "0.4.1",
manage/package.json:    "grunt-contrib-imagemin": "0.4.1",

5 修改yo的版本号为2.0

[root@node1050 romana]# grep "\"yo\":" * -R
admin/package.json:    "yo": "2.0.0",
admin/node_modules/yo/package.json:    "yo": "lib/cli.js",
admin/node_modules/yo/package.json:    "yo": [
admin/node_modules/generator-angular/package.json:    "yo": ">=1.0.0-rc.1.1"
admin/node_modules/tabtab/examples/yo-complete/package.json:    "yo": [
dashboard/node_modules/generator-backbone/package.json:    "yo": ">=1.0.0-rc.1.1"
login/package.json:    "yo": "2.0.0",
login/node_modules/yo/package.json:    "yo": "lib/cli.js",
login/node_modules/yo/package.json:    "yo": [
login/node_modules/tabtab/examples/yo-complete/package.json:    "yo": [
login/node_modules/generator-backbone/package.json:    "yo": ">=1.0.0-rc.1.1"

6 执行编译

PHANTOMJS_CDNURL=http://npm.taobao.org/dist/phantomjs make build-real

7 编译完成后打包

[root@calamari romana]# make dist
making dist tarball in ../romana_1.2.2.tar.gz for d in manage admin login dashboard; do \ echo $d; \ (cd $d; \ npm install --silent; \ grunt --no-color saveRevision) \ done --------something more---------- Done, without errors. 185667 blocks tar file made in ../romana_1.2.2.tar.gzcp ../calamari-clients_1.2.2.tar.gz ./

8 修改makefile

[root@node1050 romana]# git diff Makefile
diff --git a/Makefile b/Makefile
index 420a94a..add506e 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ SRC := $(shell pwd)# set these only if not set with ?=VERSION ?= $(shell $(SRC)/get-versions.sh VERSION)REVISION ?= $(shell $(SRC)/get-versions.sh REVISION)
-BUILD_PRODUCT_TGZ=$(SRC)/calamari-clients-build-output.tar.gz
+BUILD_PRODUCT_TGZ=$(SRC)/calamari-clients_1.2.2.tar.gzRPM_REVISION ?= $(shell $(SRC)/get-versions.sh -r REVISION)RPMBUILD=$(SRC)/../rpmbuild

9 修改client.spec

[root@node1050 romana]# git diff clients.spec
diff --git a/clients.spec b/clients.spec
index 6cea00b..b53fd79 100644
--- a/clients.spec
+++ b/clients.spec
@@ -31,6 +31,15 @@ echo "install"mkdir -p %{buildroot}cd %{buildroot}tar xfz %{tarname}
+cd %{name}-%{version}
+for dir in manage admin login dashboard
+do
+mkdir -p ../opt/calamari/webapp/content/"$dir"
+cp -pr "$dir"/dist/* ../opt/calamari/webapp/content/"$dir"/
+done
+cd ../
+rm -rf /tmp/%{name}-%{version}
+mv %{name}-%{version} /tmp/%cleanecho "clean"

10 安装rpm

cd /root/rpmbuild/RPMS/x86_64/
yum install calamari-clients-1.2.2-0.el7.centos.x86_64.rpm

安装过程中遇到的问题

yum npm安装之后再执行npm install -g npm会把npm删掉

npm WARN grunt-contrib-compass@0.6.0 requires a peer of grunt@~0.4.0 but none was installed

在执行gem update --system之前不光要删除gem sources --remove https://rubygems.org还要删除https://gems.ruby-china.org

calamari-client 执行make build-real过程中会提示没有pngquant，需要手动安装

yum install npm ruby ruby-devel rubygems libpng-devel libpng-devel pngquant optipng libjpeg
yum install optipng
yum install libjpeg
npm install --global jpegtran-bin
npm install --global gifsicle

在执行npm时有错误提示：

npm: relocation error: npm: symbol SSL_set_cert_cb, version libssl.so.10 not defined in file libssl.so.10 with link time reference

此时需要升级openssl，执行：

yum update openssl

需要修改下面的文件中

login/node_modules/phantomjs/install.js
manage/node_modules/phantomjs/install.js
dashboard/node_modules/phantomjs/install.js
替换phantomjs-1.9.8-linux-x86_64.tar.bz2下载路径

//var DEFAULT_CDN = 'https://github.com/Medium/phantomjs/releases/download/v1.9.19'
var DEFAULT_CDN = 'http://npm.taobao.org/mirrors/phantomjs'

diamond安装

1 下载源码

git clone https://github.com/ceph/Diamond

2 编译安装

cd Diamond
git checkout origin/calamari
make rpm
yum install ./dist/diamond-3.4.67-0.noarch.rpm

3 修改配置diamond.conf

cp /etc/diamond/diamond.conf.example /etc/diamond/diamond.conf
[root@node1050 Diamond]# diff /etc/diamond/diamond.conf.example /etc/diamond/diamond.conf
59c59
< host = graphite
---
> host = node1050
74c74
< host = graphite
---
> host = node1050

4 启动diamond

/etc/init.d/diamond restart

5 修改salt-minon配置并启动

[root@node1050 Diamond]# mkdir /etc/salt/minion.d
[root@node1050 Diamond]# echo "master: node1050" > /etc/salt/minion.d/calamari.conf
[root@node1050 Diamond]# systemctl restart salt-minion

7 其他节点啊装minion

将diamond-3.4.67-0.noarch.rpm拷贝到其他ceph节点并执行安装，修改配置及启动
在其他ceph节点安装salt-minion并修改配置启动

8 系统启动

在calamari-server节点执行

salt-key -L
salt-key -Achmod 777 -R /var/log/calamari/*
systemctl restart supervisord
systemctl restart httpd

重新安装slat

http://rpm.pbone.net/ 中查找（rpmfind.net中已经没有该版本的包）
salt-2014.1.13-1.fc21.noarch.rpm
salt-master-2014.1.13-1.fc21.noarch.rpm
salt-minion-2014.1.13-1.fc21.noarch.rpm

遇到的问题

界面未显示登录界面，显示内部错误

查看/var/log/calamari/calamari.log无内容显示，查看/var/log/calamari/cthulhu.log有如下错误

    conn = bind.contextual_connect()File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1661, in contextual_connectself.pool.connect(),File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/pool.py", line 272, in connectreturn _ConnectionFairy(self).checkout()File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/pool.py", line 431, in __init__rec = self._connection_record = pool._do_get()File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/pool.py", line 788, in _do_getcon = self._create_connection()File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/pool.py", line 225, in _create_connectionreturn _ConnectionRecord(self)File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/pool.py", line 318, in __init__self.connection = self.__connect()File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/pool.py", line 379, in __connectconnection = self.__pool._creator()File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/engine/strategies.py", line 80, in connectreturn dialect.connect(*cargs, **cparams)File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 283, in connectreturn self.dbapi.connect(*cargs, **cparams)File "/opt/calamari/venv/lib/python2.7/site-packages/psycopg2/__init__.py", line 164, in connectconn = _connect(dsn, connection_factory=connection_factory, async=async)File "/opt/calamari/venv/lib/python2.7/site-packages/psycogreen/gevent.py", line 29, in gevent_wait_callbackstate = conn.poll()
OperationalError: (OperationalError) could not connect to server: Connection refusedIs the server running on host "localhost" (::1) and acceptingTCP/IP connections on port 5432?
could not connect to server: Connection refusedIs the server running on host "localhost" (127.0.0.1) and acceptingTCP/IP connections on port 5432?None None
2017-10-23 10:49:32,094 - ERROR - cthulhu Recovery failed

执行setenforce 0关闭selinux

界面一直显示为初次安装

发现调用cluster接口的时候返回的cluster信息为空，所以前端认为没有集群信息

执行salt ‘*’ ceph.get_heartbeats出现下面的错误

修改代码/opt/calamari/salt/salt/_modules/ceph.py

修改上面的代码之后，再次调用接口可以获取正确的cluster信息

界面无法正常显示

/var/log/calamari/calamari.log中出现如下错误

Traceback (most recent call last):File "/opt/calamari/venv/lib/python2.7/site-packages/django/core/handlers/base.py", line 115, in get_responseresponse = callback(request, *callback_args, **callback_kwargs)File "/opt/calamari/venv/lib/python2.7/site-packages/rest_framework/viewsets.py", line 78, in viewreturn self.dispatch(request, *args, **kwargs)File "/opt/calamari/venv/lib/python2.7/site-packages/calamari_rest_api-0.1-py2.7.egg/calamari_rest/views/rpc_view.py", line 92, in dispatchreturn super(RPCViewSet, self).dispatch(request, *args, **kwargs)File "/opt/calamari/venv/lib/python2.7/site-packages/django/views/decorators/csrf.py", line 77, in wrapped_viewreturn view_func(*args, **kwargs)File "/opt/calamari/venv/lib/python2.7/site-packages/rest_framework/views.py", line 399, in dispatchresponse = self.handle_exception(exc)File "/opt/calamari/venv/lib/python2.7/site-packages/calamari_rest_api-0.1-py2.7.egg/calamari_rest/views/rpc_view.py", line 109, in handle_exceptionreturn super(RPCViewSet, self).handle_exception(exc)File "/opt/calamari/venv/lib/python2.7/site-packages/rest_framework/views.py", line 396, in dispatchresponse = handler(request, *args, **kwargs)File "/opt/calamari/venv/lib/python2.7/site-packages/calamari_rest_api-0.1-py2.7.egg/calamari_rest/views/v1.py", line 316, in getosd_data = osd_data.get()File "/opt/calamari/venv/lib/python2.7/site-packages/calamari_rest_api-0.1-py2.7.egg/calamari_rest/views/v1.py", line 167, in generate'mds': cls._calculate_mds_counters(mds_map),File "/opt/calamari/venv/lib/python2.7/site-packages/calamari_rest_api-0.1-py2.7.egg/calamari_rest/views/v1.py", line 295, in _calculate_mds_counters#up = len(mds_map['up'])
TypeError: 'NoneType' object has no attribute '__getitem__'

修改代码/opt/calamari/venv/lib/python2.7/site-packages/calamari_rest_api-0.1-py2.7.egg/calamari_rest/views/v1.py
[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-o3J8YKEG-1592052236927)(/images/v1.png)]
/var/log/calamari/cthulhu.log中有如下错误

2017-10-23 19:37:00,644 - ERROR - cthulhu Exception handling message with tag ceph/cluster/354c211d-e8d1-4dcd-8635-f0fb7ec51826
Traceback (most recent call last):File "/opt/calamari/venv/lib/python2.7/site-packages/calamari_cthulhu-0.1-py2.7.egg/cthulhu/manager/cluster_monitor.py", line 244, in _runself.on_heartbeat(data['id'], data['data'])File "/opt/calamari/venv/lib/python2.7/site-packages/calamari_cthulhu-0.1-py2.7.egg/cthulhu/gevent_util.py", line 35, in wrappedreturn func(*args, **kwargs)File "/opt/calamari/venv/lib/python2.7/site-packages/calamari_cthulhu-0.1-py2.7.egg/cthulhu/manager/cluster_monitor.py", line 348, in on_heartbeatcluster_data['versions'][sync_type.str])
KeyError: 'mds_map'

修改代码
/opt/calamari/venv/lib/python2.7/site-packages/calamari_cthulhu-0.1-py2.7.egg/cthulhu/manager/cluster_monitor.py

/var/log/diamond/diamond.log中有错误

出现断言错误
修改代码/usr/share/diamond/collectors/ceph/ceph.py

参考资料

《ceph分布式存储实战》
https://zphj1987.gitbooks.io/calamaribook/content/second/an_zhuang_calamari_-_server.html
http://blog.csdn.net/lzw06061139/article/details/51174588

声明：转载本博文章须在文章明显处注明作者及附上原文链接，便于读者找到原文的更新版。