创建存储池#若少于5个OSD， 设置pg_num为128

#5~10个OSD，设置pg_num为512

#10~50个OSD，设置pg_num为4096

ceph osd pool create ceph-demo 64 64

#查看存储池

ceph osd lspools查看存储池参数[root@node-1 ~]# ceph osd pool get ceph-demo pg_num

pg_num: 64

[root@node-1 ~]# ceph osd pool get ceph-demo pgp_num

pgp_num: 64

[root@node-1 ~]# ceph osd pool get ceph-demo size

size: 3修改存储池参数ceph osd pool get ceph-demo pg_num 32

ceph osd pool get ceph-demo pgp_num 32创建镜像rbd create -p ceph-demo --image rbd-demo.img --size 10G

或者 rbd create ceph-demo/rbd-demo-1.img --size 10G

[root@node-1 ~]# rbd -p ceph-demo ls

rbd-demo.img

rbd-demo-1.img查看镜像详细信息root@node-1 ~]# rbd info ceph-demo/rbd-demo.img

rbd image 'rbd-demo.img':

size 10 GiB in 2560 objects

order 22 (4 MiB objects)

snapshot_count: 0

id: 1134749f26f5

block_name_prefix: rbd_data.1134749f26f5

format: 2

features: layering, exclusive-lock, object-map, fast-diff, deep-flatten

op_features:

flags:

create_timestamp: Tue Sep 22 15:39:03 2020

access_timestamp: Tue Sep 22 15:39:03 2020

modify_timestamp: Tue Sep 22 15:39:03 2020删除镜像rbd rm ceph-demo/rbd-demo-1.img

[root@node-1 ~]# rbd -p ceph-demo ls

rbd-demo.img块设备映射rbd map ceph-demo/rbd-demo.img

rbd: sysfs write failed

RBD image feature set mismatch. Try disabling features unsupported by the kernel with "rbd feature disable".

In some cases useful info is found in syslog - try "dmesg | tail".

rbd: map failed: (6) No such device or address

上面有报错是因为有些特性centos7的内核不支持，所以需要关了[root@node-1 ~]# rbd feature disable ceph-demo/rbd-demo.img deep-flatten

[root@node-1 ~]# rbd feature disable ceph-demo/rbd-demo.img fast-diff

[root@node-1 ~]# rbd feature disable ceph-demo/rbd-demo.img object-map

rbd: failed to update image features: (22) Invalid argument

2020-09-25 14:03:56.251 7efe8d341c80 -1 librbd::Operations: one or more requested features are already disabled

[root@node-1 ~]# rbd feature disable ceph-demo/rbd-demo.img exclusive-lock再次映射[root@node-1 ~]# rbd map ceph-demo/rbd-demo.img

/dev/rbd0

#查看设备

[root@node-1 ~]# rbd device list

id pool namespace image snap device

0 ceph-demo rbd-demo.img - /dev/rbd0格式化mkfs.ext4 /dev/rbd0查看设备[root@node-1 ~]# lsblk

NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT

sda 8:0 0 100G 0 disk

├─sda1 8:1 0 1G 0 part /boot

└─sda2 8:2 0 99G 0 part

└─centos-root 253:0 0 99G 0 lvm /

sdb 8:16 0 50G 0 disk

└─ceph--82366968--ae58--446f--88db--57f5246aa08d-osd--block--19d85d44--9015--4580--bf2d--eae1afe0c0d6

253:1 0 50G 0 lvm

sdc 8:32 0 50G 0 disk

sr0 11:0 1 792M 0 rom

rbd0 252:0 0 10G 0 disk挂载块设备mkdir /mnt/rbd-demo

mount /dev/rbd0 /mnt/rbd-demo

[root@node-1 ~]# df -h /dev/rbd0

Filesystem Size Used Avail Use% Mounted on

/dev/rbd0 9.8G 37M 9.2G 1% /mnt/rbd-demorbd扩容[root@node-1 rbd-demo]# rbd resize ceph-demo/rbd-demo.img --size 20G

Resizing image: 100% complete...done.

[root@node-1 rbd-demo]# rbd info ceph-demo/rbd-demo.img

rbd image 'rbd-demo.img':

size 20 GiB in 5120 objects

order 22 (4 MiB objects)

snapshot_count: 0

id: 1134749f26f5

block_name_prefix: rbd_data.1134749f26f5

format: 2

features: layering

op_features:

flags:

create_timestamp: Tue Sep 22 15:39:03 2020

access_timestamp: Tue Sep 22 15:39:03 2020

modify_timestamp: Tue Sep 22 15:39:03 2020

# 查看设备已经扩容到20G主机扩容[root@node-1 rbd-demo]# df -h /dev/rbd0

Filesystem Size Used Avail Use% Mounted on

/dev/rbd0 9.8G 37M 9.2G 1% /mnt/rbd-demo

#查看发现挂载的盘没有扩容刷新磁盘空间

[root@node-1 rbd-demo]# resize2fs /dev/rbd0

resize2fs 1.42.9 (28-Dec-2013)

Filesystem at /dev/rbd0 is mounted on /mnt/rbd-demo; on-line resizing required

old_desc_blocks = 2, new_desc_blocks = 3

The filesystem on /dev/rbd0 is now 5242880 blocks long.

#再次查看设备大小

[root@node-1 rbd-demo]# df -h /dev/rbd0

Filesystem Size Used Avail Use% Mounted on

/dev/rbd0 20G 44M 19G 1% /mnt/rbd-demo查看数据object#grep 的为rbd info 查看的block_name_prefix的值，他作为object的前缀

rados -p ceph-demo ls | grep rbd_data.1134749f26f5查看object[root@node-1 rbd-demo]# rados -p ceph-demo stat rbd_data.1134749f26f5.0000000000000e00

ceph-demo/rbd_data.1134749f26f5.0000000000000e00 mtime 2020-09-25 14:19:39.000000, size 4096

# 可以看到每个object大小默认4M查看objec数据存储[root@node-1 rbd-demo]# ceph osd map ceph-demo rbd_data.1134749f26f5.0000000000000120

osdmap e35 pool 'ceph-demo' (1) object 'rbd_data.1134749f26f5.0000000000000120' -> pg 1.b2f3e87f (1.3f) -> up ([1,2,0], p1) acting ([1,2,0], p1)

查看当前object落在了id为1.3f的pg上，pg存放在id为[1,2,0]三个OSD上，可以通过ceph osd tree查看osd分别在那个主机测试写入数据dd if=/dev/zero of=test.image bs=1M count=1024

rados -p ceph-demo ls | grep rbd_data.1134749f26f5 | wc -l

299

#查看object数量在增加报错处理[root@node-1 ~]# ceph health detail

HEALTH_WARN 1 pool(s) full

POOL_FULL 1 pool(s) full

pool 'ceph-demo' is full (running out of quota)

#每个object默认4M 大概算下设置pool的最大object值

ceph osd pool set-quota ceph-demo max_objects 12800