卸载osd
df | grep ceph | awk '{print $NF}' | xargs -n1 umount
删除ceph
rm -rf /var/lib/ceph
rm -rf /var/log/ceph
rm -rf /etc/ceph/*
systemctl stop ceph-radosgw.target
systemctl stop ceph-crash.service
systemctl status ceph-crash.service
rm -rf /var/log/radosgw
卸载ceph rpm包
rpm -qa | grep ceph | xargs -n1 yum -y remove
rm /etc/yum.repos.d/ceph*.repo -f
rpm -qa | grep rados | xargs -n1 yum remove -y
yum remove -y python-rados
5.1 单台 Ceph 节点宕机处理步骤
登陆 ceph monitor 节点，查询 ceph 状态：
ceph health detail
将故障节点上的所有 osd 设置成 out，该步骤会触发数据 recovery, 需要等待数据迁移完成, 同时观察虚拟机是否正常：
ceph osd out osd_id
从 crushmap 将 osd 移除，该步骤会触发数据 reblance，等待数据迁移完成，同时观察虚拟机是否正常：
ceph osd crush remove osd_name
删除 osd 的认证： ceph auth del osd_name
删除 osd ：ceph osd rm osd_id
清理lvm残留， 卸载osd重新启用盘时使用。
lvdisplay | grep "LV Path" | awk '{print $NF}' | xargs -i lvremove -f {}
vgdisplay | grep "VG Name" | awk '{print $NF}' | xargs -i vgremove {}
pvremove /dev/sd[b-j]
ceph-volume lvm list
ls /dev/mapper/
systemctl stop ceph-osd@8.service
umount -l
umount -l /var/lib/ceph/osd/ceph-8
ceph yum源
vim /etc/yum.repos.d/ceph.repo
[ceph]
name=ceph
baseurl=http://mirrors.aliyun.com/ceph/rpm-nautilus/el7/x86_64/
gpgcheck=0
[ceph-noarch]
name=cephnoarch
baseurl=http://mirrors.aliyun.com/ceph/rpm-nautilus/el7/noarch/
gpgcheck=0
scp /etc/yum.repos.d/ceph.repo node2:/etc/yum.repos.d/
scp /etc/yum.repos.d/ceph.repo node3:/etc/yum.repos.d/
scp /etc/yum.repos.d/ceph.repo node4:/etc/yum.repos.d/
scp /etc/yum.repos.d/ceph.repo node5:/etc/yum.repos.d/
scp /etc/yum.repos.d/ceph.repo node6:/etc/yum.repos.d/
yum clean all
yum makecache
1. 安装
生成fsid
uuidgen
生成秘钥环
ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
chown ceph:ceph /tmp/ceph.mon.keyring
monmaptool --create --add node1 10.33.132.50 --add node3 10.33.132.52 --add node4 10.33.132.53 --fsid a5303bea-c6ba-4690-9fb2-49ed046be391 /tmp/monmap
monmaptool --print /tmp/monmap
创建默认数据目录
mkdir /var/lib/ceph/mon/ceph-node1
chmod 777 -R /var/lib/ceph/mon/ceph-node1
分发配置文件, 启动mon节点
scp /etc/ceph/ceph.conf node2:/etc/ceph/ceph.conf
scp /etc/ceph/ceph.conf node3:/etc/ceph/ceph.conf
scp /etc/ceph/ceph.conf node4:/etc/ceph/ceph.conf
scp /etc/ceph/ceph.conf node5:/etc/ceph/ceph.conf
scp /etc/ceph/ceph.conf node6:/etc/ceph/ceph.conf
scp /tmp/ceph.mon.keyring node3:/tmp/ceph.mon.keyring
scp /etc/ceph/ceph.client.admin.keyring node3:/etc/ceph/ceph.client.admin.keyring
scp /tmp/monmap node3:/tmp/monmap
scp /tmp/ceph.mon.keyring node4:/tmp/ceph.mon.keyring
scp /etc/ceph/ceph.client.admin.keyring node4:/etc/ceph/ceph.client.admin.keyring
scp /tmp/monmap node4:/tmp/monmap
mkdir /var/lib/ceph/mon/ceph-node3
chmod 777 -R /var/lib/ceph/mon/ceph-node3
mkdir /var/lib/ceph/mon/ceph-node4
chmod 777 -R /var/lib/ceph/mon/ceph-node4
ceph-mon --mkfs -i node3 --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
ceph-mon --mkfs -i node4 --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
chown -R ceph:ceph /var/lib/ceph/mon/ceph-node1
chown -R ceph:ceph /var/lib/ceph/mon/ceph-node3
chown -R ceph:ceph /var/lib/ceph/mon/ceph-node4
systemctl start ceph-mon@node1 && systemctl enable ceph-mon@node1
systemctl status ceph-mon@node1
systemctl start ceph-mon@node3 && systemctl enable ceph-mon@node3
systemctl status ceph-mon@node1
systemctl start ceph-mon@node4 && systemctl enable ceph-mon@node4
systemctl status ceph-mon@node1
ceph -s 查看状态
mon is allowing insecure global_id reclaim
ceph config set mon auth_allow_insecure_global_id_reclaim false
关闭不安全的回收模式
3 monitors have not enabled msgr2
ceph mon enable-msgr2
创建MGR
在运行ceph-mon守护程序的每个节点上，还应该设置一个ceph-mgr守护程序。
创建密钥目录
所有mgr节点都要执行
sudo -u ceph mkdir /var/lib/ceph/mgr/ceph-`hostname -s`
cd /var/lib/ceph/mgr/ceph-`hostname -s`
ceph auth get-or-create mgr.`hostname -s` mon 'allow profile mgr' osd 'allow *' mds 'allow *' > keyring
systemctl enable ceph-mgr@`hostname -s` && systemctl start ceph-mgr@`hostname -s`
查看osd fsid,看下是否已经存在
ceph-volume lvm list
# 准备osd
ceph-volume lvm prepare --data /dev/sdb
ceph-volume lvm prepare --data /dev/sdc
ceph-volume lvm prepare --data /dev/sdd
ceph-volume lvm prepare --data /dev/sde
ceph-volume lvm prepare --data /dev/sdf
ceph-volume lvm prepare --data /dev/sdg
ceph-volume lvm prepare --data /dev/sdh
ceph-volume lvm prepare --data /dev/sdi
ceph-volume lvm prepare --data /dev/sdj
osd fsid 0d839504-effc-4dcd-9afc-28f4f4d31764
osd id 0
devices /dev/sdb
osd fsid 4b57cd50-5566-449d-bdb8-a7de11034fea
osd id 1
devices /dev/sdc
osd fsid 2f3722fe-eee8-4ad5-a3e8-48b6f5f344e8
osd id 2
devices /dev/sdd
osd fsid 1ff13481-4c5e-43f2-9d0a-50f7d5a78bb9
osd id 3
devices /dev/sde
osd fsid d38af12c-4e0f-4810-a658-e4c062b25908
osd id 4
devices /dev/sdf
osd fsid 24d8938c-10a3-4ed6-9d70-366c55ae6152
osd id 5
devices /dev/sdg
osd fsid 02726643-954c-4744-ad7b-bdac475c0e9d
osd id 6
devices /dev/sdh
osd fsid 5ab11637-fd8b-4e99-b7e0-27288f74574f
osd id 7
devices /dev/sdi
osd fsid 2c57b012-24b9-485a-89fe-5b3418051349
osd id 8
devices /dev/sdj
拷贝秘钥文件到其他主机
scp /var/lib/ceph/bootstrap-osd/ceph.keyring node2:/var/lib/ceph/bootstrap-osd/ceph.keyring
scp /var/lib/ceph/bootstrap-osd/ceph.keyring node3:/var/lib/ceph/bootstrap-osd/ceph.keyring
scp /var/lib/ceph/bootstrap-osd/ceph.keyring node4:/var/lib/ceph/bootstrap-osd/ceph.keyring
scp /var/lib/ceph/bootstrap-osd/ceph.keyring node5:/var/lib/ceph/bootstrap-osd/ceph.keyring
scp /var/lib/ceph/bootstrap-osd/ceph.keyring node6:/var/lib/ceph/bootstrap-osd/ceph.keyring
# 激活osd, 查看没问题复制命令执行
ceph-volume lvm list | egrep "osd id|osd fsid" | sed '$!N;s/n/t/' | awk '{print $NF," ",$3}' | xargs -i echo ceph-volume lvm activate {}
pool 3 '.rgw.root' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 64 pgp_num 64 autoscale_mode on last_change 155 flags hashpspool,creating stripe_width 0
pool 1 '.rgw.root' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 64 pgp_num 64 last_change 28780 flags hashpspool stripe_width 0 expected_num_objects 1 application rgw
pool 120 '.rgw.root' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 64 pgp_num 64 last_change 28780 flags hashpspool stripe_width 0 expected_num_objects 1 application rgw
pool 121 'default.rgw.control' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 64 pgp_num 64 last_change 28780 flags hashpspool stripe_width 0 expected_num_objects 1 application rgw
pool 122 'default.rgw.meta' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 256 pgp_num 256 last_change 28780 flags hashpspool stripe_width 0 expected_num_objects 1 application rgw
pool 123 'default.rgw.log' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 64 pgp_num 64 last_change 28780 flags hashpspool stripe_width 0 expected_num_objects 1 application rgw
pool 124 'default.rgw.buckets.index' replicated size 3 min_size 1 crush_rule 1 object_hash rjenkins pg_num 512 pgp_num 512 last_change 30441 flags hashpspool stripe_width 0 expected_num_objects 1 application rgw
pool 125 'default.rgw.buckets.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 10240 pgp_num 10240 last_change 9116 flags hashpspool stripe_width 0 application rgw
pool 126 'default.rgw.buckets.non-ec' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 64 pgp_num 64 last_change 28780 flags hashpspool stripe_width 0 expected_num_objects 1 application rgw
pool 1 '.rgw.root' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 64 pgp_num 64 autoscale_mode warn last_change 233 flags hashpspool stripe_width 0 application rgw
pool 2 'default.rgw.control' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 64 pgp_num 64 autoscale_mode warn last_change 234 flags hashpspool stripe_width 0 application rgw
pool 3 'default.rgw.meta' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 256 pgp_num 256 autoscale_mode warn last_change 235 flags hashpspool stripe_width 0 application rgw
pool 4 'default.rgw.log' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 64 pgp_num 64 autoscale_mode warn last_change 236 flags hashpspool stripe_width 0 application rgw
pool 5 'default.rgw.buckets.index' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 512 pgp_num 512 autoscale_mode warn last_change 237 flags hashpspool stripe_width 0 application rgw
pool 6 'default.rgw.buckets.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8192 pgp_num 8192 autoscale_mode warn last_change 238 lfor 0/0/222 flags hashpspool stripe_width 0 application rgw
pool 7 'default.rgw.buckets.non-ec' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 64 pgp_num 64 autoscale_mode warn last_change 239 flags hashpspool stripe_width 0 application rgw
# 参考
https://blog.51cto.com/yuweibing/2139084
创建host桶
for i in {1..6};do ceph osd crush add-bucket ssd-node$i host;done
for i in {1..6};do ceph osd crush add-bucket hdd-node$i host;done
创建ssd 根桶
ceph osd crush add-bucket ssd root
移动host桶 到根桶
for i in {1..6};do ceph osd crush move hdd-node$i root=default;done
for i in {1..6};do ceph osd crush move ssd-node$i root=ssd;done
移动主机到指定的桶中
osd 中的ssd盘
ceph-volume lvm list | egrep "sdb|==="
ssd osd.0 osd.9 osd.18 osd.27 osd.36 osd.45
ceph osd crush set osd.0 0.894 host=ssd-node1
ceph osd crush set osd.9 0.894 host=ssd-node2
ceph osd crush set osd.18 0.894 host=ssd-node3
ceph osd crush set osd.27 0.894 host=ssd-node4
ceph osd crush set osd.36 0.894 host=ssd-node5
ceph osd crush set osd.45 0.894 host=ssd-node6
# 修改磁盘类型为ssd
ceph osd crush rm-device-class 0
ceph osd crush set-device-class ssd 0 9 18 27 36 45
for((i=1;i<=8;i ));do ceph osd crush set osd."$i" 7.3 host=hdd-node1; done
for((i=10;i<=17;i ));do ceph osd crush set osd."$i" 7.3 host=hdd-node2; done
for((i=19;i<=26;i ));do ceph osd crush set osd."$i" 7.3 host=hdd-node3; done
for((i=28;i<=35;i ));do ceph osd crush set osd."$i" 7.3 host=hdd-node4; done
for((i=37;i<=44;i ));do ceph osd crush set osd."$i" 7.3 host=hdd-node5; done
for((i=46;i<=53;i ));do ceph osd crush set osd."$i" 7.3 host=hdd-node6; done
安装 radosrgw
yum install ceph-radosgw -y
# 创建pool
ceph osd pool create .rgw.root 64 64
ceph osd pool create default.rgw.control 64 64
ceph osd pool create default.rgw.meta 256 256
ceph osd pool create default.rgw.log 64 64
ceph osd pool create default.rgw.buckets.index 512 512
ceph osd pool create default.rgw.buckets.data 8192 8192
ceph osd pool create default.rgw.buckets.non-ec 64 64
创建crush
ceph osd crush rule create-replicated root-ssd ssd host
指定pool crush, 除了data都指向ssd
ceph osd pool ls | grep -v data | xargs -i ceph osd pool set {} crush_rule root-ssd
# 制定application 为rgw
ceph osd pool ls | xargs -i ceph osd pool application enable {} rgw
查看crush 有哪些变化
ceph osd getcrushmap -o crushmap
crushtool -d crushmap -o crushmap
cat crushmap
# 新建RADOSGW用户和keyring
ceph-authtool --create-keyring /etc/ceph/ceph.client.radosgw.keyring
chown ceph:ceph /etc/ceph/ceph.client.radosgw.keyring
生成ceph-radosgw服务对应的用户和key
ceph-authtool /etc/ceph/ceph.client.radosgw.keyring -n client.rgw.node1 --gen-key
ceph-authtool /etc/ceph/ceph.client.radosgw.keyring -n client.rgw.node2 --gen-key
ceph-authtool /etc/ceph/ceph.client.radosgw.keyring -n client.rgw.node3 --gen-key
ceph-authtool /etc/ceph/ceph.client.radosgw.keyring -n client.rgw.node4 --gen-key
ceph-authtool /etc/ceph/ceph.client.radosgw.keyring -n client.rgw.node5 --gen-key
ceph-authtool /etc/ceph/ceph.client.radosgw.keyring -n client.rgw.node6 --gen-key
添加用户访问权限
ceph-authtool -n client.rgw.node1 --cap osd 'allow rwx' --cap mon 'allow rwx' /etc/ceph/ceph.client.radosgw.keyring
ceph-authtool -n client.rgw.node2 --cap osd 'allow rwx' --cap mon 'allow rwx' /etc/ceph/ceph.client.radosgw.keyring
ceph-authtool -n client.rgw.node3 --cap osd 'allow rwx' --cap mon 'allow rwx' /etc/ceph/ceph.client.radosgw.keyring
ceph-authtool -n client.rgw.node4 --cap osd 'allow rwx' --cap mon 'allow rwx' /etc/ceph/ceph.client.radosgw.keyring
ceph-authtool -n client.rgw.node5 --cap osd 'allow rwx' --cap mon 'allow rwx' /etc/ceph/ceph.client.radosgw.keyring
ceph-authtool -n client.rgw.node6 --cap osd 'allow rwx' --cap mon 'allow rwx' /etc/ceph/ceph.client.radosgw.keyring
将keyring导入集群中
ceph -k /etc/ceph/ceph.client.admin.keyring auth add client.rgw.node1 -i /etc/ceph/ceph.client.radosgw.keyring
ceph -k /etc/ceph/ceph.client.admin.keyring auth add client.rgw.node2 -i /etc/ceph/ceph.client.radosgw.keyring
ceph -k /etc/ceph/ceph.client.admin.keyring auth add client.rgw.node3 -i /etc/ceph/ceph.client.radosgw.keyring
ceph -k /etc/ceph/ceph.client.admin.keyring auth add client.rgw.node4 -i /etc/ceph/ceph.client.radosgw.keyring
ceph -k /etc/ceph/ceph.client.admin.keyring auth add client.rgw.node5 -i /etc/ceph/ceph.client.radosgw.keyring
ceph -k /etc/ceph/ceph.client.admin.keyring auth add client.rgw.node6 -i /etc/ceph/ceph.client.radosgw.keyring
编辑配置文件
cat >> /etc/ceph/ceph.conf << EOF
[client.rgw.node1]
host=node1
keyring=/etc/ceph/ceph.client.radosgw.keyring
log file=/var/log/radosgw/client.radosgw.gateway.log
rgw_frontends = civetweb port=8080 num_threads=2048
[client.rgw.node2]
host=node2
keyring=/etc/ceph/ceph.client.radosgw.keyring
log file=/var/log/radosgw/client.radosgw.gateway.log
rgw_frontends = civetweb port=8080 num_threads=2048
[client.rgw.node3]
host=node3
keyring=/etc/ceph/ceph.client.radosgw.keyring
log file=/var/log/radosgw/client.radosgw.gateway.log
rgw_frontends = civetweb port=8080 num_threads=2048
[client.rgw.node4]
host=node4
keyring=/etc/ceph/ceph.client.radosgw.keyring
log file=/var/log/radosgw/client.radosgw.gateway.log
rgw_frontends = civetweb port=8080 num_threads=2048
[client.rgw.node5]
host=node5
keyring=/etc/ceph/ceph.client.radosgw.keyring
log file=/var/log/radosgw/client.radosgw.gateway.log
rgw_frontends = civetweb port=8080 num_threads=2048
[client.rgw.node6]
host=node6
keyring=/etc/ceph/ceph.client.radosgw.keyring
log file=/var/log/radosgw/client.radosgw.gateway.log
rgw_frontends = civetweb port=8080 num_threads=2048
EOF
创建日志目录
mkdir /var/log/radosgw
chown ceph:ceph /var/log/radosgw
拷贝keyring、 ceph.conf到其他主机
scp /etc/ceph/ceph.client.radosgw.keyring node2:/etc/ceph/ceph.client.radosgw.keyring
scp /etc/ceph/ceph.client.radosgw.keyring node3:/etc/ceph/ceph.client.radosgw.keyring
scp /etc/ceph/ceph.client.radosgw.keyring node4:/etc/ceph/ceph.client.radosgw.keyring
scp /etc/ceph/ceph.client.radosgw.keyring node5:/etc/ceph/ceph.client.radosgw.keyring
scp /etc/ceph/ceph.client.radosgw.keyring node6:/etc/ceph/ceph.client.radosgw.keyring
scp /etc/ceph/ceph.conf node2:/etc/ceph/ceph.conf
scp /etc/ceph/ceph.conf node3:/etc/ceph/ceph.conf
scp /etc/ceph/ceph.conf node4:/etc/ceph/ceph.conf
scp /etc/ceph/ceph.conf node5:/etc/ceph/ceph.conf
scp /etc/ceph/ceph.conf node6:/etc/ceph/ceph.conf
其他主机执行修改属组
chown ceph:ceph /etc/ceph/ceph.client.radosgw.keyring
创建目录并授权
mkdir /var/log/radosgw
chown ceph:ceph /var/log/radosgw
启动rgw服务
systemctl start ceph-radosgw@rgw.`hostname -s` && systemctl enable ceph-radosgw@rgw.`hostname -s`
测试rgw可用性
{
"user_id": "cltx_test",
"display_name": "cltx_test for php user",
"email": "",
"suspended": 0,
"max_buckets": 1000,
"subusers": [],
"keys": [
{
"user": "cltx_test",
"access_key": "VOLTH06Z29O7N0YEAVBR",
"secret_key": "WWljAH2UhwjFCfKc0vgfgbd11eif9lAjAvLwr8Je"
}
],
"swift_keys": [],
"caps": [],
"op_mask": "read, write, delete",
"default_placement": "",
"default_storage_class": "",
"placement_tags": [],
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"temp_url_keys": [],
"type": "rgw",
"mfa_ids": []
}
boto3 压测
rados bench 30 -b 4K -t 32 write --no-cleanup -p test 压测，pool name test
dashboard 安装
1. yum install ceph-mgr-dashboard
2. ceph mgr module enable dashboard
3. 创建证书
ceph dashboard create-self-signed-cert
4. 创建用户名密码
ceph dashboard set-login-credentials admin admin
5.但是会报错 Error EINVAL: Please specify the file containing the password/secret with "-i" option.
把密码写入到 /etc/ceph/dashboard_passwd.txt,名字随便写
6. 访问dashbord
ceph mgr services
