MON
Xem MON leader
ceph quorum_status --format json-pretty | jq -r .quorum_leader_name
Maintenance per OSD
ceph osd set-group noup osd.?
ceph osd unset-group noup osd.?
Maintenance flag
# Set
ceph osd set noout
ceph osd set norebalance
ceph osd set norecover
ceph osd set noscrub
ceph osd set nodeep-scrub
# Unset
ceph osd unset noout
ceph osd unset norebalance
ceph osd unset norecover
ceph osd unset noscrub
ceph osd unset nodeep-scrub
Set Maintenance Once Node
ceph orch host maintenance enter <hostname>
Add node
Add host
ceph orch host add <hostname> <hostip>
ceph orch ps <hostname>
List device
ceph orch device ls <hostname> --wide --refresh
Deploy host
ceph orch host label add <hostname> <lable1,lable2,...>
Verify osd tree
ceph orch host ls
ceph osd tree
ceph orch ps --daemon_type mon
ceph orch ls --service_type mon
Move host from default other root
ceph osd crush move <hostname> root=<bukcet_name>
OSD
Find OSD maping Disk
for path in /var/lib/ceph/$(ls /var/lib/ceph/ | grep -E '^[0-9a-f]{8}-[0-9a-f]{4}' | head -1)/osd.*; do
osd_id=$(basename $path | cut -d'.' -f2)
block_path="$path/block"
if [ -e "$block_path" ]; then
dm_dev=$(readlink -f "$block_path" | xargs basename)
phys_disk=$(lsblk -s -o NAME,TYPE /dev/$dm_dev 2>/dev/null | awk '$2=="disk"{print $1}' | head -1)
echo "osd.$osd_id -> /dev/$dm_dev -> /dev/$phys_disk"
fi
done
osd.219 -> /dev/dm-2 -> /dev/└─sdd
osd.220 -> /dev/dm-4 -> /dev/└─sde
osd.221 -> /dev/dm-45 -> /dev/└─sdx
osd.222 -> /dev/dm-8 -> /dev/└─sdg
osd.223 -> /dev/dm-10 -> /dev/└─sdh
osd.224 -> /dev/dm-12 -> /dev/└─sdi
osd.225 -> /dev/dm-14 -> /dev/└─sdj
osd.226 -> /dev/dm-16 -> /dev/└─sdk
osd.227 -> /dev/dm-18 -> /dev/└─sdl
osd.228 -> /dev/dm-20 -> /dev/└─sdm
osd.229 -> /dev/dm-22 -> /dev/└─sdn
osd.230 -> /dev/dm-24 -> /dev/└─sdo
osd.231 -> /dev/dm-26 -> /dev/└─sdp
osd.232 -> /dev/dm-28 -> /dev/└─sdq
osd.233 -> /dev/dm-30 -> /dev/└─sdr
osd.234 -> /dev/dm-32 -> /dev/└─sds
osd.235 -> /dev/dm-34 -> /dev/└─sdt
osd.236 -> /dev/dm-36 -> /dev/└─sdu
osd.237 -> /dev/dm-38 -> /dev/└─sdv
osd.238 -> /dev/dm-40 -> /dev/└─sdw
osd.239 -> /dev/dm-42 -> /dev/└─sdb
osd.240 -> /dev/dm-44 -> /dev/└─sdc
osd.745 -> /dev/dm-47 -> /dev/└─sdy
Stop/Start/Restart OSD
ceph orch daemon stop osd.<id>
ceph orch daemon start osd.<id>
ceph orch daemon restart osd.<id>
Tìm các OSD đang Down
shell> ceph osd tree -f json | jq -r '.nodes[] | select(.type=="osd" and .status=="down") | .id'
18
22
Start các OSD đang down (WARNING)
for id in $(ceph osd tree -f json | jq -r '.nodes[] | select(.type=="osd" and .status=="down") | .id'); do
echo "==> Restarting osd.$id"
ceph orch daemon start osd.$id || true
done
Remove các OSD đang down (WARNING)
for id in $(ceph osd tree -f json | jq -r '.nodes[] | select(.type=="osd" and .status=="down") | .id'); do
echo "==> Remove osd.$id"
ceph osd out $id || true
ceph orch daemon stop osd.$id || true
ceph osd crush remove osd.$id || true
ceph osd rm $id || true
ceph auth del osd.$id || true
ceph orch daemon rm osd.$id --force || true
done
Remove OSD
Thực hiện thứ tự theo command dưới
id=26
ceph osd out ${id}
ceph orch daemon stop osd.${id}
ceph osd crush remove osd.${id}
ceph osd rm ${id}
ceph auth del osd.${id}
ceph orch daemon rm osd.${id} --force
BACKFILL && RECOVERY
List the nodes in the subtree.
ceph osd crush ls <subtree>
ceph osd crush dump -f json | jq -r '.buckets[] | select(.type_name=="root") | .name'
Set backfill and recovery using a list of OSDs by list osd.
osd_ids='0 1 2 3 4'
value=1
for id in $osd_ids; do
ceph tell osd.${id} injectargs --osd_recovery_max_active_ssd=${value} --osd-max-backfills=${value}
done
Get backfill and recovery information using a list of OSDs by node name
node_name='VSTOR-CEPH-MBF9-OSD-53'
osd_ids=$(ceph osd crush ls ${node_name}|head -n5)
for osd_id in $osd_ids; do
echo "----- ${osd_id} -----";
ceph config show ${osd_id} | egrep "osd_recovery_max_active|osd_max_backfills"
done
Set backfill and recovery using a list of OSDs by node name
node_name='VSTOR-CEPH-MBF9-OSD-53'
value=1
osd_ids=$(ceph osd crush ls ${node_name})
for osd_id in $osd_ids; do
echo "----- ${osd_id} -----";
ceph tell "${osd_id}" injectargs --osd_recovery_max_active_hdd=${value} --osd-max-backfills=${value}
done
RADOS COMMAND
ceph orch ps --daemon_type rgw
ceph orch ps --service_name rgw.vngv3
REMOVE RADOSGW
Trước tiên xem đúng tên service RGW
root@CEPH-LAB-NODE161:~# ceph orch ls --service_type rgw
NAME PORTS RUNNING REFRESHED AGE PLACEMENT
rgw.default ?:8020 3/3 12s ago 9d CEPH-LAB-NODE161;CEPH-LAB-NODE162;CEPH-LAB-NODE163
Xoá service đó
root@CEPH-LAB-NODE161:~# ceph orch rm rgw.default --force
Removed service rgw.default
Verify đã biến mất
root@CEPH-LAB-NODE161:~# ceph orch ps --daemon_type rgw
No daemons reported
root@CEPH-LAB-NODE161:~# ceph orch ls --service_type rgw
No services reported
Xoá các daemon RGW còn “stale” nếu vẫn còn
ceph orch ps --daemon_type rgw
Ví dụ
ceph orch daemon rm rgw.default.CEPH-LAB-NODE161.imyqct --force
ceph orch daemon rm rgw.default.CEPH-LAB-NODE162.wzgrci --force
ceph orch daemon rm rgw.default.CEPH-LAB-NODE163.zsyqxi --force
Bật cho phép xoá pool (tạm thời)
ceph config set mon mon_allow_pool_delete true
List các pool RGW để chắc chắn
shell> ceph osd pool ls | egrep '^(default\.rgw\.|\.rgw\.root$)'
default.rgw.buckets.data
default.rgw.buckets.index
default.rgw.buckets.non-ec
.rgw.root
default.rgw.log
default.rgw.control
default.rgw.meta
Xoá toàn bộ pool RGW
shell> for p in $(ceph osd pool ls | egrep '^(default\.rgw\.|\.rgw\.root$)'); do
echo "Deleting pool: $p"
ceph osd pool rm "$p" "$p" --yes-i-really-really-mean-it
done
Deleting pool: default.rgw.buckets.data
pool 'default.rgw.buckets.data' removed
Deleting pool: default.rgw.buckets.index
pool 'default.rgw.buckets.index' removed
Deleting pool: default.rgw.buckets.non-ec
pool 'default.rgw.buckets.non-ec' removed
Deleting pool: .rgw.root
pool '.rgw.root' removed
Deleting pool: default.rgw.log
pool 'default.rgw.log' removed
Deleting pool: default.rgw.control
pool 'default.rgw.control' removed
Deleting pool: default.rgw.meta
pool 'default.rgw.meta' removed
Nếu mục tiêu là “dọn sạch cluster” và không dùng RGW nữa, thì chỉ cần giữ .mgr là hợp lý.
for p in $(ceph osd pool ls); do
if [[ "$p" == ".mgr" ]]; then
echo "KEEP: $p"
else
echo "DELETE: $p"
ceph osd pool rm "$p" "$p" --yes-i-really-really-mean-it
fi
done
Xoá pool xong thì tắt lại (khuyến nghị)
ceph config set mon mon_allow_pool_delete false
Verify
shell> ceph osd pool ls
.mgr
CRASH
Archive để nó không còn hiện ls-new
ceph crash archive <crash_name>
Hoặc archive tất cả crash đang NEW
ceph crash archive-all
Xóa hẳn crash record
ceph crash rm <crash_name>
Kiểm tra lại:
ceph crash ls-new
ceph crash ls
Xem thông tin crash
ceph crash info <crash_name>
ISO AUTOINSTALL
MBFQ9
http://172.25.21.1/iso/ubuntu/22.04/ubuntu-22.04-autoinstall.iso # ubuntu/abc123
Script add username
#!/usr/bin/env bash
# Define config file
sshd_config="/etc/ssh/sshd_config"
shadow_filename="/etc/shadow"
# Define username and key
new_user='hoanghd3'
public_key='ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDYzXdG9bdffuwQC/1FjrbupIO0NRoCBRbcXbLakzCVdLWybLDGUzxKgNm5p50BTKKCFG04HqFC5EVNVb52tLY/VVaxqcJyG2W41ZrdMCQ0ELmePrjrOVy+zjaD+dINarS93MteDvSNqv1zkcGfGhqBqOrh46ZX1TrK+AtsXoxGmw4oX3SjbWnFqBlOhxanI/VtVgVZIxUezgERjpKRCgICQgvwiK+8xAtTDYd2nltAGGBCWFS10s+jtqXwbPFIXSfgVOiHuqYN/k3sChj5nvRk2bxMJNSfcmY0HVhIteklB0ZWVIPWEzL5uttJU14FakaI5Q2bod9QcUX/5B0xxe5239z6t7e2LGMsWIGa6ET4rre4VjuTeZQIoPe1Ar43UGzApt9c3045Vjgi99EPqJ7fOIc/fn6nPST5jQfgnbd3PcQ4Tr7loQJ8DToNTwr/pquX9OBgwqjlK5qvjvpN+nuk641mA2kbzDdfC4fI/Nj71EYfQ6et+RabdjlimyKtAka80TdRhrG0ZQoXLcO73P9Fq+jVeMyyCKvYbnSVw0zLRgZ6ccESMCxJw676QhHx4HdBOW1SAcZHPsafeIhGLTg4VBa7disbxK7OjEpQJYIn4+COp2nvItQVelrXwAQS/EUP9XOpwhiEgxTpX3SMIZiAhB/k8MdL9WqSqhU2IOfVuw=='
hashpass='$6$qBoGXbb2$z6N9MHA2Kgq.SBjAzaJRdd.T/GB6MntcDtZH0jvtI1foZBOj.XcUvngGYQeY9/Ia5tAeYUSVHcpT3O.R6HWoj0'
# Check if user exists
if id "$new_user" &>/dev/null; then
echo "User ${new_user} already exists. Skipping creation."
else
useradd -m -s /bin/bash "$new_user"
mkdir -p "/home/${new_user}/.ssh"
usermod -aG sudo "$new_user"
fi
# Add public key
echo "$public_key" > "/home/${new_user}/.ssh/authorized_keys"
# Set permissions
chmod 700 "/home/${new_user}/.ssh"
chmod 600 "/home/${new_user}/.ssh/authorized_keys"
chown -R "${new_user}:${new_user}" "/home/${new_user}/.ssh"
# Update SSHD config
if ! grep -q "^AllowUsers" "$sshd_config"; then
echo "AllowUsers ${new_user}" >> "$sshd_config"
echo "AllowUsers line added to $sshd_config."
elif ! grep -q "^AllowUsers.*\b${new_user}\b" "$sshd_config"; then
sed -i "/^AllowUsers/ s/$/ ${new_user}/" "$sshd_config"
echo "User ${new_user} added to AllowUsers."
else
echo "User ${new_user} already in AllowUsers."
fi
# Set password
if grep -q "^${new_user}:" "$shadow_filename"; then
sed -i "s|^${new_user}:[^:]*|${new_user}:${hashpass}|" "$shadow_filename"
echo "Password for user ${new_user} has been changed."
else
echo "User ${new_user} not found in ${shadow_filename}."
fi
# Restart SSHD to apply changes
systemctl restart sshd
echo "User ${new_user} has been created and SSH service restarted."
ceph orch daemon restart rgw.vngv3.VSTOR-CEPH-MBF9-RGW-101.vhwswb
hoặc redeploy đúng daemon đó
ceph orch daemon redeploy rgw.vngv3.VSTOR-CEPH-MBF9-RGW-101.vhwswb
ceph orch ls --service_name rgw.vngv3 -f yaml
hoặc
ceph orch ls --service_name rgw.vngv3 -f json-pretty ceph orch ps --service_name rgw.vngv3
Set maxbackfill to 1 all OSDs avoid overloading the cluster
subtree='mbf9-02'
for id in $(ceph osd ls-tree ${subtree}); do
echo "----- osd.${id} -----";
ceph tell "osd.${id}" injectargs --osd_recovery_max_active_hdd=1 --osd-max-backfills=1
done
Verify
subtree='mbf9-02'
for id in $(ceph osd ls-tree ${subtree}|head -n20); do
echo "----- osd.${id} -----";
ceph config show osd.${id} | egrep "osd_recovery_max_active|osd_max_backfills"
done
List all buckets
shell> ceph osd crush dump -f json | jq -r '.buckets[] | select(.type_name=="root") | .name'
ssd-01
ssd-01~ssd
default
default~ssd
ssd-02
ssd-02~ssd
ssd-03
ssd-03~ssd
PUBC07-SSD-01
PUBC07-SSD-01~ssd
shell> root@SOC-CEPH-PUB-C07-MON-011:/var/log/ceph/9efd2cce-365b-11ee-853e-a59020fddc84# ceph osd ls-tree ssd-02 --format json [366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983]
ceph osd crush dump -f json | jq -r '.buckets[] | select(.type_name=="root") | .name'
root@SOC-CEPH-PUB-C07-MON-012:/home/hoanghd3# ceph osd crush dump -f json | jq -r '.buckets[] | select(.type_name=="root") | .name' ssd-01 ssd-01~ssd default default~ssd ssd-02 ssd-02~ssd ssd-03 ssd-03~ssd PUBC07-SSD-01 PUBC07-SSD-01~ssd
root@SOC-CEPH-PUB-C07-MON-012:/home/hoanghd3# ceph osd crush ls ssd-03 SOC-CEPH-PUB-C07-OSD-052 SOC-CEPH-PUB-C07-OSD-054 SOC-CEPH-PUB-C07-OSD-055 SOC-CEPH-PUB-C07-OSD-058 SOC-CEPH-PUB-C07-OSD-059 SOC-CEPH-PUB-C07-OSD-050 SOC-CEPH-PUB-C07-OSD-051 SOC-CEPH-PUB-C07-OSD-053 SOC-CEPH-PUB-C07-OSD-056 SOC-CEPH-PUB-C07-OSD-057 SOC-CEPH-PUB-C07-OSD-060 SOC-CEPH-PUB-C07-OSD-046
subtree='ssd-03' for id in $(ceph osd ls-tree ${subtree}); do echo "----- osd.${id} -----"; ceph tell "osd.${id}" injectargs --osd_recovery_max_active_hdd=1 --osd-max-backfills=1 done
for id in $(ceph osd ls-tree ${subtree}|head -n20); do echo "----- osd.${id} -----"; ceph config show osd.${id} | egrep "osd_recovery_max_active|osd_max_backfills" done
