Chạy trên node OSD
Lưu ý:
- SLEEP: mặc định là 0.2, tăng 0.5-1.0 nếu node đang tải cao
- POOLID: giúp tính OSD đang dùng bao nhiêu cho riêng pool này
#!/usr/bin/env python3
import json, re, subprocess, time, socket, datetime
import urllib.request
PG_API_BASE = "http://10.166.11.11:1024"
SLEEP = 0.2
def sh(cmd):
return subprocess.check_output(cmd, text=True, stderr=subprocess.STDOUT)
def j(cmd):
return json.loads(sh(cmd))
def GiB(x): return x / (1024**3)
def TiB(x): return x / (1024**4)
def discover_osd_containers():
out = sh(["podman", "ps", "--format", "{{.Names}}"])
names = [x.strip() for x in out.splitlines() if x.strip()]
res = []
pat = re.compile(r"^ceph-([0-9a-fA-F\-]{8,})-osd-(\d+)$")
for n in names:
low = n.lower()
if any(x in low for x in ["mon","mgr","mds","rgw","crash","exporter","node-exporter","prometheus"]):
continue
m = pat.match(n)
if m:
res.append((n, m.group(1), int(m.group(2))))
return sorted(res, key=lambda x: x[2])
def api_get_pools(osd_id: int):
url = f"{PG_API_BASE}/pg-by-osd/{osd_id}"
with urllib.request.urlopen(url, timeout=10) as resp:
data = json.loads(resp.read().decode("utf-8"))
if not data.get("ok"):
raise RuntimeError(f"API error: {data}")
return data.get("pools", [])
def daemon_json(container, osd_id, args):
return j(["podman", "exec", container, "ceph", "daemon", f"osd.{osd_id}"] + args)
def main():
hostname = socket.gethostname()
run_ts = datetime.datetime.now(datetime.timezone.utc).isoformat()
rows = []
for cname, fsid, oid in discover_osd_containers():
row = {
"ok": False,
"timestamp_utc": run_ts,
"hostname": hostname,
"fsid": fsid,
"container": cname,
"osd": f"osd.{oid}",
}
try:
pools = api_get_pools(oid)
row["pools_from_api"] = pools
perf = daemon_json(cname, oid, ["perf", "dump"])
osd_total = perf["osd"]["stat_bytes"]
osd_used = perf["osd"]["stat_bytes_used"]
osd_avail = perf["osd"].get("stat_bytes_avail", None)
db_used = perf.get("bluefs", {}).get("db_used_bytes", 0)
# bluefs device info
bdi = daemon_json(cname, oid, ["bluestore", "bluefs", "device", "info"])
dev = bdi.get("dev", {})
# note: key "bluefs max available" có space
bluefs_max_avail = dev.get("bluefs max available", None)
# frag score
fragj = daemon_json(cname, oid, ["bluestore", "allocator", "score", "block"])
frag = fragj.get("fragmentation_rating", None)
pool_alloc_sum = 0
pool_data_stored_sum = 0
per_pool = {}
for p in pools:
ps = daemon_json(cname, oid, ["dump_pool_statfs", str(p)])
alloc = ps.get("allocated", 0)
stored = ps.get("data_stored", 0)
pool_alloc_sum += alloc
pool_data_stored_sum += stored
per_pool[str(p)] = {
"allocated_bytes": alloc,
"data_stored_bytes": stored
}
unacc_bytes = osd_used - pool_alloc_sum - db_used
row.update({
"ok": True,
"osd_total_bytes": osd_total,
"osd_used_bytes": osd_used,
"osd_avail_bytes": osd_avail,
"osd_used_pct": (osd_used * 100.0 / osd_total) if osd_total else None,
"bluefs_db_used_bytes": db_used,
"bluefs_device": dev.get("device", None),
"bluefs_device_total_bytes": dev.get("total", None),
"bluefs_device_free_bytes": dev.get("free", None),
"bluefs_used_bytes": dev.get("bluefs_used", None),
"bluefs_max_avail_bytes": bluefs_max_avail,
"frag": frag,
"pool_alloc_sum_bytes": pool_alloc_sum,
"pool_data_stored_sum_bytes": pool_data_stored_sum,
"per_pool": per_pool,
"unaccounted_bytes": unacc_bytes,
"osd_used_TiB": TiB(osd_used),
"pool_alloc_sum_TiB": TiB(pool_alloc_sum),
"unaccounted_GiB": GiB(unacc_bytes),
"db_used_GiB": GiB(db_used),
"bluefs_device_free_GiB": GiB(dev["free"]) if isinstance(dev.get("free"), (int, float)) else None,
"bluefs_device_total_TiB": TiB(dev["total"]) if isinstance(dev.get("total"), (int, float)) else None,
})
except Exception as e:
row["error"] = str(e).strip()
rows.append(row)
time.sleep(SLEEP)
print(json.dumps(rows, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
Chạy
python3 osd_unaccounted_autopools_plus.py > /home/hoanghd3/osd_autopools_plus.json
Check nhanh
jq '.[0]' /home/hoanghd3/osd_autopools_plus.json
jq '[.[] | select(.ok==true) | .unaccounted_bytes] | add / 1099511627776' /home/hoanghd3/osd_autopools_plus.json
Output là JSON list với:
- tự lấy hostname
- tự detect FSID từ tên container
ceph-<FSID>-osd-<ID>(khỏi hardcode) - có timestamp (ISO8601) cho mỗi lần chạy
- vẫn lọc chỉ OSD containers, không dính crash/mon/mgr/exporter
- output là một mảng JSON (list), mỗi OSD = 1 object
Ví dụ output:
root@SOC-CEPH-PUB-C06-OSD-050:/home/hoanghd3# jq '.[0]' /home/hoanghd3/osd_autopools_plus.json
{
"ok": true,
"timestamp_utc": "2026-01-13T17:45:59.565834+00:00",
"hostname": "SOC-CEPH-PUB-C06-OSD-050",
"fsid": "fe23f650-2b8f-11ee-9575-17906427caf5",
"container": "ceph-fe23f650-2b8f-11ee-9575-17906427caf5-osd-608",
"osd": "osd.608",
"pools_from_api": [
4
],
"osd_total_bytes": 1600315719680,
"osd_used_bytes": 1016167370752,
"osd_avail_bytes": 584148348928,
"osd_used_pct": 63.4979309554738,
"bluefs_db_used_bytes": 4420141056,
"bluefs_device": "BDEV_DB",
"bluefs_device_total_bytes": 1600315719680,
"bluefs_device_free_bytes": 584146690048,
"bluefs_used_bytes": 4420206592,
"bluefs_max_avail_bytes": 297222275072,
"frag": 0.8989855746934527,
"pool_alloc_sum_bytes": 1011458252800,
"pool_data_stored_sum_bytes": 1073563047769,
"per_pool": {
"4": {
"allocated_bytes": 1011458252800,
"data_stored_bytes": 1073563047769
}
},
"unaccounted_bytes": 288976896,
"osd_used_TiB": 0.9241988398134708,
"pool_alloc_sum_TiB": 0.9199159219861031,
"unaccounted_GiB": 0.2691307067871094,
"db_used_GiB": 4.1165771484375,
"bluefs_device_free_GiB": 544.0289993286133,
"bluefs_device_total_TiB": 1.4554786682128906
}
root@SOC-CEPH-PUB-C06-OSD-050:/home/hoanghd3# jq '[.[] | select(.ok==true) | .unaccounted_bytes] | add / 1099511627776' /home/hoanghd3/osd_autopools_plus.json
0.0042819008231163025
MON chạy Flask API, còn script trên OSD node gọi API để lấy list pool-id đang có PG trên OSD đó, rồi script sẽ tự dump_pool_statfs theo đúng pool-id thay vì hardcode POOLID=3.
Mình đưa bạn 2 file:
- Flask API (chạy trên MON) → trả JSON từ
ceph pg ls-by-osd <id> - OSD-side script (chạy trên OSD node) → gọi API để lấy pool list, rồi tính
unaccountedchính xác hơn
Lưu ý bảo mật: API này chỉ nên mở nội bộ, có token (header) và allowlist IP nếu được.
1) Flask API trên MON node
File: /opt/ceph_pg_api/app.py
#!/usr/bin/env python3
import os
import re
import json
import subprocess
from flask import Flask, request, jsonify
app = Flask(__name__)
# Token đơn giản để tránh ai gọi bậy
API_TOKEN = os.getenv("CEPH_PG_API_TOKEN", "")
def require_token():
if not API_TOKEN:
return True # nếu bạn không set token thì bỏ qua check
return request.headers.get("X-Api-Token", "") == API_TOKEN
def run(cmd):
# timeout để tránh treo
out = subprocess.check_output(cmd, text=True, stderr=subprocess.STDOUT, timeout=20)
return out
def parse_pools_from_text(output: str):
"""
Parse pool id từ output dạng text của `ceph pg ls-by-osd <id>`
PGID thường có dạng: <poolid>.<something>
"""
pools = set()
pgids = []
for line in output.splitlines():
line = line.strip()
if not line:
continue
# skip header lines kiểu "PG_STAT" hoặc "pg_stat"
if line.lower().startswith("pg_stat") or line.lower().startswith("pgid"):
continue
# lấy token đầu tiên có dạng N.xxx
m = re.search(r"\b(\d+)\.[0-9a-fA-F]+\b", line)
if m:
pool = int(m.group(1))
pools.add(pool)
pgids.append(m.group(0))
return sorted(pools), pgids
@app.get("/health")
def health():
return jsonify({"ok": True})
@app.get("/pg-by-osd/<int:osd_id>")
def pg_by_osd(osd_id: int):
if not require_token():
return jsonify({"ok": False, "error": "unauthorized"}), 401
# ưu tiên json output nếu ceph hỗ trợ
# nhiều cluster hỗ trợ -f json cho subcommand này; nếu fail thì fallback text parsing
try:
out = run(["ceph", "pg", "ls-by-osd", str(osd_id), "-f", "json"])
data = json.loads(out)
# cấu trúc json tùy version; mình làm mềm:
# cố gắng lấy danh sách pgid ở nhiều chỗ khác nhau
pgids = []
pools = set()
# case: {"pg_stats":[{"pgid":"3.abc",...},...]}
if isinstance(data, dict) and "pg_stats" in data and isinstance(data["pg_stats"], list):
for x in data["pg_stats"]:
pgid = x.get("pgid") or x.get("pgid_str") or x.get("pg")
if isinstance(pgid, str) and "." in pgid:
pgids.append(pgid)
pools.add(int(pgid.split(".", 1)[0]))
# case khác: list trực tiếp
elif isinstance(data, list):
for x in data:
pgid = x.get("pgid") if isinstance(x, dict) else None
if isinstance(pgid, str) and "." in pgid:
pgids.append(pgid)
pools.add(int(pgid.split(".", 1)[0]))
# nếu json parse không ra gì thì fallback text
if not pools:
raise ValueError("json parsed but no pgids found")
return jsonify({
"ok": True,
"osd_id": osd_id,
"pools": sorted(pools),
"pg_count": len(pgids),
"pgids_sample": pgids[:50], # sample để response không quá to
"source": "json"
})
except Exception:
# fallback: text
out = run(["ceph", "pg", "ls-by-osd", str(osd_id)])
pools, pgids = parse_pools_from_text(out)
return jsonify({
"ok": True,
"osd_id": osd_id,
"pools": pools,
"pg_count": len(pgids),
"pgids_sample": pgids[:50],
"source": "text"
})
if __name__ == "__main__":
# dev only; production dùng gunicorn
app.run(host="0.0.0.0", port=5001)
Cài & chạy nhanh (test)
# trên MON
python3 -m venv /opt/ceph_pg_api/venv
/opt/ceph_pg_api/venv/bin/pip install flask
export CEPH_PG_API_TOKEN="your-secret-token"
/opt/ceph_pg_api/venv/bin/python /opt/ceph_pg_api/app.py
Test:
curl -s -H 'X-Api-Token: your-secret-token' http://127.0.0.1:5001/pg-by-osd/229 | jq .
Chạy production bằng systemd + gunicorn (khuyên dùng)
/opt/ceph_pg_api/venv/bin/pip install gunicorn
/etc/systemd/system/ceph-pg-api.service:
[Unit]
Description=Ceph PG API (ls-by-osd)
After=network.target
[Service]
User=root
WorkingDirectory=/opt/ceph_pg_api
Environment=CEPH_PG_API_TOKEN=your-secret-token
ExecStart=/opt/ceph_pg_api/venv/bin/gunicorn -w 2 -b 0.0.0.0:5001 app:app
Restart=always
RestartSec=2
[Install]
WantedBy=multi-user.target
Enable:
systemctl daemon-reload
systemctl enable --now ceph-pg-api
systemctl status ceph-pg-api
2) Script trên OSD node: gọi API để lấy pool list rồi tính “unaccounted” chuẩn
Bạn dùng version “JSON list output” và thêm phần gọi API như sau.
File: osd_unaccounted_autopools.py (chạy trên OSD node)
Chạy:
python3 osd_unaccounted_autopools.py > /tmp/osd_autopools.json
jq '.[0]' /tmp/osd_autopools.json
root@SOC-CEPH-PUB-C06-OSD-050:/home/hoanghd3# curl http://10.166.21.11:1024/pg-by-osd/229
{"ok":true,"osd_id":229,"pg_count":84,"pgids_sample":["3.ad","3.ca","3.d4","3.18a","3.1a3","3.33c","3.3e7","3.506","3.668","3.6e3","3.6e8","3.7df","3.7e7","3.839","3.87d","3.936","3.938","3.960","3.a6b","3.bbb","3.bfc","3.c63","3.c94","3.db1","3.f6a","3.ff7","3.1050","3.1299","3.131c","3.1371","3.1687","3.1771","3.184d","3.191b","3.1921","3.192c","3.19a6","3.1b00","3.1b80","3.1c35","3.1c6c","3.1c99","3.1cce","3.1cf5","3.1eae","3.1f53","3.1f90","3.1ff3","3.23a5","3.2411"],"pools":[3],"source":"json"}
