Logo

Personal Ops Runbook

Personal runbook covering infrastructure operations for Cloud, Kubernetes, OpenStack, and Ceph environments. Includes deployment and teardown procedures, node management, cluster monitoring setup, and incident response workflows compiled from day-to-day operational work. Intended strictly for personal reference — configurations and scripts are environment-specific and not guaranteed to work as-is elsewhere.

BUILD LIGHTOS OFFLINE KIT — cần internet, chạy 1 lấn

Biến môi trường proxy

export http_proxy=http://10.237.7.250:3128
export https_proxy=http://10.237.7.250:3128
export no_proxy=10.237.0.0/16,localhost,127.0.0.1
grep -q '^proxy=' /etc/dnf/dnf.conf || echo 'proxy=http://10.237.7.250:3128' >> /etc/dnf/dnf.conf

Tạo các thư mục

KIT=~/lightos-offline-kit
mkdir -p $KIT/{01-repo,02-ansible,03-controller,04-docs}
mkdir -p $KIT/03-controller/{ansible,lbcli,docker,monitoring,node-exporter}

====== 01-repo : RPM Lightbits + deps ======

# Bật repo Lightbits (thay USERKEY thật)
cat > /etc/yum.repos.d/lightos.repo << 'EOF'
[lightos]
name=Lightbits LightOS
baseurl=https://dl.lightbitslabs.com/<USERKEY>/lightos-3-18-2-rhl-9/rpm/el/9/$basearch
enabled=1
gpgcheck=0
EOF
dnf makecache

# Goi Lightbits chinh + resolve toan bo deps
mkdir -p $KIT/01-repo/target_rpms $KIT/01-repo/deps
dnf download --resolve --alldeps --destdir=$KIT/01-repo/target_rpms \
  lightos duroslight management-node-manager management-cluster-manager \
  management-api-service management-profile-generator discovery-service \
  discovery-client management-lightbox-exporter management-upgrade-manager \
  etcd gftl userlbe klight_utils nvme-cli

# Deps OS pho bien cho cai dat (tach rieng)
dnf download --resolve --alldeps --destdir=$KIT/01-repo/deps \
  chrony mdadm createrepo_c

# Tao metadata repo
createrepo_c $KIT/01-repo/target_rpms
createrepo_c $KIT/01-repo/deps
echo "target=$(ls $KIT/01-repo/target_rpms/*.rpm|wc -l) deps=$(ls $KIT/01-repo/deps/*.rpm|wc -l)"

====== 02-ansible : bo deploy GOC Lightbits ======

# Bung tgz goc hang giao (light-app-install-environment-*.tgz)
tar -xzf ~/light-app-install-environment-v3.18.2*.tgz -C $KIT/02-ansible
ls $KIT/02-ansible/   # phai co: ansible ansible.cfg playbooks plugins roles

====== 03-controller/ansible : ansible offline ======

# pip wheels cho ansible + deps (dung dung version dang chay)
pip3.12 download \
  'ansible==9.13.0' 'ansible-core==2.16.19' 'jinja2==3.1.6' 'PyYAML==6.0.3' \
  'cryptography==49.0.0' 'netaddr==1.3.0' 'packaging==26.2' 'resolvelib==1.0.1' \
  -d $KIT/03-controller/ansible/pip-wheels
# Toan bo collections da cai
tar -czf $KIT/03-controller/ansible/ansible-collections.tar.gz -C ~/.ansible collections

====== 03-controller/lbcli : lbcli offline ======

# Repo lbcli public (khong can USERKEY)
curl -1sLf 'https://dl.lightbitslabs.com/public/lbcli/setup.rpm.sh' | sudo -E bash
dnf download --resolve --alldeps --destdir=$KIT/03-controller/lbcli management-lbcli

====== 03-controller/docker : docker-ce offline ======

dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
dnf download --resolve --alldeps --destdir=$KIT/03-controller/docker \
  docker-ce docker-ce-cli containerd.io docker-ce-rootless-extras

====== 03-controller/monitoring : RPM + Docker image ======

# RPM monitoring (tu repo Lightbits)
dnf download --resolve --alldeps --destdir=$KIT/03-controller/monitoring \
  lightos-monitoring-images lightos-monitoring-clustering

# Docker image: pull roi save (BAT BUOC cho air-gap)
docker pull prom/prometheus:v2.42.0
docker pull grafana/grafana:9.3.6
docker save prom/prometheus:v2.42.0 grafana/grafana:9.3.6 \
  -o $KIT/03-controller/monitoring/monitoring-docker-images.tar

====== 03-controller/node-exporter : binary ======

VER=1.8.2
curl -LO https://github.com/prometheus/node_exporter/releases/download/v${VER}/node_exporter-${VER}.linux-amd64.tar.gz
tar xzf node_exporter-${VER}.linux-amd64.tar.gz
cp node_exporter-${VER}.linux-amd64/node_exporter $KIT/03-controller/node-exporter/
cat > $KIT/03-controller/node-exporter/node_exporter.service << 'EOF'
[Unit]
Description=Prometheus Node Exporter
After=network.target
[Service]
User=node_exporter
Group=node_exporter
Type=simple
ExecStart=/usr/local/bin/node_exporter --web.listen-address=:9100
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF

====== 04-docs : README ======

cat > $KIT/04-docs/README.md << 'EOF'
# LightOS Offline Kit - AlmaLinux/RHEL 9.8 x86_64 + Lightbits 3.18.2

## Structure
- 01-repo/       : Lightbits RPM + deps (deploy cluster node)
- 02-ansible/    : original Lightbits deploy kit (playbooks/roles)
- 03-controller/ : lbcli + docker + monitoring (image tar) + node_exporter + ansible wheels
- 04-docs/       : this file

## Assumptions
- Target OS installed from AlmaLinux 9.8 standard ISO (base packages present).
- Kit does NOT include base OS packages (glibc, openssl, systemd...).
- lbcli install may need --nobest if kit glibc is newer than target.

## Deploy = fully offline (see 1-Deploy.md)
## Destroy = see 0-Destroy.md
EOF

====== Dong goi + checksum ======

cd ~
tar -czf lightos-offline-kit.tar.gz -C ~ lightos-offline-kit
sha256sum lightos-offline-kit.tar.gz > lightos-offline-kit.tar.gz.sha256
ls -lh lightos-offline-kit.tar.gz*

====== Verify kit truoc khi dem di ======

mkdir -p /tmp/vk && tar -xzf ~/lightos-offline-kit.tar.gz -C /tmp/vk
K=/tmp/vk/lightos-offline-kit
echo "repo: $(ls $K/01-repo/target_rpms/*.rpm|wc -l)+$(ls $K/01-repo/deps/*.rpm|wc -l)"
echo "ansible: $(ls $K/02-ansible/playbooks/*.yml|wc -l) playbooks, $(ls $K/02-ansible/roles/|wc -l) roles"
echo "monitoring image: $(ls -lh $K/03-controller/monitoring/*.tar|awk '{print $5}')"
echo "wheels: $(ls $K/03-controller/ansible/pip-wheels/*.whl|wc -l)"
# Test ansible cai offline trong venv sach
python3.12 -m venv /tmp/tv && /tmp/tv/bin/pip install --no-index \
  --find-links $K/03-controller/ansible/pip-wheels ansible >/dev/null 2>&1 \
  && /tmp/tv/bin/ansible --version | head -1
rm -rf /tmp/vk /tmp/tv