Personal Ops Runbook

[root@vm-admin-01 ~]# docker inspect lightbits-prometheus | grep -A20 Mounts
        "Mounts": [
            {
                "Type": "bind",
                "Source": "/var/lib/monitoring-clustering/prometheus",
                "Destination": "/etc/prometheus",
                "Mode": "",
                "RW": true,
                "Propagation": "rprivate"
            },
            {
                "Type": "volume",
                "Name": "prometheus_data",
                "Source": "/var/lib/docker/volumes/prometheus_data/_data",
                "Destination": "/prometheus",
                "Driver": "local",
                "Mode": "z",
                "RW": true,
                "Propagation": ""
            },
            {
                "Type": "bind",

[root@vm-admin-01 ~]# find / -name "prometheus.yml" 2>/dev/null | grep -i monitor
/var/lib/monitoring-clustering/prometheus/prometheus.yml

# my global config
global:
  scrape_interval:     15s # By default, scrape targets every 15 seconds.
  evaluation_interval: 15s # By default, scrape targets every 15 seconds.
  scrape_timeout: 10s

  # Attach these labels to any time series or alerts when communicating with
  # external systems (federation, remote storage, Alertmanager).
  external_labels:
    monitor: 'lightos-duros'

  # Adjust max out-of-order time - metric can be collect few seconds before
  # allow some out out-of-order window, this allow old sample to be collected insted dropped when scraped
storage:
  tsdb:
    out_of_order_time_window: 5m

# Load and evaluate rules in this file every 'evaluation_interval' seconds.
rule_files:
- "alert.rules.yaml"
- "etcd3_alert_rules.yml"
- "record.rules.yaml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
- job_name: lightos
  scheme: http
  scrape_timeout: 13s
  scrape_interval: 15s
  metrics_path: /metrics
  honor_timestamps: True
  params:
    collect[]:
    - systemd
    - clustering
    - datapath
    - duroslight
    - meminfo
    - textfile
    - lightfield
    - netstat
    - netdev
    - cpufreq
    - chrony
  file_sd_configs:
  - refresh_interval: 10s
    files:
    - '/targets/lightbox-exporter/*.yaml'

- job_name: lightos-smart
  scheme: http
  scrape_timeout: 10s
  scrape_interval: 5m
  metrics_path: /metrics
  honor_timestamps: True
  params:
    collect[]:
    - smart
  file_sd_configs:
  - refresh_interval: 10s
    files:
    - '/targets/lightbox-exporter/*.yaml'

- job_name: api-service
  scheme: https
  tls_config:
    insecure_skip_verify: true
  scrape_timeout: 13s
  scrape_interval: 15s
  metrics_path: /metrics
  honor_timestamps: True
  file_sd_configs:
  - refresh_interval: 10s
    files:
    - '/targets/api-service/*.yaml'

- job_name: discovery-service
  scheme: http
  scrape_timeout: 10s
  scrape_interval: 15s
  metrics_path: /probe
  params:
    service:
    - discovery-service
  honor_timestamps: True
  file_sd_configs:
  - refresh_interval: 10s
    files:
    - '/targets/lightbox-exporter/*.yaml'

- job_name: etcd
  scheme: http
  scrape_timeout: 13s
  scrape_interval: 15s
  metrics_path: /probe
  params:
    service:
    - etcd
  honor_timestamps: True
  file_sd_configs:
  - refresh_interval: 10s
    files:
    - '/targets/lightbox-exporter/*.yaml'

- job_name: node-manager
  scheme: http
  scrape_timeout: 13s
  scrape_interval: 15s
  metrics_path: /probe
  params:
    service:
    - node-manager
  honor_timestamps: True
  file_sd_configs:
  - refresh_interval: 10s
    files:
    - '/targets/lightbox-exporter/*.yaml'

- job_name: cluster-manager
  scheme: http
  scrape_timeout: 13s
  scrape_interval: 15s
  metrics_path: /probe
  params:
    service:
    - cluster-manager
  honor_timestamps: True
  file_sd_configs:
  - refresh_interval: 10s
    files:
    - '/targets/lightbox-exporter/*.yaml'

- job_name: node-exporter
  scrape_timeout: 13s
  scrape_interval: 15s
  honor_labels: true
  static_configs:
    - targets: ['10.237.94.100:9100']
      labels:
        instance: "vm-admin-01"
        cluster: "LIGHTBITS"
    - targets: ['10.237.94.101:9100']
      labels:
        instance: "oss-nvme-01"
        cluster: "LIGHTBITS"
    - targets: ['10.237.94.102:9100']
      labels:
        instance: "oss-nvme-02"
        cluster: "LIGHTBITS"
    - targets: ['10.237.94.103:9100']
      labels:
        instance: "oss-nvme-03"
        cluster: "LIGHTBITS"
    - targets: ['10.237.94.104:9100']
      labels:
        instance: "oss-nvme-04"
        cluster: "LIGHTBITS"
    - targets: ['10.237.94.105:9100']
      labels:
        instance: "oss-nvme-05"
        cluster: "LIGHTBITS"
    - targets: ['10.237.94.106:9100']
      labels:
        instance: "oss-nvme-06"
        cluster: "LIGHTBITS"
alerting:
  alertmanagers:
  - api_version: v1
    scheme: http
    timeout: 10s
    static_configs:
    - targets:
      - "alertmanager:9093"

export {http,https}_proxy=http://10.237.7.250:3128
export no_proxy=10.237.0.0/16,localhost,127.0.0.1

cd /tmp
VER=1.8.2
curl -LO https://github.com/prometheus/node_exporter/releases/download/v${VER}/node_exporter-${VER}.linux-amd64.tar.gz
dnf install -y tar
tar xzf node_exporter-${VER}.linux-amd64.tar.gz
cp node_exporter-${VER}.linux-amd64/node_exporter /usr/local/bin/
useradd -rs /bin/false node_exporter 2>/dev/null || true

cat > /etc/systemd/system/node_exporter.service << 'EOF'
[Unit]
Description=Prometheus Node Exporter
After=network.target

[Service]
User=node_exporter
Group=node_exporter
Type=simple
ExecStart=/usr/local/bin/node_exporter --web.listen-address=:9100
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable --now node_exporter
systemctl status node_exporter --no-pager