Personal Ops Runbook

✨ Log Monitoring Pipeline with OpenTelemetry, Loki, Grafana, and Telegram Alerts

⚙️ System Overview

+------------------+
|  /var/log/*.log  |     <-- log file
+--------+---------+
         |
         v
+-----------------------+
|  OpenTelemetry Agent  |     <-- thu thập & parse log
| (filelog receiver)    |
+-----------------------+
         |
         v
+------------------+
|     Loki         |     <-- lưu trữ log (time-series)
+------------------+
         |
         v
+------------------+
|     Grafana      |     <-- hiển thị log, cảnh báo
+------------------+
         |
         v
+------------------+
|  Alertmanager +  |
|  Telegram Alerts |
+------------------+

🛠️ Install OpenTelemetry Collector

mkdir /opt/observiq-otel-collector
cd /opt/observiq-otel-collector
wget -O otelcol.tar.gz "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.117.0/otelcol-contrib_0.117.0_linux_amd64.tar.gz"
tar -xvf otelcol.tar.gz -C .

🖊️ Config: /opt/observiq-otel-collector/config.yml

cat > /opt/observiq-otel-collector/config.yml << 'OEF'
receivers:
  filelog/syslog:
    include:
      - /var/log/syslog
    start_at: end
    attributes:
      service: otel_agent
      farm: lab
      host_name: k3s-master1-100-151
      host_ip: 192.168.100.151
      log_type: syslog
    operators:
      - type: regex_parser
        regex: '^(?P<timestamp>[A-Za-z]{3}.*\d{1,2} \d{2}:\d{2}:\d{2}) (?P<hostname>[^\s]+)\s(?P<appname>[^\[]+)(?:\[(?P<process_id>\d+)\])?:\s(?P<message>.+)$'
        timestamp:
          parse_from: attributes.timestamp
          layout_type: strptime
          layout: '%b %e %H:%M:%S'

exporters:
  loki:
    endpoint: http://192.168.100.151:3100/loki/api/v1/push

  debug:
    verbosity: detailed

processors:
  memory_limiter:
    check_interval: 1s
    limit_percentage: 30
    spike_limit_percentage: 20

  batch:
    timeout: 5s
    send_batch_size: 8192
    send_batch_max_size: 20000000

  attributes/loki-labels:
    actions:
      - action: insert
        key: loki.attribute.labels
        value: hostname, appname, process_id, message, exporter
      - action: insert
        key: exporter
        value: OTLP

  resource/loki-labels:
    attributes:
      - action: insert
        key: loki.resource.labels
        value: farm, host_name, host_ip, log_type, service
      - action: insert
        key: loki.format
        value: raw

service:
  pipelines:
    logs/syslog:
      receivers: [filelog/syslog]
      processors: [memory_limiter, batch, resource/loki-labels, attributes/loki-labels]
      exporters: [loki, debug]
OEF

Run it:

/opt/observiq-otel-collector/otelcol-contrib --config /opt/observiq-otel-collector/config.yml

🧱️ Install Loki + Grafana

📁 Prepare folders

mkdir -p ./loki/tmp/loki/{chunks,index,cache,wal,compactor}
mkdir -p ./loki/rules
mkdir -p ./grafana
chown -R 472:472 ./grafana
chown -R 10001:10001 ./loki

📄 docker-compose.loki.yml

cat > ./docker-compose.yml << 'OEF'
version: '3'

services:
  loki:
    image: grafana/loki:2.9.0
    command: -config.file=/etc/loki/local-config.yaml
    ports:
      - "3100:3100"
    volumes:
      - ./loki:/etc/loki
      - ./loki/tmp:/tmp/loki
      - ./loki/rules/fake:/etc/loki/rules/fake

  grafana:
    image: grafana/grafana:10.2.3
    ports:
      - "3000:3000"
    volumes:
      - ./grafana:/var/lib/grafana
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
OEF

📄 ./loki/local-config.yaml

cat > ./loki/local-config.yaml << 'OEF'
auth_enabled: false

server:
  http_listen_port: 3100
  grpc_listen_port: 9095

ingester:
  wal:
    enabled: true
    dir: /tmp/loki/wal
  lifecycler:
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
  chunk_idle_period: 5m
  max_chunk_age: 1h

schema_config:
  configs:
    - from: 2022-01-01
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

storage_config:
  boltdb_shipper:
    active_index_directory: /tmp/loki/index
    cache_location: /tmp/loki/cache
    shared_store: filesystem
  filesystem:
    directory: /tmp/loki/chunks

compactor:
  working_directory: /tmp/loki/compactor
  shared_store: filesystem

limits_config:
  enforce_metric_name: false
  reject_old_samples: true
  reject_old_samples_max_age: 168h

chunk_store_config:
  max_look_back_period: 0s

table_manager:
  retention_deletes_enabled: true
  retention_period: 4320h

ruler:
  storage:
    type: local
    local:
      directory: /etc/loki/rules
  rule_path: /etc/loki/rules
  alertmanager_url: http://192.168.100.151:9093
  enable_alertmanager_v2: true
  enable_api: true
OEF

🚀 Run stack

docker-compose -f docker-compose.yml down
docker-compose -f docker-compose.yml up -d

🚧 Install Alertmanager + Telegram

📄 /etc/alertmanager/config.yml

global:
  resolve_timeout: 1m

route:
  receiver: telegram
  group_wait: 10s
  group_interval: 30s
  repeat_interval: 2h

receivers:
  - name: telegram
    telegram_configs:
      - bot_token: "<your_bot_token>"
        chat_id: <your_chat_id>
        parse_mode: HTML

Start Alertmanager:

docker run -d --name alertmanager \
  -v /etc/alertmanager:/etc/alertmanager \
  -p 9093:9093 \
  prom/alertmanager \
  --config.file=/etc/alertmanager/config.yml

🔒 Loki Alert Rule Example

📄 ./loki/rules/syslog-rules.yaml

cat > ./loki/rules/fake/syslog-rules.yaml << 'OEF'
namespace: storage.syslog
groups:
- name: system_log_alerts
  interval: 1m
  rules:
  - alert: SimulatedDiskError
    expr: |
      count_over_time({appname="blk_update_request"} |= "error" [1m]) > 0
    for: 0m
    labels:
      severity: critical
      product: vStorage
      category: disk_simulation
      oncall: true
      chain: alert-with-opsgenie
    annotations:
      summary: "Simulated Disk Error Detected"
      description: |
        🚨 Log from {{ $labels.hostname }} (PID {{ $labels.process_id }}):
        {{ $labels.message }}
      contact: thienln, hoanghd3, chuongdn
      priority: P3
OEF

🚧 Log Generator Script for Testing

📄 test.sh

#!/bin/bash
LOG_FILE="/var/log/kern1.log"

while true; do
  TIMESTAMP="$(date '+[%a %b %d %T %Y]')"
  echo "$TIMESTAMP blk_update_request: critical medium error, dev sdj" >> "$LOG_FILE"
  sleep 1
done

chmod +x ./test.sh
./test.sh

🖉 Bash Audit Logging to Syslog

if ! grep -q "BEGIN - AUDIT COMMAND" /etc/bash.bashrc; then
  cat <<EOF >> /etc/bash.bashrc
# BEGIN - AUDIT COMMAND
function log2syslog {
  declare COMMAND
  COMMAND=\$(fc -ln -0)
  logger -p local1.notice -t command -i -- "\${USER}:\${SSH_CONNECTION}:\${SUDO_USER}:\${COMMAND}"
}
trap log2syslog DEBUG
# END - AUDIT COMMAND
EOF
fi

source /etc/bash.bashrc

📲 Trigger test log:

logger -t blk_update_request "critical medium error, dev sdj, sector 1468989224 op 0x0:(READ) flags 0x0 phys_seg 9 prio class 0 (test alert)"
logger -t command "demo alert logs"

🔍 Check loaded rules

apt install jq -y
curl -s http://localhost:3100/loki/api/v1/rules | jq

You should see alert groups and rules listed if Loki loaded them properly.

✅ Alert Rule for Disk Errors

- alert: DemoAlertLog
  expr: |
    sum by(appname, hostname, message) (
      vector(1)
      and ({appname="command"} |= "demo alert logs")
    )
  for: 0m
  labels:
    severity: info
  annotations:
    summary: "Triggered log: {{ $labels.message }}"
    description: |
      Log from {{ $labels.hostname }} (app={{ $labels.appname }}):
      {{ $labels.message }}


- alert: SimulatedDiskError
  expr: |
    sum by(appname, hostname, process_id, message) (
      vector(1) and ({appname="blk_update_request"} |= "error")
    )
  for: 0m
  labels:
    severity: warning
    category: disk_simulation
  annotations:
    summary: "Simulated Disk Error Detected"
    description: |
      🚨 Log from {{ $labels.hostname }} (PID {{ $labels.process_id }}):
      {{ $labels.message }}

✅ Final alert Rule for Disk Errors

namespace: storage.syslog
groups:
- name: system_log_alerts
  interval: 1m
  rules:
  - alert: SimulatedDiskError
    expr: |
      count_over_time({appname="blk_update_request"} |= "error" [1m]) > 0
    for: 0m
    labels:
      severity: critical
      product: vStorage
      category: disk_simulation
      oncall: true
      chain: alert-with-opsgenie
    annotations:
      summary: "Simulated Disk Error Detected"
      description: |
        🚨 Log from {{ $labels.hostname }} (PID {{ $labels.process_id }}):
        {{ $labels.message }}
      contact: thienln, hoanghd3, chuongdn
      priority: P3

System now fully wired: OpenTelemetry → Loki → Alertmanager → Telegram
with full log labeling and real-time alerting ✅