initial POC

README.md

@@ -1,2 +1,68 @@
-# cheap_system_monitor
+# Cheap System Monitor
 
+A small Linux system monitor for Ubuntu and Raspberry Pi OS (including the
+Raspberry Pi 5). It stores metrics in a fixed-size RRD database and produces
+PNG graphs for the last 3 hours, 1 day, 2 weeks, and 1 year.
+
+It collects:
+
+- processor load averages (1, 5, and 15 minutes)
+- CPU user, system, and nice usage
+- total network receive/transmit throughput
+- disk usage for `/` and up to two additional mountpoints
+- memory usage
+
+## Install
+
+Python 3 is already included in supported distributions. Install RRDTool,
+clone/copy this directory, and run the installer:
+
+```sh
+sudo apt update
+sudo apt install rrdtool
+sudo ./install.sh
+```
+
+Edit `/etc/cheap-system-monitor.conf` to choose network interfaces,
+mountpoints, and the graph directory. For example, use
+`mountpoints = /, /boot/firmware, /mnt/data` to monitor two additional disks;
+every configured path must be mounted and accessible.
+Restarting is unnecessary after configuration changes—the next timer runs use
+the new values. If the RRD has already been created, do not change the order of
+mountpoints unless you intentionally want the historical graph labels to refer
+to different disks.
+
+The generated PNG files and a responsive, auto-refreshing `index.html` are
+placed in `/var/www/html/system-monitor` by default.
+With a web server installed, such as nginx or Apache, they can be served
+directly; otherwise they can simply be opened or copied as PNG files.
+
+## Check operation
+
+```sh
+systemctl list-timers 'cheap-system-monitor*'
+journalctl -u cheap-system-monitor.service -n 20
+rrdtool info /var/lib/cheap-system-monitor/system.rrd
+```
+
+Run either job manually with:
+
+```sh
+sudo systemctl start cheap-system-monitor.service
+sudo systemctl start cheap-system-monitor-graphs.service
+```
+
+The first meaningful network rate appears after two collection samples. A new
+installation naturally needs time to accumulate history; RRD archives are
+consolidated to five-minute samples for two weeks and hourly samples for a year.
+
+## Run from the source tree
+
+For development or a non-system installation, copy the example configuration,
+change its paths to writable locations, then use:
+
+```sh
+cp cheap-system-monitor.conf.example cheap-system-monitor.conf
+python3 system_monitor.py --config cheap-system-monitor.conf
+python3 generate_graphs.py --config cheap-system-monitor.conf
+```

cheap-system-monitor.conf.example

@@ -0,0 +1,13 @@
+[monitor]
+# The service user must be able to create/write these locations.
+rrd_file = /var/lib/cheap-system-monitor/system.rrd
+graph_dir = /var/www/html/system-monitor
+
+# Always include /. Add up to two other mounted paths, separated by commas.
+mountpoints = /
+
+# "auto" sums all interfaces except loopback. Or use e.g. eth0,wlan0.
+interfaces = auto
+
+# CPU percentages are measured over this interval each minute.
+cpu_sample_seconds = 1.0

generate_graphs.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+"""Generate system-monitor PNG graphs from its RRD database."""
+
+from __future__ import annotations
+
+import argparse
+import configparser
+import html
+from pathlib import Path
+import subprocess
+
+
+PERIODS = {"3hours": "3h", "1day": "1d", "2weeks": "2w", "1year": "1y"}
+COLORS = ["#2563EB", "#DC2626", "#16A34A", "#9333EA"]
+
+
+def graph(output: Path, rrd: Path, start: str, title: str, unit: str,
+          series: list[tuple[str, str, str]], logarithmic: bool = False) -> None:
+    args = ["rrdtool", "graph", str(output), "--start", f"end-{start}",
+            "--end", "now", "--width", "900", "--height", "260",
+            "--title", title, "--vertical-label", unit, "--slope-mode",
+            "--watermark", "cheap-system-monitor"]
+    if logarithmic:
+        args += ["--logarithmic", "--lower-limit", "1"]
+    for index, (ds, label, cf) in enumerate(series):
+        var = f"v{index}"
+        args += [f"DEF:{var}={rrd}:{ds}:{cf}",
+                 f"LINE{2 if index < 3 else 1}:{var}{COLORS[index % len(COLORS)]}:{label}",
+                 f"GPRINT:{var}:LAST:Current\\:%8.2lf%s",
+                 f"GPRINT:{var}:AVERAGE:Average\\:%8.2lf%s",
+                 f"GPRINT:{var}:MAX:Maximum\\:%8.2lf%s\\n"]
+    subprocess.run(args, check=True)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--config", default="/etc/cheap-system-monitor.conf")
+    args = parser.parse_args()
+    config_parser = configparser.ConfigParser()
+    if not config_parser.read(args.config):
+        raise SystemExit(f"Configuration file not found: {args.config}")
+    config = config_parser["monitor"]
+    rrd = Path(config["rrd_file"])
+    if not rrd.exists():
+        raise SystemExit(f"RRD file does not exist: {rrd}")
+    output_dir = Path(config.get("graph_dir", "/var/www/html/system-monitor"))
+    output_dir.mkdir(parents=True, exist_ok=True)
+    mounts = [item.strip() for item in config["mountpoints"].split(",") if item.strip()]
+    if not mounts or mounts[0] != "/":
+        mounts.insert(0, "/")
+    definitions = [
+        ("load", "Processor load", "load", [("load1", "1 minute", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load15", "15 minutes", "AVERAGE")], False),
+        ("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE")], False),
+        ("network", "Network throughput", "bytes/s", [("net_in", "received", "AVERAGE"), ("net_out", "sent", "AVERAGE")], False),
+        ("disk", "Disk usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)], False),
+        ("memory", "Memory usage", "%", [("memory", "used", "AVERAGE")], False),
+    ]
+    for period_name, start in PERIODS.items():
+        for filename, title, unit, series, logarithmic in definitions:
+            graph(output_dir / f"{filename}-{period_name}.png", rrd, start,
+                  f"{title} — {period_name}", unit, series, logarithmic)
+    cards = "\n".join(
+        f'<section><h2>{html.escape(title)}</h2>' + "".join(
+            f'<figure><figcaption>{period}</figcaption>'
+            f'<img src="{filename}-{period}.png" alt="{html.escape(title)}, {period}"></figure>'
+            for period in PERIODS
+        ) + "</section>"
+        for filename, title, _unit, _series, _logarithmic in definitions
+    )
+    page = f"""<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width">
+<meta http-equiv="refresh" content="300"><title>System monitor</title>
+<style>body{{font:16px system-ui;margin:auto;max-width:1100px;padding:1rem;background:#f8fafc;color:#172033}}
+section{{background:white;padding:1rem;margin:1rem 0;border-radius:.5rem;box-shadow:0 1px 4px #0002}}
+figure{{margin:1rem 0}} img{{width:100%;height:auto}} figcaption{{font-weight:600}}</style>
+</head><body><h1>System monitor</h1>{cards}</body></html>"""
+    (output_dir / "index.html").write_text(page)
+
+
+if __name__ == "__main__":
+    main()

install.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+set -eu
+
+if [ "$(id -u)" -ne 0 ]; then
+    echo "Run this installer as root (sudo ./install.sh)" >&2
+    exit 1
+fi
+
+command -v rrdtool >/dev/null 2>&1 || {
+    echo "rrdtool is required: apt install rrdtool" >&2
+    exit 1
+}
+
+install -d /usr/local/lib/cheap-system-monitor /var/lib/cheap-system-monitor
+install -m 0755 system_monitor.py generate_graphs.py /usr/local/lib/cheap-system-monitor/
+install -m 0644 systemd/*.service systemd/*.timer /etc/systemd/system/
+if [ ! -e /etc/cheap-system-monitor.conf ]; then
+    install -m 0644 cheap-system-monitor.conf.example /etc/cheap-system-monitor.conf
+fi
+
+systemctl daemon-reload
+systemctl enable --now cheap-system-monitor.timer cheap-system-monitor-graphs.timer
+echo "Installed. Edit /etc/cheap-system-monitor.conf if needed."

system_monitor.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""Collect Linux host metrics and store them in an RRD database."""
+
+from __future__ import annotations
+
+import argparse
+import configparser
+import os
+from pathlib import Path
+import subprocess
+import time
+
+
+DEFAULT_CONFIG = "/etc/cheap-system-monitor.conf"
+DS_NAMES = (
+    "load1", "load5", "load15", "cpu_user", "cpu_system", "cpu_nice",
+    "net_in", "net_out", "disk_root", "disk_1", "disk_2", "memory",
+)
+
+
+def read_config(filename: str) -> configparser.SectionProxy:
+    parser = configparser.ConfigParser()
+    parser.read_dict({"monitor": {
+        "rrd_file": "/var/lib/cheap-system-monitor/system.rrd",
+        "mountpoints": "/",
+        "interfaces": "auto",
+        "cpu_sample_seconds": "1.0",
+    }})
+    if not parser.read(filename):
+        raise SystemExit(f"Configuration file not found: {filename}")
+    return parser["monitor"]
+
+
+def run_rrdtool(*arguments: str) -> None:
+    try:
+        subprocess.run(["rrdtool", *arguments], check=True)
+    except FileNotFoundError:
+        raise SystemExit("rrdtool is not installed (try: sudo apt install rrdtool)")
+    except subprocess.CalledProcessError as error:
+        raise SystemExit(f"rrdtool failed with exit status {error.returncode}")
+
+
+def create_rrd(path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    ds = [
+        "DS:load1:GAUGE:180:0:U", "DS:load5:GAUGE:180:0:U",
+        "DS:load15:GAUGE:180:0:U", "DS:cpu_user:GAUGE:180:0:100",
+        "DS:cpu_system:GAUGE:180:0:100", "DS:cpu_nice:GAUGE:180:0:100",
+        "DS:net_in:DERIVE:180:0:U", "DS:net_out:DERIVE:180:0:U",
+        "DS:disk_root:GAUGE:180:0:100", "DS:disk_1:GAUGE:180:0:100",
+        "DS:disk_2:GAUGE:180:0:100", "DS:memory:GAUGE:180:0:100",
+    ]
+    # 1-minute data for a day, 5-minute data for two weeks, hourly for a year.
+    archives = [
+        "RRA:AVERAGE:0.5:1:1440", "RRA:MAX:0.5:1:1440",
+        "RRA:AVERAGE:0.5:5:4032", "RRA:MAX:0.5:5:4032",
+        "RRA:AVERAGE:0.5:60:8784", "RRA:MAX:0.5:60:8784",
+    ]
+    run_rrdtool("create", str(path), "--step", "60", *ds, *archives)
+
+
+def read_cpu() -> tuple[int, int, int, int]:
+    fields = Path("/proc/stat").read_text().splitlines()[0].split()
+    if fields[0] != "cpu" or len(fields) < 8:
+        raise RuntimeError("unexpected /proc/stat format")
+    values = [int(value) for value in fields[1:]]
+    user, nice, system, idle = values[:4]
+    idle += values[4] if len(values) > 4 else 0  # include iowait
+    # Linux reports guest time as already included in user/nice, so only the
+    # first eight fields belong in the total (through steal time).
+    return user, system, nice, sum(values[:8])
+
+
+def cpu_percentages(sample_seconds: float) -> tuple[float, float, float]:
+    before = read_cpu()
+    time.sleep(sample_seconds)
+    after = read_cpu()
+    total = after[3] - before[3]
+    if total <= 0:
+        return 0.0, 0.0, 0.0
+    return tuple(100.0 * (after[i] - before[i]) / total for i in range(3))
+
+
+def memory_percent() -> float:
+    entries = {}
+    for line in Path("/proc/meminfo").read_text().splitlines():
+        key, value = line.split(":", 1)
+        entries[key] = int(value.split()[0])
+    total = entries["MemTotal"]
+    available = entries.get("MemAvailable", entries.get("MemFree", 0))
+    return 100.0 * (total - available) / total
+
+
+def disk_percent(mountpoint: str) -> float:
+    stats = os.statvfs(mountpoint)
+    used = stats.f_blocks - stats.f_bfree
+    available = stats.f_bavail
+    return 100.0 * used / (used + available) if used + available else 0.0
+
+
+def network_counters(interface_setting: str) -> tuple[int, int]:
+    wanted = {item.strip() for item in interface_setting.split(",") if item.strip()}
+    automatic = not wanted or wanted == {"auto"}
+    received = transmitted = 0
+    for line in Path("/proc/net/dev").read_text().splitlines()[2:]:
+        interface, data = line.split(":", 1)
+        interface = interface.strip()
+        if (automatic and interface == "lo") or (not automatic and interface not in wanted):
+            continue
+        fields = data.split()
+        received += int(fields[0])
+        transmitted += int(fields[8])
+    return received, transmitted
+
+
+def collect(config: configparser.SectionProxy) -> list[float | int | str]:
+    load = os.getloadavg()
+    cpu = cpu_percentages(config.getfloat("cpu_sample_seconds"))
+    net = network_counters(config["interfaces"])
+    mounts = [item.strip() for item in config["mountpoints"].split(",") if item.strip()]
+    if not mounts or mounts[0] != "/":
+        mounts.insert(0, "/")
+    if len(mounts) > 3:
+        raise SystemExit("At most three mountpoints are supported (including /)")
+    disks: list[float | str] = [disk_percent(item) for item in mounts]
+    disks.extend(["U"] * (3 - len(disks)))
+    return [*load, *cpu, *net, *disks, memory_percent()]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--config", default=DEFAULT_CONFIG)
+    parser.add_argument("--create-only", action="store_true")
+    args = parser.parse_args()
+    config = read_config(args.config)
+    rrd_file = Path(config["rrd_file"])
+    if not rrd_file.exists():
+        create_rrd(rrd_file)
+    if not args.create_only:
+        values = collect(config)
+        update = "N:" + ":".join(str(round(v, 3)) if isinstance(v, float) else str(v) for v in values)
+        run_rrdtool("update", str(rrd_file), "--template", ":".join(DS_NAMES), update)
+
+
+if __name__ == "__main__":
+    main()

systemd/cheap-system-monitor-graphs.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=Generate cheap system monitor graphs
+After=cheap-system-monitor.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/lib/cheap-system-monitor/generate_graphs.py
+User=root

systemd/cheap-system-monitor-graphs.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Generate cheap system monitor graphs every five minutes
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=5min
+AccuracySec=10s
+Persistent=true
+
+[Install]
+WantedBy=timers.target

systemd/cheap-system-monitor.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=Collect cheap system monitor metrics
+After=local-fs.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/lib/cheap-system-monitor/system_monitor.py
+User=root

systemd/cheap-system-monitor.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Collect cheap system monitor metrics every minute
+
+[Timer]
+OnBootSec=1min
+OnUnitActiveSec=1min
+AccuracySec=1s
+Persistent=true
+
+[Install]
+WantedBy=timers.target