From 8af31c94b509bb16f64f469f4e6ed2c66a089b09 Mon Sep 17 00:00:00 2001 From: Ignace Date: Tue, 11 Aug 2026 17:18:15 +0200 Subject: [PATCH] initial POC --- README.md | 68 ++++++++- cheap-system-monitor.conf.example | 13 ++ generate_graphs.py | 81 +++++++++++ install.sh | 23 +++ system_monitor.py | 146 ++++++++++++++++++++ systemd/cheap-system-monitor-graphs.service | 8 ++ systemd/cheap-system-monitor-graphs.timer | 11 ++ systemd/cheap-system-monitor.service | 8 ++ systemd/cheap-system-monitor.timer | 11 ++ 9 files changed, 368 insertions(+), 1 deletion(-) create mode 100644 cheap-system-monitor.conf.example create mode 100755 generate_graphs.py create mode 100755 install.sh create mode 100755 system_monitor.py create mode 100644 systemd/cheap-system-monitor-graphs.service create mode 100644 systemd/cheap-system-monitor-graphs.timer create mode 100644 systemd/cheap-system-monitor.service create mode 100644 systemd/cheap-system-monitor.timer diff --git a/README.md b/README.md index 497c678..afd6e4b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,68 @@ -# cheap_system_monitor +# Cheap System Monitor +A small Linux system monitor for Ubuntu and Raspberry Pi OS (including the +Raspberry Pi 5). It stores metrics in a fixed-size RRD database and produces +PNG graphs for the last 3 hours, 1 day, 2 weeks, and 1 year. + +It collects: + +- processor load averages (1, 5, and 15 minutes) +- CPU user, system, and nice usage +- total network receive/transmit throughput +- disk usage for `/` and up to two additional mountpoints +- memory usage + +## Install + +Python 3 is already included in supported distributions. Install RRDTool, +clone/copy this directory, and run the installer: + +```sh +sudo apt update +sudo apt install rrdtool +sudo ./install.sh +``` + +Edit `/etc/cheap-system-monitor.conf` to choose network interfaces, +mountpoints, and the graph directory. For example, use +`mountpoints = /, /boot/firmware, /mnt/data` to monitor two additional disks; +every configured path must be mounted and accessible. +Restarting is unnecessary after configuration changes—the next timer runs use +the new values. If the RRD has already been created, do not change the order of +mountpoints unless you intentionally want the historical graph labels to refer +to different disks. + +The generated PNG files and a responsive, auto-refreshing `index.html` are +placed in `/var/www/html/system-monitor` by default. +With a web server installed, such as nginx or Apache, they can be served +directly; otherwise they can simply be opened or copied as PNG files. + +## Check operation + +```sh +systemctl list-timers 'cheap-system-monitor*' +journalctl -u cheap-system-monitor.service -n 20 +rrdtool info /var/lib/cheap-system-monitor/system.rrd +``` + +Run either job manually with: + +```sh +sudo systemctl start cheap-system-monitor.service +sudo systemctl start cheap-system-monitor-graphs.service +``` + +The first meaningful network rate appears after two collection samples. A new +installation naturally needs time to accumulate history; RRD archives are +consolidated to five-minute samples for two weeks and hourly samples for a year. + +## Run from the source tree + +For development or a non-system installation, copy the example configuration, +change its paths to writable locations, then use: + +```sh +cp cheap-system-monitor.conf.example cheap-system-monitor.conf +python3 system_monitor.py --config cheap-system-monitor.conf +python3 generate_graphs.py --config cheap-system-monitor.conf +``` diff --git a/cheap-system-monitor.conf.example b/cheap-system-monitor.conf.example new file mode 100644 index 0000000..c774d56 --- /dev/null +++ b/cheap-system-monitor.conf.example @@ -0,0 +1,13 @@ +[monitor] +# The service user must be able to create/write these locations. +rrd_file = /var/lib/cheap-system-monitor/system.rrd +graph_dir = /var/www/html/system-monitor + +# Always include /. Add up to two other mounted paths, separated by commas. +mountpoints = / + +# "auto" sums all interfaces except loopback. Or use e.g. eth0,wlan0. +interfaces = auto + +# CPU percentages are measured over this interval each minute. +cpu_sample_seconds = 1.0 diff --git a/generate_graphs.py b/generate_graphs.py new file mode 100755 index 0000000..677aa62 --- /dev/null +++ b/generate_graphs.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Generate system-monitor PNG graphs from its RRD database.""" + +from __future__ import annotations + +import argparse +import configparser +import html +from pathlib import Path +import subprocess + + +PERIODS = {"3hours": "3h", "1day": "1d", "2weeks": "2w", "1year": "1y"} +COLORS = ["#2563EB", "#DC2626", "#16A34A", "#9333EA"] + + +def graph(output: Path, rrd: Path, start: str, title: str, unit: str, + series: list[tuple[str, str, str]], logarithmic: bool = False) -> None: + args = ["rrdtool", "graph", str(output), "--start", f"end-{start}", + "--end", "now", "--width", "900", "--height", "260", + "--title", title, "--vertical-label", unit, "--slope-mode", + "--watermark", "cheap-system-monitor"] + if logarithmic: + args += ["--logarithmic", "--lower-limit", "1"] + for index, (ds, label, cf) in enumerate(series): + var = f"v{index}" + args += [f"DEF:{var}={rrd}:{ds}:{cf}", + f"LINE{2 if index < 3 else 1}:{var}{COLORS[index % len(COLORS)]}:{label}", + f"GPRINT:{var}:LAST:Current\\:%8.2lf%s", + f"GPRINT:{var}:AVERAGE:Average\\:%8.2lf%s", + f"GPRINT:{var}:MAX:Maximum\\:%8.2lf%s\\n"] + subprocess.run(args, check=True) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", default="/etc/cheap-system-monitor.conf") + args = parser.parse_args() + config_parser = configparser.ConfigParser() + if not config_parser.read(args.config): + raise SystemExit(f"Configuration file not found: {args.config}") + config = config_parser["monitor"] + rrd = Path(config["rrd_file"]) + if not rrd.exists(): + raise SystemExit(f"RRD file does not exist: {rrd}") + output_dir = Path(config.get("graph_dir", "/var/www/html/system-monitor")) + output_dir.mkdir(parents=True, exist_ok=True) + mounts = [item.strip() for item in config["mountpoints"].split(",") if item.strip()] + if not mounts or mounts[0] != "/": + mounts.insert(0, "/") + definitions = [ + ("load", "Processor load", "load", [("load1", "1 minute", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load15", "15 minutes", "AVERAGE")], False), + ("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE")], False), + ("network", "Network throughput", "bytes/s", [("net_in", "received", "AVERAGE"), ("net_out", "sent", "AVERAGE")], False), + ("disk", "Disk usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)], False), + ("memory", "Memory usage", "%", [("memory", "used", "AVERAGE")], False), + ] + for period_name, start in PERIODS.items(): + for filename, title, unit, series, logarithmic in definitions: + graph(output_dir / f"{filename}-{period_name}.png", rrd, start, + f"{title} — {period_name}", unit, series, logarithmic) + cards = "\n".join( + f'

{html.escape(title)}

' + "".join( + f'
{period}
' + f'{html.escape(title)}, {period}
' + for period in PERIODS + ) + "
" + for filename, title, _unit, _series, _logarithmic in definitions + ) + page = f""" + +System monitor + +

System monitor

{cards}""" + (output_dir / "index.html").write_text(page) + + +if __name__ == "__main__": + main() diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..89e508b --- /dev/null +++ b/install.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu + +if [ "$(id -u)" -ne 0 ]; then + echo "Run this installer as root (sudo ./install.sh)" >&2 + exit 1 +fi + +command -v rrdtool >/dev/null 2>&1 || { + echo "rrdtool is required: apt install rrdtool" >&2 + exit 1 +} + +install -d /usr/local/lib/cheap-system-monitor /var/lib/cheap-system-monitor +install -m 0755 system_monitor.py generate_graphs.py /usr/local/lib/cheap-system-monitor/ +install -m 0644 systemd/*.service systemd/*.timer /etc/systemd/system/ +if [ ! -e /etc/cheap-system-monitor.conf ]; then + install -m 0644 cheap-system-monitor.conf.example /etc/cheap-system-monitor.conf +fi + +systemctl daemon-reload +systemctl enable --now cheap-system-monitor.timer cheap-system-monitor-graphs.timer +echo "Installed. Edit /etc/cheap-system-monitor.conf if needed." diff --git a/system_monitor.py b/system_monitor.py new file mode 100755 index 0000000..8feaa43 --- /dev/null +++ b/system_monitor.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Collect Linux host metrics and store them in an RRD database.""" + +from __future__ import annotations + +import argparse +import configparser +import os +from pathlib import Path +import subprocess +import time + + +DEFAULT_CONFIG = "/etc/cheap-system-monitor.conf" +DS_NAMES = ( + "load1", "load5", "load15", "cpu_user", "cpu_system", "cpu_nice", + "net_in", "net_out", "disk_root", "disk_1", "disk_2", "memory", +) + + +def read_config(filename: str) -> configparser.SectionProxy: + parser = configparser.ConfigParser() + parser.read_dict({"monitor": { + "rrd_file": "/var/lib/cheap-system-monitor/system.rrd", + "mountpoints": "/", + "interfaces": "auto", + "cpu_sample_seconds": "1.0", + }}) + if not parser.read(filename): + raise SystemExit(f"Configuration file not found: {filename}") + return parser["monitor"] + + +def run_rrdtool(*arguments: str) -> None: + try: + subprocess.run(["rrdtool", *arguments], check=True) + except FileNotFoundError: + raise SystemExit("rrdtool is not installed (try: sudo apt install rrdtool)") + except subprocess.CalledProcessError as error: + raise SystemExit(f"rrdtool failed with exit status {error.returncode}") + + +def create_rrd(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + ds = [ + "DS:load1:GAUGE:180:0:U", "DS:load5:GAUGE:180:0:U", + "DS:load15:GAUGE:180:0:U", "DS:cpu_user:GAUGE:180:0:100", + "DS:cpu_system:GAUGE:180:0:100", "DS:cpu_nice:GAUGE:180:0:100", + "DS:net_in:DERIVE:180:0:U", "DS:net_out:DERIVE:180:0:U", + "DS:disk_root:GAUGE:180:0:100", "DS:disk_1:GAUGE:180:0:100", + "DS:disk_2:GAUGE:180:0:100", "DS:memory:GAUGE:180:0:100", + ] + # 1-minute data for a day, 5-minute data for two weeks, hourly for a year. + archives = [ + "RRA:AVERAGE:0.5:1:1440", "RRA:MAX:0.5:1:1440", + "RRA:AVERAGE:0.5:5:4032", "RRA:MAX:0.5:5:4032", + "RRA:AVERAGE:0.5:60:8784", "RRA:MAX:0.5:60:8784", + ] + run_rrdtool("create", str(path), "--step", "60", *ds, *archives) + + +def read_cpu() -> tuple[int, int, int, int]: + fields = Path("/proc/stat").read_text().splitlines()[0].split() + if fields[0] != "cpu" or len(fields) < 8: + raise RuntimeError("unexpected /proc/stat format") + values = [int(value) for value in fields[1:]] + user, nice, system, idle = values[:4] + idle += values[4] if len(values) > 4 else 0 # include iowait + # Linux reports guest time as already included in user/nice, so only the + # first eight fields belong in the total (through steal time). + return user, system, nice, sum(values[:8]) + + +def cpu_percentages(sample_seconds: float) -> tuple[float, float, float]: + before = read_cpu() + time.sleep(sample_seconds) + after = read_cpu() + total = after[3] - before[3] + if total <= 0: + return 0.0, 0.0, 0.0 + return tuple(100.0 * (after[i] - before[i]) / total for i in range(3)) + + +def memory_percent() -> float: + entries = {} + for line in Path("/proc/meminfo").read_text().splitlines(): + key, value = line.split(":", 1) + entries[key] = int(value.split()[0]) + total = entries["MemTotal"] + available = entries.get("MemAvailable", entries.get("MemFree", 0)) + return 100.0 * (total - available) / total + + +def disk_percent(mountpoint: str) -> float: + stats = os.statvfs(mountpoint) + used = stats.f_blocks - stats.f_bfree + available = stats.f_bavail + return 100.0 * used / (used + available) if used + available else 0.0 + + +def network_counters(interface_setting: str) -> tuple[int, int]: + wanted = {item.strip() for item in interface_setting.split(",") if item.strip()} + automatic = not wanted or wanted == {"auto"} + received = transmitted = 0 + for line in Path("/proc/net/dev").read_text().splitlines()[2:]: + interface, data = line.split(":", 1) + interface = interface.strip() + if (automatic and interface == "lo") or (not automatic and interface not in wanted): + continue + fields = data.split() + received += int(fields[0]) + transmitted += int(fields[8]) + return received, transmitted + + +def collect(config: configparser.SectionProxy) -> list[float | int | str]: + load = os.getloadavg() + cpu = cpu_percentages(config.getfloat("cpu_sample_seconds")) + net = network_counters(config["interfaces"]) + mounts = [item.strip() for item in config["mountpoints"].split(",") if item.strip()] + if not mounts or mounts[0] != "/": + mounts.insert(0, "/") + if len(mounts) > 3: + raise SystemExit("At most three mountpoints are supported (including /)") + disks: list[float | str] = [disk_percent(item) for item in mounts] + disks.extend(["U"] * (3 - len(disks))) + return [*load, *cpu, *net, *disks, memory_percent()] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", default=DEFAULT_CONFIG) + parser.add_argument("--create-only", action="store_true") + args = parser.parse_args() + config = read_config(args.config) + rrd_file = Path(config["rrd_file"]) + if not rrd_file.exists(): + create_rrd(rrd_file) + if not args.create_only: + values = collect(config) + update = "N:" + ":".join(str(round(v, 3)) if isinstance(v, float) else str(v) for v in values) + run_rrdtool("update", str(rrd_file), "--template", ":".join(DS_NAMES), update) + + +if __name__ == "__main__": + main() diff --git a/systemd/cheap-system-monitor-graphs.service b/systemd/cheap-system-monitor-graphs.service new file mode 100644 index 0000000..984b20c --- /dev/null +++ b/systemd/cheap-system-monitor-graphs.service @@ -0,0 +1,8 @@ +[Unit] +Description=Generate cheap system monitor graphs +After=cheap-system-monitor.service + +[Service] +Type=oneshot +ExecStart=/usr/local/lib/cheap-system-monitor/generate_graphs.py +User=root diff --git a/systemd/cheap-system-monitor-graphs.timer b/systemd/cheap-system-monitor-graphs.timer new file mode 100644 index 0000000..157ed8f --- /dev/null +++ b/systemd/cheap-system-monitor-graphs.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Generate cheap system monitor graphs every five minutes + +[Timer] +OnBootSec=2min +OnUnitActiveSec=5min +AccuracySec=10s +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/systemd/cheap-system-monitor.service b/systemd/cheap-system-monitor.service new file mode 100644 index 0000000..6c8fffd --- /dev/null +++ b/systemd/cheap-system-monitor.service @@ -0,0 +1,8 @@ +[Unit] +Description=Collect cheap system monitor metrics +After=local-fs.target + +[Service] +Type=oneshot +ExecStart=/usr/local/lib/cheap-system-monitor/system_monitor.py +User=root diff --git a/systemd/cheap-system-monitor.timer b/systemd/cheap-system-monitor.timer new file mode 100644 index 0000000..279d2bf --- /dev/null +++ b/systemd/cheap-system-monitor.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Collect cheap system monitor metrics every minute + +[Timer] +OnBootSec=1min +OnUnitActiveSec=1min +AccuracySec=1s +Persistent=true + +[Install] +WantedBy=timers.target