nicer graphs

README.md

@@ -11,6 +11,7 @@ It collects:
 - total network receive/transmit throughput
 - disk usage for `/` and up to two additional mountpoints
 - memory usage
+- system uptime in days
 
 ## Install
 
@@ -62,6 +63,11 @@ The first meaningful network rate appears after two collection samples. A new
 installation naturally needs time to accumulate history; RRD archives are
 consolidated to five-minute samples for two weeks and hourly samples for a year.
 
+The RRD schema is fixed when the database is created. When upgrading from a
+version without uptime monitoring, move or remove the existing `system.rrd`
+before the next collection if preserving its old history is not required. The
+collector will then create a new database containing the uptime data source.
+
 ## Run from the source tree
 
 For development or a non-system installation, copy the example configuration,

generate_graphs.py

@@ -12,23 +12,43 @@ import subprocess
 
 PERIODS = {"3hours": "3h", "1day": "1d", "2weeks": "2w", "1year": "1y"}
 COLORS = ["#2563EB", "#DC2626", "#16A34A", "#9333EA"]
+LOAD_COLORS = ["#DC2626", "#F59E0B", "#FACC15"]  # red, amber, yellow
 
 
 def graph(output: Path, rrd: Path, start: str, title: str, unit: str,
-          series: list[tuple[str, str, str]], logarithmic: bool = False) -> None:
+          series: list[tuple[str, str, str]], logarithmic: bool = False,
+          stacked: bool = False, disk_memory: bool = False,
+          network_mixed: bool = False) -> None:
     args = ["rrdtool", "graph", str(output), "--start", f"end-{start}",
             "--end", "now", "--width", "900", "--height", "260",
             "--title", title, "--vertical-label", unit, "--slope-mode",
             "--watermark", "cheap-system-monitor"]
     if logarithmic:
         args += ["--logarithmic", "--lower-limit", "1"]
+    value_suffix = "%%" if unit == "%" else ""
     for index, (ds, label, cf) in enumerate(series):
         var = f"v{index}"
+        if network_mixed and ds == "net_out":
+            drawing = f"AREA:{var}#86EFAC:{label}"
+        elif network_mixed and ds == "net_in":
+            drawing = f"LINE3:{var}#2563EB:{label}"
+        elif disk_memory and ds == "memory":
+            drawing = f"LINE3:{var}#DC2626:{label}"
+        elif disk_memory:
+            # Transparency keeps multiple mountpoints visible when their
+            # percentage areas overlap.
+            drawing = f"AREA:{var}{COLORS[index % len(COLORS)]}80:{label}"
+        elif stacked:
+            stack = ":STACK" if index else ""
+            stacked_colors = LOAD_COLORS if title.startswith("Processor load") else COLORS
+            drawing = f"AREA:{var}{stacked_colors[index]}:{label}{stack}"
+        else:
+            drawing = f"LINE{2 if index < 3 else 1}:{var}{COLORS[index % len(COLORS)]}:{label}"
         args += [f"DEF:{var}={rrd}:{ds}:{cf}",
-                 f"LINE{2 if index < 3 else 1}:{var}{COLORS[index % len(COLORS)]}:{label}",
-                 f"GPRINT:{var}:LAST:Current\\:%8.2lf%s",
-                 f"GPRINT:{var}:AVERAGE:Average\\:%8.2lf%s",
-                 f"GPRINT:{var}:MAX:Maximum\\:%8.2lf%s\\n"]
+                 drawing,
+                 f"GPRINT:{var}:LAST:Current\\:%8.2lf%s{value_suffix}",
+                 f"GPRINT:{var}:AVERAGE:Average\\:%8.2lf%s{value_suffix}",
+                 f"GPRINT:{var}:MAX:Maximum\\:%8.2lf%s{value_suffix}\\n"]
     subprocess.run(args, check=True)
 
 
@@ -49,23 +69,24 @@ def main() -> None:
     if not mounts or mounts[0] != "/":
         mounts.insert(0, "/")
     definitions = [
-        ("load", "Processor load", "load", [("load1", "1 minute", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load15", "15 minutes", "AVERAGE")], False),
-        ("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE")], False),
-        ("network", "Network throughput", "bytes/s", [("net_in", "received", "AVERAGE"), ("net_out", "sent", "AVERAGE")], False),
-        ("disk", "Disk usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)], False),
-        ("memory", "Memory usage", "%", [("memory", "used", "AVERAGE")], False),
+        ("load", "Processor load", "load", [("load1", "1 minute", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load15", "15 minutes", "AVERAGE")], False, True, False, False),
+        ("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE")], False, True, False, False),
+        ("network", "Network throughput", "bytes/s", [("net_out", "sent", "AVERAGE"), ("net_in", "received", "AVERAGE")], False, False, False, True),
+        ("disk", "Disk and memory usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)] + [("memory", "memory", "AVERAGE")], False, False, True, False),
+        ("uptime", "System uptime", "days", [("uptime", "uptime", "AVERAGE")], False, False, False, False),
     ]
     for period_name, start in PERIODS.items():
-        for filename, title, unit, series, logarithmic in definitions:
+        for filename, title, unit, series, logarithmic, stacked, disk_memory, network_mixed in definitions:
             graph(output_dir / f"{filename}-{period_name}.png", rrd, start,
-                  f"{title} — {period_name}", unit, series, logarithmic)
+                  f"{title} — {period_name}", unit, series, logarithmic,
+                  stacked, disk_memory, network_mixed)
     cards = "\n".join(
         f'<section><h2>{html.escape(title)}</h2>' + "".join(
             f'<figure><figcaption>{period}</figcaption>'
             f'<img src="{filename}-{period}.png" alt="{html.escape(title)}, {period}"></figure>'
             for period in PERIODS
         ) + "</section>"
-        for filename, title, _unit, _series, _logarithmic in definitions
+        for filename, title, _unit, _series, _logarithmic, _stacked, _disk_memory, _network_mixed in definitions
     )
     page = f"""<!doctype html>
 <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width">

system_monitor.py

@@ -14,7 +14,7 @@ import time
 DEFAULT_CONFIG = "/opt/cheap_system_monitor/monitor.conf"
 DS_NAMES = (
     "load1", "load5", "load15", "cpu_user", "cpu_system", "cpu_nice",
-    "net_in", "net_out", "disk_root", "disk_1", "disk_2", "memory",
+    "net_in", "net_out", "disk_root", "disk_1", "disk_2", "memory", "uptime",
 )
 
 
@@ -49,6 +49,7 @@ def create_rrd(path: Path) -> None:
         "DS:net_in:DERIVE:180:0:U", "DS:net_out:DERIVE:180:0:U",
         "DS:disk_root:GAUGE:180:0:100", "DS:disk_1:GAUGE:180:0:100",
         "DS:disk_2:GAUGE:180:0:100", "DS:memory:GAUGE:180:0:100",
+        "DS:uptime:GAUGE:180:0:U",
     ]
     # 1-minute data for a day, 5-minute data for two weeks, hourly for a year.
     archives = [
@@ -91,6 +92,11 @@ def memory_percent() -> float:
     return 100.0 * (total - available) / total
 
 
+def uptime_days() -> float:
+    seconds = float(Path("/proc/uptime").read_text().split()[0])
+    return seconds / 86400.0
+
+
 def disk_percent(mountpoint: str) -> float:
     stats = os.statvfs(mountpoint)
     used = stats.f_blocks - stats.f_bfree
@@ -124,7 +130,7 @@ def collect(config: configparser.SectionProxy) -> list[float | int | str]:
         raise SystemExit("At most three mountpoints are supported (including /)")
     disks: list[float | str] = [disk_percent(item) for item in mounts]
     disks.extend(["U"] * (3 - len(disks)))
-    return [*load, *cpu, *net, *disks, memory_percent()]
+    return [*load, *cpu, *net, *disks, memory_percent(), uptime_days()]
 
 
 def main() -> None: