graphing optimized

README.md

@@ -77,18 +77,19 @@ sources. Temperature is read from Linux thermal zones. Active Raspberry Pi
 throttling is read with `vcgencmd`; it appears as unknown on other systems.
 Failed services are counted with `systemctl`. OOM kills come from the kernel's
 cumulative `/proc/vmstat` counter and are graphed as events per minute.
-Storage-health graphs show available space in GiB and inode consumption for
+Available space, inode consumption, and read-only state are retained as inputs
-each configured mountpoint. A read-only filesystem is shown at 100% on the
+to the per-mountpoint health score rather than displayed as separate graphs.
-status graph. I/O wait is included in the CPU graph. These portable indicators
+I/O wait is included in the CPU graph. These portable indicators work with SD
-work with SD cards and USB drives without requiring SMART support.
+cards and USB drives without requiring SMART support.
 
-The storage operational-health score summarizes the worst configured
+The storage operational-health graph shows one score for each configured
-mountpoint. A score of 10 means no current problem is visible, while a
+mountpoint. A score of 10 means no current problem is visible, while a read-only
-read-only filesystem scores 0. Space and inode usage begin reducing the score
+filesystem scores 0. Space and inode usage begin reducing that filesystem's
-above 75%; either reaches 0 when exhausted. I/O wait above 10% can subtract up
+score above 75%; either reaches 0 when exhausted. Host-wide I/O wait above 10%
-to three points. This is an operational-risk score, not a measurement of flash
+can subtract up to three points from every score because Linux's CPU wait metric
-wear or remaining media life; SD cards and many USB bridges do not expose the
+cannot attribute the delay to one mountpoint. This is an operational-risk score,
-hardware data needed to estimate those reliably.
+not a measurement of flash wear or remaining media life; SD cards and many USB
+bridges do not expose the hardware data needed to estimate those reliably.
 
 ## Run from the source tree
 

generate_graphs.py

@@ -19,7 +19,7 @@ def graph(output: Path, rrd: Path, start: str, title: str, unit: str,
           series: list[tuple[str, str, str]], logarithmic: bool = False,
           stacked: bool = False, disk_memory: bool = False,
           network_mixed: bool = False, thermal_mixed: bool = False,
-          trouble_mixed: bool = False, storage_status: bool = False) -> None:
+          trouble_mixed: bool = False) -> None:
     args = ["rrdtool", "graph", str(output), "--start", f"end-{start}",
             "--end", "now", "--width", "900", "--height", "260",
             "--title", title, "--vertical-label", unit, "--slope-mode",
@@ -28,9 +28,7 @@ def graph(output: Path, rrd: Path, start: str, title: str, unit: str,
         args += ["--logarithmic", "--lower-limit", "1"]
     if thermal_mixed:
         args += ["--lower-limit", "0", "--upper-limit", "100", "--rigid"]
-    if storage_status:
+    if any(ds.startswith("health_") for ds, _label, _cf in series):
-        args += ["--lower-limit", "0", "--upper-limit", "100", "--rigid"]
-    if any(ds == "storage_health" for ds, _label, _cf in series):
         args += ["--lower-limit", "0", "--upper-limit", "10", "--rigid"]
     value_suffix = "%%" if unit == "%" else ""
     for index, (ds, label, cf) in enumerate(series):
@@ -51,14 +49,6 @@ def graph(output: Path, rrd: Path, start: str, title: str, unit: str,
                      f"GPRINT:{var}:AVERAGE:Average\\:%8.2lf",
                      f"GPRINT:{var}:MAX:Maximum\\:%8.2lf\\n"]
             continue
-        if storage_status and ds.startswith("readonly_"):
-            args += [f"DEF:{var}_raw={rrd}:{ds}:{cf}",
-                     f"CDEF:{var}={var}_raw,100,*",
-                     f"LINE2:{var}{COLORS[index % len(COLORS)]}:{label}",
-                     f"GPRINT:{var}_raw:LAST:Current\\:%8.0lf",
-                     f"GPRINT:{var}_raw:AVERAGE:Average\\:%8.2lf",
-                     f"GPRINT:{var}_raw:MAX:Maximum\\:%8.0lf\\n"]
-            continue
         if network_mixed and ds == "net_out":
             drawing = f"AREA:{var}#86EFAC:{label}"
         elif network_mixed and ds == "net_in":
@@ -100,23 +90,21 @@ def main() -> None:
     if not mounts or mounts[0] != "/":
         mounts.insert(0, "/")
     definitions = [
-        ("load", "Processor load", "load", [("load15", "15 minutes", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load1", "1 minute", "AVERAGE")], False, True, False, False, False, False, False),
+        ("load", "Processor load", "load", [("load15", "15 minutes", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load1", "1 minute", "AVERAGE")], False, True, False, False, False, False),
-        ("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE"), ("io_wait", "I/O wait", "AVERAGE")], False, True, False, False, False, False, False),
+        ("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE"), ("io_wait", "I/O wait", "AVERAGE")], False, True, False, False, False, False),
-        ("thermal", "CPU temperature and throttling", "°C", [("temperature", "temperature", "AVERAGE"), ("throttled", "throttled", "MAX")], False, False, False, False, True, False, False),
+        ("thermal", "CPU temperature and throttling", "°C", [("temperature", "temperature", "AVERAGE"), ("throttled", "throttled", "MAX")], False, False, False, False, True, False),
-        ("trouble", "Failed services and OOM kills", "count", [("failed_services", "failed services", "MAX"), ("oom_kills", "OOM kills/min", "MAX")], False, False, False, False, False, True, False),
+        ("trouble", "Failed services and OOM kills", "count", [("failed_services", "failed services", "MAX"), ("oom_kills", "OOM kills/min", "MAX")], False, False, False, False, False, True),
-        ("network", "Network throughput", "bytes/s", [("net_out", "sent", "AVERAGE"), ("net_in", "received", "AVERAGE")], False, False, False, True, False, False, False),
+        ("network", "Network throughput", "bytes/s", [("net_out", "sent", "AVERAGE"), ("net_in", "received", "AVERAGE")], False, False, False, True, False, False),
-        ("disk", "Disk and memory usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)] + [("memory", "memory", "AVERAGE")], False, False, True, False, False, False, False),
+        ("disk", "Disk and memory usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)] + [("memory", "memory", "AVERAGE")], False, False, True, False, False, False),
-        ("disk-free", "Available filesystem space", "GiB", [(f"free_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)], False, False, False, False, False, False, False),
+        ("storage-health", "Storage operational health", "0–10", [(f"health_{'root' if i == 0 else i}", mount, "MIN") for i, mount in enumerate(mounts)], False, False, False, False, False, False),
-        ("storage-status", "Inode usage and read-only filesystems", "%", [(f"inode_{'root' if i == 0 else i}", f"{mount} inodes", "MAX") for i, mount in enumerate(mounts)] + [(f"readonly_{'root' if i == 0 else i}", f"{mount} read-only", "MAX") for i, mount in enumerate(mounts)], False, False, False, False, False, False, True),
+        ("uptime", "System uptime", "days", [("uptime", "uptime", "AVERAGE")], False, False, False, False, False, False),
-        ("storage-health", "Storage operational health", "0–10", [("storage_health", "health score", "MIN")], False, False, False, False, False, False, False),
-        ("uptime", "System uptime", "days", [("uptime", "uptime", "AVERAGE")], False, False, False, False, False, False, False),
     ]
     for period_name, start in PERIODS.items():
-        for filename, title, unit, series, logarithmic, stacked, disk_memory, network_mixed, thermal_mixed, trouble_mixed, storage_status in definitions:
+        for filename, title, unit, series, logarithmic, stacked, disk_memory, network_mixed, thermal_mixed, trouble_mixed in definitions:
             graph(output_dir / f"{filename}-{period_name}.png", rrd, start,
                   f"{title} — {period_name}", unit, series, logarithmic,
                   stacked, disk_memory, network_mixed, thermal_mixed,
-                  trouble_mixed, storage_status)
+                  trouble_mixed)
     page_files = {
         "3hours": "index.html",
         "1day": "1day.html",

system_monitor.py

@@ -20,7 +20,7 @@ DS_NAMES = (
     "failed_services", "oom_kills",
     "free_root", "free_1", "free_2", "inode_root", "inode_1", "inode_2",
     "readonly_root", "readonly_1", "readonly_2",
-    "storage_health",
+    "health_root", "health_1", "health_2",
 )
 
 
@@ -64,7 +64,8 @@ def create_rrd(path: Path) -> None:
         "DS:inode_1:GAUGE:180:0:100", "DS:inode_2:GAUGE:180:0:100",
         "DS:readonly_root:GAUGE:180:0:1", "DS:readonly_1:GAUGE:180:0:1",
         "DS:readonly_2:GAUGE:180:0:1",
-        "DS:storage_health:GAUGE:180:0:10",
+        "DS:health_root:GAUGE:180:0:10", "DS:health_1:GAUGE:180:0:10",
+        "DS:health_2:GAUGE:180:0:10",
     ]
     # 1-minute data for a day, 5-minute data for two weeks, hourly for a year.
     archives = [
@@ -206,20 +207,18 @@ def disk_metrics(mountpoint: str) -> tuple[float, float, float | str, int]:
 
 
 def storage_health_score(
-        storage: list[tuple[float, float, float | str, int]],
+        storage: tuple[float, float, float | str, int],
         io_wait: float) -> float:
-    """Score current storage operability from 0 (unusable) to 10 (healthy)."""
+    """Score one filesystem's operability from 0 (unusable) to 10 (healthy)."""
-    if any(read_only for _used, _free, _inodes, read_only in storage):
+    used, _free, inodes, read_only = storage
+    if read_only:
         return 0.0
 
-    score = 10.0
+    # No penalty through 75%; decline linearly to zero at 100%.
-    for used, _free, inodes, _read_only in storage:
+    score = max(0.0, min(10.0, (100.0 - used) / 25.0 * 10.0))
-        # No penalty through 75%; decline linearly to zero at 100%.
+    if isinstance(inodes, float):
-        space_score = max(0.0, min(10.0, (100.0 - used) / 25.0 * 10.0))
+        inode_score = max(0.0, min(10.0, (100.0 - inodes) / 25.0 * 10.0))
-        score = min(score, space_score)
+        score = min(score, inode_score)
-        if isinstance(inodes, float):
-            inode_score = max(0.0, min(10.0, (100.0 - inodes) / 25.0 * 10.0))
-            score = min(score, inode_score)
 
     # I/O wait is workload-sensitive, so it can reduce the score by at most
     # three points: no penalty through 10%, maximum penalty at 50%.
@@ -258,12 +257,16 @@ def collect(config: configparser.SectionProxy) -> list[float | int | str]:
     read_only: list[int | str] = [item[3] for item in storage]
     for values in (disks, free, inodes, read_only):
         values.extend(["U"] * (3 - len(values)))
+    health: list[float | str] = [
+        storage_health_score(item, cpu[3]) for item in storage
+    ]
+    health.extend(["U"] * (3 - len(health)))
     return [
         *load, *cpu, *net, *disks, memory_percent(), uptime_days(),
         cpu_temperature(), throttle_state(),
         failed_service_count(), oom_kill_count(),
         *free, *inodes, *read_only,
-        storage_health_score(storage, cpu[3]),
+        *health,
     ]