diff --git a/README.md b/README.md index fb77d63..e2d27ae 100644 --- a/README.md +++ b/README.md @@ -77,18 +77,19 @@ sources. Temperature is read from Linux thermal zones. Active Raspberry Pi throttling is read with `vcgencmd`; it appears as unknown on other systems. Failed services are counted with `systemctl`. OOM kills come from the kernel's cumulative `/proc/vmstat` counter and are graphed as events per minute. -Storage-health graphs show available space in GiB and inode consumption for -each configured mountpoint. A read-only filesystem is shown at 100% on the -status graph. I/O wait is included in the CPU graph. These portable indicators -work with SD cards and USB drives without requiring SMART support. +Available space, inode consumption, and read-only state are retained as inputs +to the per-mountpoint health score rather than displayed as separate graphs. +I/O wait is included in the CPU graph. These portable indicators work with SD +cards and USB drives without requiring SMART support. -The storage operational-health score summarizes the worst configured -mountpoint. A score of 10 means no current problem is visible, while a -read-only filesystem scores 0. Space and inode usage begin reducing the score -above 75%; either reaches 0 when exhausted. I/O wait above 10% can subtract up -to three points. This is an operational-risk score, not a measurement of flash -wear or remaining media life; SD cards and many USB bridges do not expose the -hardware data needed to estimate those reliably. +The storage operational-health graph shows one score for each configured +mountpoint. A score of 10 means no current problem is visible, while a read-only +filesystem scores 0. Space and inode usage begin reducing that filesystem's +score above 75%; either reaches 0 when exhausted. Host-wide I/O wait above 10% +can subtract up to three points from every score because Linux's CPU wait metric +cannot attribute the delay to one mountpoint. This is an operational-risk score, +not a measurement of flash wear or remaining media life; SD cards and many USB +bridges do not expose the hardware data needed to estimate those reliably. ## Run from the source tree diff --git a/generate_graphs.py b/generate_graphs.py index ee51788..d954f5f 100755 --- a/generate_graphs.py +++ b/generate_graphs.py @@ -19,7 +19,7 @@ def graph(output: Path, rrd: Path, start: str, title: str, unit: str, series: list[tuple[str, str, str]], logarithmic: bool = False, stacked: bool = False, disk_memory: bool = False, network_mixed: bool = False, thermal_mixed: bool = False, - trouble_mixed: bool = False, storage_status: bool = False) -> None: + trouble_mixed: bool = False) -> None: args = ["rrdtool", "graph", str(output), "--start", f"end-{start}", "--end", "now", "--width", "900", "--height", "260", "--title", title, "--vertical-label", unit, "--slope-mode", @@ -28,9 +28,7 @@ def graph(output: Path, rrd: Path, start: str, title: str, unit: str, args += ["--logarithmic", "--lower-limit", "1"] if thermal_mixed: args += ["--lower-limit", "0", "--upper-limit", "100", "--rigid"] - if storage_status: - args += ["--lower-limit", "0", "--upper-limit", "100", "--rigid"] - if any(ds == "storage_health" for ds, _label, _cf in series): + if any(ds.startswith("health_") for ds, _label, _cf in series): args += ["--lower-limit", "0", "--upper-limit", "10", "--rigid"] value_suffix = "%%" if unit == "%" else "" for index, (ds, label, cf) in enumerate(series): @@ -51,14 +49,6 @@ def graph(output: Path, rrd: Path, start: str, title: str, unit: str, f"GPRINT:{var}:AVERAGE:Average\\:%8.2lf", f"GPRINT:{var}:MAX:Maximum\\:%8.2lf\\n"] continue - if storage_status and ds.startswith("readonly_"): - args += [f"DEF:{var}_raw={rrd}:{ds}:{cf}", - f"CDEF:{var}={var}_raw,100,*", - f"LINE2:{var}{COLORS[index % len(COLORS)]}:{label}", - f"GPRINT:{var}_raw:LAST:Current\\:%8.0lf", - f"GPRINT:{var}_raw:AVERAGE:Average\\:%8.2lf", - f"GPRINT:{var}_raw:MAX:Maximum\\:%8.0lf\\n"] - continue if network_mixed and ds == "net_out": drawing = f"AREA:{var}#86EFAC:{label}" elif network_mixed and ds == "net_in": @@ -100,23 +90,21 @@ def main() -> None: if not mounts or mounts[0] != "/": mounts.insert(0, "/") definitions = [ - ("load", "Processor load", "load", [("load15", "15 minutes", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load1", "1 minute", "AVERAGE")], False, True, False, False, False, False, False), - ("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE"), ("io_wait", "I/O wait", "AVERAGE")], False, True, False, False, False, False, False), - ("thermal", "CPU temperature and throttling", "°C", [("temperature", "temperature", "AVERAGE"), ("throttled", "throttled", "MAX")], False, False, False, False, True, False, False), - ("trouble", "Failed services and OOM kills", "count", [("failed_services", "failed services", "MAX"), ("oom_kills", "OOM kills/min", "MAX")], False, False, False, False, False, True, False), - ("network", "Network throughput", "bytes/s", [("net_out", "sent", "AVERAGE"), ("net_in", "received", "AVERAGE")], False, False, False, True, False, False, False), - ("disk", "Disk and memory usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)] + [("memory", "memory", "AVERAGE")], False, False, True, False, False, False, False), - ("disk-free", "Available filesystem space", "GiB", [(f"free_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)], False, False, False, False, False, False, False), - ("storage-status", "Inode usage and read-only filesystems", "%", [(f"inode_{'root' if i == 0 else i}", f"{mount} inodes", "MAX") for i, mount in enumerate(mounts)] + [(f"readonly_{'root' if i == 0 else i}", f"{mount} read-only", "MAX") for i, mount in enumerate(mounts)], False, False, False, False, False, False, True), - ("storage-health", "Storage operational health", "0–10", [("storage_health", "health score", "MIN")], False, False, False, False, False, False, False), - ("uptime", "System uptime", "days", [("uptime", "uptime", "AVERAGE")], False, False, False, False, False, False, False), + ("load", "Processor load", "load", [("load15", "15 minutes", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load1", "1 minute", "AVERAGE")], False, True, False, False, False, False), + ("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE"), ("io_wait", "I/O wait", "AVERAGE")], False, True, False, False, False, False), + ("thermal", "CPU temperature and throttling", "°C", [("temperature", "temperature", "AVERAGE"), ("throttled", "throttled", "MAX")], False, False, False, False, True, False), + ("trouble", "Failed services and OOM kills", "count", [("failed_services", "failed services", "MAX"), ("oom_kills", "OOM kills/min", "MAX")], False, False, False, False, False, True), + ("network", "Network throughput", "bytes/s", [("net_out", "sent", "AVERAGE"), ("net_in", "received", "AVERAGE")], False, False, False, True, False, False), + ("disk", "Disk and memory usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)] + [("memory", "memory", "AVERAGE")], False, False, True, False, False, False), + ("storage-health", "Storage operational health", "0–10", [(f"health_{'root' if i == 0 else i}", mount, "MIN") for i, mount in enumerate(mounts)], False, False, False, False, False, False), + ("uptime", "System uptime", "days", [("uptime", "uptime", "AVERAGE")], False, False, False, False, False, False), ] for period_name, start in PERIODS.items(): - for filename, title, unit, series, logarithmic, stacked, disk_memory, network_mixed, thermal_mixed, trouble_mixed, storage_status in definitions: + for filename, title, unit, series, logarithmic, stacked, disk_memory, network_mixed, thermal_mixed, trouble_mixed in definitions: graph(output_dir / f"{filename}-{period_name}.png", rrd, start, f"{title} — {period_name}", unit, series, logarithmic, stacked, disk_memory, network_mixed, thermal_mixed, - trouble_mixed, storage_status) + trouble_mixed) page_files = { "3hours": "index.html", "1day": "1day.html", diff --git a/system_monitor.py b/system_monitor.py index be1be6b..821e38c 100755 --- a/system_monitor.py +++ b/system_monitor.py @@ -20,7 +20,7 @@ DS_NAMES = ( "failed_services", "oom_kills", "free_root", "free_1", "free_2", "inode_root", "inode_1", "inode_2", "readonly_root", "readonly_1", "readonly_2", - "storage_health", + "health_root", "health_1", "health_2", ) @@ -64,7 +64,8 @@ def create_rrd(path: Path) -> None: "DS:inode_1:GAUGE:180:0:100", "DS:inode_2:GAUGE:180:0:100", "DS:readonly_root:GAUGE:180:0:1", "DS:readonly_1:GAUGE:180:0:1", "DS:readonly_2:GAUGE:180:0:1", - "DS:storage_health:GAUGE:180:0:10", + "DS:health_root:GAUGE:180:0:10", "DS:health_1:GAUGE:180:0:10", + "DS:health_2:GAUGE:180:0:10", ] # 1-minute data for a day, 5-minute data for two weeks, hourly for a year. archives = [ @@ -206,20 +207,18 @@ def disk_metrics(mountpoint: str) -> tuple[float, float, float | str, int]: def storage_health_score( - storage: list[tuple[float, float, float | str, int]], + storage: tuple[float, float, float | str, int], io_wait: float) -> float: - """Score current storage operability from 0 (unusable) to 10 (healthy).""" - if any(read_only for _used, _free, _inodes, read_only in storage): + """Score one filesystem's operability from 0 (unusable) to 10 (healthy).""" + used, _free, inodes, read_only = storage + if read_only: return 0.0 - score = 10.0 - for used, _free, inodes, _read_only in storage: - # No penalty through 75%; decline linearly to zero at 100%. - space_score = max(0.0, min(10.0, (100.0 - used) / 25.0 * 10.0)) - score = min(score, space_score) - if isinstance(inodes, float): - inode_score = max(0.0, min(10.0, (100.0 - inodes) / 25.0 * 10.0)) - score = min(score, inode_score) + # No penalty through 75%; decline linearly to zero at 100%. + score = max(0.0, min(10.0, (100.0 - used) / 25.0 * 10.0)) + if isinstance(inodes, float): + inode_score = max(0.0, min(10.0, (100.0 - inodes) / 25.0 * 10.0)) + score = min(score, inode_score) # I/O wait is workload-sensitive, so it can reduce the score by at most # three points: no penalty through 10%, maximum penalty at 50%. @@ -258,12 +257,16 @@ def collect(config: configparser.SectionProxy) -> list[float | int | str]: read_only: list[int | str] = [item[3] for item in storage] for values in (disks, free, inodes, read_only): values.extend(["U"] * (3 - len(values))) + health: list[float | str] = [ + storage_health_score(item, cpu[3]) for item in storage + ] + health.extend(["U"] * (3 - len(health))) return [ *load, *cpu, *net, *disks, memory_percent(), uptime_days(), cpu_temperature(), throttle_state(), failed_service_count(), oom_kill_count(), *free, *inodes, *read_only, - storage_health_score(storage, cpu[3]), + *health, ]