From 177c7479b651a0210b9fcb396f31b50035e0dd57 Mon Sep 17 00:00:00 2001 From: Ignace Date: Wed, 12 Aug 2026 07:59:29 +0200 Subject: [PATCH] adfded temperature, failed services and disk health --- README.md | 27 +++++++- generate_graphs.py | 53 ++++++++++++--- system_monitor.py | 158 +++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 215 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 365b647..fb77d63 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,11 @@ It collects: - disk usage for `/` and up to two additional mountpoints - memory usage - system uptime in days +- CPU temperature and active Raspberry Pi throttling +- failed systemd services and kernel out-of-memory kills +- available filesystem space, inode usage, and read-only filesystem state +- CPU I/O wait +- a combined storage operational-health score from 0 to 10 ## Install @@ -65,9 +70,25 @@ installation naturally needs time to accumulate history; RRD archives are consolidated to five-minute samples for two weeks and hourly samples for a year. The RRD schema is fixed when the database is created. When upgrading from a -version without uptime monitoring, move or remove the existing `system.rrd` -before the next collection if preserving its old history is not required. The -collector will then create a new database containing the uptime data source. +version without the latest monitoring data sources, move or remove the +existing `system.rrd` before the next collection if preserving its old history +is not required. The collector will then create a database with the new data +sources. Temperature is read from Linux thermal zones. Active Raspberry Pi +throttling is read with `vcgencmd`; it appears as unknown on other systems. +Failed services are counted with `systemctl`. OOM kills come from the kernel's +cumulative `/proc/vmstat` counter and are graphed as events per minute. +Storage-health graphs show available space in GiB and inode consumption for +each configured mountpoint. A read-only filesystem is shown at 100% on the +status graph. I/O wait is included in the CPU graph. These portable indicators +work with SD cards and USB drives without requiring SMART support. + +The storage operational-health score summarizes the worst configured +mountpoint. A score of 10 means no current problem is visible, while a +read-only filesystem scores 0. Space and inode usage begin reducing the score +above 75%; either reaches 0 when exhausted. I/O wait above 10% can subtract up +to three points. This is an operational-risk score, not a measurement of flash +wear or remaining media life; SD cards and many USB bridges do not expose the +hardware data needed to estimate those reliably. ## Run from the source tree diff --git a/generate_graphs.py b/generate_graphs.py index f429f41..ee51788 100755 --- a/generate_graphs.py +++ b/generate_graphs.py @@ -18,16 +18,47 @@ LOAD_COLORS = ["#FACC15", "#F59E0B", "#DC2626"] # bottom to top def graph(output: Path, rrd: Path, start: str, title: str, unit: str, series: list[tuple[str, str, str]], logarithmic: bool = False, stacked: bool = False, disk_memory: bool = False, - network_mixed: bool = False) -> None: + network_mixed: bool = False, thermal_mixed: bool = False, + trouble_mixed: bool = False, storage_status: bool = False) -> None: args = ["rrdtool", "graph", str(output), "--start", f"end-{start}", "--end", "now", "--width", "900", "--height", "260", "--title", title, "--vertical-label", unit, "--slope-mode", "--watermark", "cheap-system-monitor"] if logarithmic: args += ["--logarithmic", "--lower-limit", "1"] + if thermal_mixed: + args += ["--lower-limit", "0", "--upper-limit", "100", "--rigid"] + if storage_status: + args += ["--lower-limit", "0", "--upper-limit", "100", "--rigid"] + if any(ds == "storage_health" for ds, _label, _cf in series): + args += ["--lower-limit", "0", "--upper-limit", "10", "--rigid"] value_suffix = "%%" if unit == "%" else "" for index, (ds, label, cf) in enumerate(series): var = f"v{index}" + if thermal_mixed and ds == "throttled": + args += [f"DEF:{var}_raw={rrd}:{ds}:{cf}", + f"CDEF:{var}={var}_raw,100,*", + f"AREA:{var}#DC262640:{label}", + f"GPRINT:{var}_raw:LAST:Current\\:%8.0lf", + f"GPRINT:{var}_raw:AVERAGE:Average\\:%8.2lf", + f"GPRINT:{var}_raw:MAX:Maximum\\:%8.0lf\\n"] + continue + if trouble_mixed and ds == "oom_kills": + args += [f"DEF:{var}_raw={rrd}:{ds}:{cf}", + f"CDEF:{var}={var}_raw,60,*", + f"AREA:{var}#DC262680:{label}", + f"GPRINT:{var}:LAST:Current\\:%8.2lf", + f"GPRINT:{var}:AVERAGE:Average\\:%8.2lf", + f"GPRINT:{var}:MAX:Maximum\\:%8.2lf\\n"] + continue + if storage_status and ds.startswith("readonly_"): + args += [f"DEF:{var}_raw={rrd}:{ds}:{cf}", + f"CDEF:{var}={var}_raw,100,*", + f"LINE2:{var}{COLORS[index % len(COLORS)]}:{label}", + f"GPRINT:{var}_raw:LAST:Current\\:%8.0lf", + f"GPRINT:{var}_raw:AVERAGE:Average\\:%8.2lf", + f"GPRINT:{var}_raw:MAX:Maximum\\:%8.0lf\\n"] + continue if network_mixed and ds == "net_out": drawing = f"AREA:{var}#86EFAC:{label}" elif network_mixed and ds == "net_in": @@ -69,17 +100,23 @@ def main() -> None: if not mounts or mounts[0] != "/": mounts.insert(0, "/") definitions = [ - ("load", "Processor load", "load", [("load15", "15 minutes", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load1", "1 minute", "AVERAGE")], False, True, False, False), - ("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE")], False, True, False, False), - ("network", "Network throughput", "bytes/s", [("net_out", "sent", "AVERAGE"), ("net_in", "received", "AVERAGE")], False, False, False, True), - ("disk", "Disk and memory usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)] + [("memory", "memory", "AVERAGE")], False, False, True, False), - ("uptime", "System uptime", "days", [("uptime", "uptime", "AVERAGE")], False, False, False, False), + ("load", "Processor load", "load", [("load15", "15 minutes", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load1", "1 minute", "AVERAGE")], False, True, False, False, False, False, False), + ("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE"), ("io_wait", "I/O wait", "AVERAGE")], False, True, False, False, False, False, False), + ("thermal", "CPU temperature and throttling", "°C", [("temperature", "temperature", "AVERAGE"), ("throttled", "throttled", "MAX")], False, False, False, False, True, False, False), + ("trouble", "Failed services and OOM kills", "count", [("failed_services", "failed services", "MAX"), ("oom_kills", "OOM kills/min", "MAX")], False, False, False, False, False, True, False), + ("network", "Network throughput", "bytes/s", [("net_out", "sent", "AVERAGE"), ("net_in", "received", "AVERAGE")], False, False, False, True, False, False, False), + ("disk", "Disk and memory usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)] + [("memory", "memory", "AVERAGE")], False, False, True, False, False, False, False), + ("disk-free", "Available filesystem space", "GiB", [(f"free_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)], False, False, False, False, False, False, False), + ("storage-status", "Inode usage and read-only filesystems", "%", [(f"inode_{'root' if i == 0 else i}", f"{mount} inodes", "MAX") for i, mount in enumerate(mounts)] + [(f"readonly_{'root' if i == 0 else i}", f"{mount} read-only", "MAX") for i, mount in enumerate(mounts)], False, False, False, False, False, False, True), + ("storage-health", "Storage operational health", "0–10", [("storage_health", "health score", "MIN")], False, False, False, False, False, False, False), + ("uptime", "System uptime", "days", [("uptime", "uptime", "AVERAGE")], False, False, False, False, False, False, False), ] for period_name, start in PERIODS.items(): - for filename, title, unit, series, logarithmic, stacked, disk_memory, network_mixed in definitions: + for filename, title, unit, series, logarithmic, stacked, disk_memory, network_mixed, thermal_mixed, trouble_mixed, storage_status in definitions: graph(output_dir / f"{filename}-{period_name}.png", rrd, start, f"{title} — {period_name}", unit, series, logarithmic, - stacked, disk_memory, network_mixed) + stacked, disk_memory, network_mixed, thermal_mixed, + trouble_mixed, storage_status) page_files = { "3hours": "index.html", "1day": "1day.html", diff --git a/system_monitor.py b/system_monitor.py index d544326..be1be6b 100755 --- a/system_monitor.py +++ b/system_monitor.py @@ -14,7 +14,13 @@ import time DEFAULT_CONFIG = "/opt/cheap_system_monitor/monitor.conf" DS_NAMES = ( "load1", "load5", "load15", "cpu_user", "cpu_system", "cpu_nice", + "io_wait", "net_in", "net_out", "disk_root", "disk_1", "disk_2", "memory", "uptime", + "temperature", "throttled", + "failed_services", "oom_kills", + "free_root", "free_1", "free_2", "inode_root", "inode_1", "inode_2", + "readonly_root", "readonly_1", "readonly_2", + "storage_health", ) @@ -46,40 +52,53 @@ def create_rrd(path: Path) -> None: "DS:load1:GAUGE:180:0:U", "DS:load5:GAUGE:180:0:U", "DS:load15:GAUGE:180:0:U", "DS:cpu_user:GAUGE:180:0:100", "DS:cpu_system:GAUGE:180:0:100", "DS:cpu_nice:GAUGE:180:0:100", + "DS:io_wait:GAUGE:180:0:100", "DS:net_in:DERIVE:180:0:U", "DS:net_out:DERIVE:180:0:U", "DS:disk_root:GAUGE:180:0:100", "DS:disk_1:GAUGE:180:0:100", "DS:disk_2:GAUGE:180:0:100", "DS:memory:GAUGE:180:0:100", "DS:uptime:GAUGE:180:0:U", + "DS:temperature:GAUGE:180:-50:200", "DS:throttled:GAUGE:180:0:1", + "DS:failed_services:GAUGE:180:0:U", "DS:oom_kills:DERIVE:180:0:U", + "DS:free_root:GAUGE:180:0:U", "DS:free_1:GAUGE:180:0:U", + "DS:free_2:GAUGE:180:0:U", "DS:inode_root:GAUGE:180:0:100", + "DS:inode_1:GAUGE:180:0:100", "DS:inode_2:GAUGE:180:0:100", + "DS:readonly_root:GAUGE:180:0:1", "DS:readonly_1:GAUGE:180:0:1", + "DS:readonly_2:GAUGE:180:0:1", + "DS:storage_health:GAUGE:180:0:10", ] # 1-minute data for a day, 5-minute data for two weeks, hourly for a year. archives = [ "RRA:AVERAGE:0.5:1:1440", "RRA:MAX:0.5:1:1440", + "RRA:MIN:0.5:1:1440", "RRA:AVERAGE:0.5:5:4032", "RRA:MAX:0.5:5:4032", + "RRA:MIN:0.5:5:4032", "RRA:AVERAGE:0.5:60:8784", "RRA:MAX:0.5:60:8784", + "RRA:MIN:0.5:60:8784", ] run_rrdtool("create", str(path), "--step", "60", *ds, *archives) -def read_cpu() -> tuple[int, int, int, int]: +def read_cpu() -> tuple[int, int, int, int, int]: fields = Path("/proc/stat").read_text().splitlines()[0].split() if fields[0] != "cpu" or len(fields) < 8: raise RuntimeError("unexpected /proc/stat format") values = [int(value) for value in fields[1:]] user, nice, system, idle = values[:4] - idle += values[4] if len(values) > 4 else 0 # include iowait + iowait = values[4] if len(values) > 4 else 0 + idle += iowait # Linux reports guest time as already included in user/nice, so only the # first eight fields belong in the total (through steal time). - return user, system, nice, sum(values[:8]) + return user, system, nice, iowait, sum(values[:8]) -def cpu_percentages(sample_seconds: float) -> tuple[float, float, float]: +def cpu_percentages(sample_seconds: float) -> tuple[float, float, float, float]: before = read_cpu() time.sleep(sample_seconds) after = read_cpu() - total = after[3] - before[3] + total = after[4] - before[4] if total <= 0: - return 0.0, 0.0, 0.0 - return tuple(100.0 * (after[i] - before[i]) / total for i in range(3)) + return 0.0, 0.0, 0.0, 0.0 + return tuple(100.0 * (after[i] - before[i]) / total for i in range(4)) def memory_percent() -> float: @@ -97,11 +116,115 @@ def uptime_days() -> float: return seconds / 86400.0 -def disk_percent(mountpoint: str) -> float: +def cpu_temperature() -> float | str: + """Return the CPU temperature in Celsius, or an RRD unknown value.""" + thermal_root = Path("/sys/class/thermal") + if not thermal_root.exists(): + return "U" + + zones = sorted(thermal_root.glob("thermal_zone*")) + preferred = [] + others = [] + for zone in zones: + try: + zone_type = (zone / "type").read_text().strip().lower() + except OSError: + zone_type = "" + target = preferred if any(name in zone_type for name in ("cpu", "soc", "package")) else others + target.append(zone) + + for zone in preferred + others: + try: + value = float((zone / "temp").read_text().strip()) + # Linux thermal-zone temperatures are normally millidegrees. + return value / 1000.0 if abs(value) >= 1000 else value + except (OSError, ValueError): + continue + return "U" + + +def throttle_state() -> int | str: + """Return 1 during active Raspberry Pi throttling, 0 otherwise.""" + try: + result = subprocess.run( + ["vcgencmd", "get_throttled"], check=True, capture_output=True, + text=True, timeout=2, + ) + except (FileNotFoundError, subprocess.CalledProcessError, subprocess.TimeoutExpired): + return "U" + + try: + flags = int(result.stdout.strip().split("=", 1)[1], 16) + except (IndexError, ValueError): + return "U" + # Bits 1, 2, and 3 mean frequency capped, throttled, and soft temperature + # limit active. Ignore the corresponding sticky "has occurred" bits. + return int(bool(flags & 0x0E)) + + +def failed_service_count() -> int | str: + """Return the number of failed systemd service units.""" + try: + result = subprocess.run( + ["systemctl", "list-units", "--state=failed", "--type=service", + "--no-legend", "--plain"], + check=True, capture_output=True, text=True, timeout=5, + ) + except (FileNotFoundError, subprocess.CalledProcessError, subprocess.TimeoutExpired): + return "U" + return sum(bool(line.strip()) for line in result.stdout.splitlines()) + + +def oom_kill_count() -> int | str: + """Return the kernel's cumulative OOM-kill counter since boot.""" + try: + for line in Path("/proc/vmstat").read_text().splitlines(): + name, value = line.split() + if name == "oom_kill": + return int(value) + except (OSError, ValueError): + pass + return "U" + + +def disk_metrics(mountpoint: str) -> tuple[float, float, float | str, int]: + """Return used %, available GiB, inode used %, and read-only status.""" stats = os.statvfs(mountpoint) used = stats.f_blocks - stats.f_bfree available = stats.f_bavail - return 100.0 * used / (used + available) if used + available else 0.0 + used_percent = 100.0 * used / (used + available) if used + available else 0.0 + block_size = stats.f_frsize or stats.f_bsize + free_gib = available * block_size / (1024 ** 3) + used_inodes = stats.f_files - stats.f_ffree + available_inodes = stats.f_favail + inode_total = used_inodes + available_inodes + inode_percent: float | str = ( + 100.0 * used_inodes / inode_total if inode_total else "U" + ) + read_only = int(bool(stats.f_flag & os.ST_RDONLY)) + return used_percent, free_gib, inode_percent, read_only + + +def storage_health_score( + storage: list[tuple[float, float, float | str, int]], + io_wait: float) -> float: + """Score current storage operability from 0 (unusable) to 10 (healthy).""" + if any(read_only for _used, _free, _inodes, read_only in storage): + return 0.0 + + score = 10.0 + for used, _free, inodes, _read_only in storage: + # No penalty through 75%; decline linearly to zero at 100%. + space_score = max(0.0, min(10.0, (100.0 - used) / 25.0 * 10.0)) + score = min(score, space_score) + if isinstance(inodes, float): + inode_score = max(0.0, min(10.0, (100.0 - inodes) / 25.0 * 10.0)) + score = min(score, inode_score) + + # I/O wait is workload-sensitive, so it can reduce the score by at most + # three points: no penalty through 10%, maximum penalty at 50%. + io_penalty = max(0.0, min(3.0, (io_wait - 10.0) / 40.0 * 3.0)) + return max(0.0, score - io_penalty) def network_counters(interface_setting: str) -> tuple[int, int]: @@ -128,9 +251,20 @@ def collect(config: configparser.SectionProxy) -> list[float | int | str]: mounts.insert(0, "/") if len(mounts) > 3: raise SystemExit("At most three mountpoints are supported (including /)") - disks: list[float | str] = [disk_percent(item) for item in mounts] - disks.extend(["U"] * (3 - len(disks))) - return [*load, *cpu, *net, *disks, memory_percent(), uptime_days()] + storage = [disk_metrics(item) for item in mounts] + disks: list[float | str] = [item[0] for item in storage] + free: list[float | str] = [item[1] for item in storage] + inodes: list[float | str] = [item[2] for item in storage] + read_only: list[int | str] = [item[3] for item in storage] + for values in (disks, free, inodes, read_only): + values.extend(["U"] * (3 - len(values))) + return [ + *load, *cpu, *net, *disks, memory_percent(), uptime_days(), + cpu_temperature(), throttle_state(), + failed_service_count(), oom_kill_count(), + *free, *inodes, *read_only, + storage_health_score(storage, cpu[3]), + ] def main() -> None: