adfded temperature, failed services and disk health

README.md

@@ -12,6 +12,11 @@ It collects:
 - disk usage for `/` and up to two additional mountpoints
 - memory usage
 - system uptime in days
+- CPU temperature and active Raspberry Pi throttling
+- failed systemd services and kernel out-of-memory kills
+- available filesystem space, inode usage, and read-only filesystem state
+- CPU I/O wait
+- a combined storage operational-health score from 0 to 10
 
 ## Install
 
@@ -65,9 +70,25 @@ installation naturally needs time to accumulate history; RRD archives are
 consolidated to five-minute samples for two weeks and hourly samples for a year.
 
 The RRD schema is fixed when the database is created. When upgrading from a
-version without uptime monitoring, move or remove the existing `system.rrd`
+version without the latest monitoring data sources, move or remove the
-before the next collection if preserving its old history is not required. The
+existing `system.rrd` before the next collection if preserving its old history
-collector will then create a new database containing the uptime data source.
+is not required. The collector will then create a database with the new data
+sources. Temperature is read from Linux thermal zones. Active Raspberry Pi
+throttling is read with `vcgencmd`; it appears as unknown on other systems.
+Failed services are counted with `systemctl`. OOM kills come from the kernel's
+cumulative `/proc/vmstat` counter and are graphed as events per minute.
+Storage-health graphs show available space in GiB and inode consumption for
+each configured mountpoint. A read-only filesystem is shown at 100% on the
+status graph. I/O wait is included in the CPU graph. These portable indicators
+work with SD cards and USB drives without requiring SMART support.
+
+The storage operational-health score summarizes the worst configured
+mountpoint. A score of 10 means no current problem is visible, while a
+read-only filesystem scores 0. Space and inode usage begin reducing the score
+above 75%; either reaches 0 when exhausted. I/O wait above 10% can subtract up
+to three points. This is an operational-risk score, not a measurement of flash
+wear or remaining media life; SD cards and many USB bridges do not expose the
+hardware data needed to estimate those reliably.
 
 ## Run from the source tree
 

generate_graphs.py

@@ -18,16 +18,47 @@ LOAD_COLORS = ["#FACC15", "#F59E0B", "#DC2626"]  # bottom to top
 def graph(output: Path, rrd: Path, start: str, title: str, unit: str,
           series: list[tuple[str, str, str]], logarithmic: bool = False,
           stacked: bool = False, disk_memory: bool = False,
-          network_mixed: bool = False) -> None:
+          network_mixed: bool = False, thermal_mixed: bool = False,
+          trouble_mixed: bool = False, storage_status: bool = False) -> None:
     args = ["rrdtool", "graph", str(output), "--start", f"end-{start}",
             "--end", "now", "--width", "900", "--height", "260",
             "--title", title, "--vertical-label", unit, "--slope-mode",
             "--watermark", "cheap-system-monitor"]
     if logarithmic:
         args += ["--logarithmic", "--lower-limit", "1"]
+    if thermal_mixed:
+        args += ["--lower-limit", "0", "--upper-limit", "100", "--rigid"]
+    if storage_status:
+        args += ["--lower-limit", "0", "--upper-limit", "100", "--rigid"]
+    if any(ds == "storage_health" for ds, _label, _cf in series):
+        args += ["--lower-limit", "0", "--upper-limit", "10", "--rigid"]
     value_suffix = "%%" if unit == "%" else ""
     for index, (ds, label, cf) in enumerate(series):
         var = f"v{index}"
+        if thermal_mixed and ds == "throttled":
+            args += [f"DEF:{var}_raw={rrd}:{ds}:{cf}",
+                     f"CDEF:{var}={var}_raw,100,*",
+                     f"AREA:{var}#DC262640:{label}",
+                     f"GPRINT:{var}_raw:LAST:Current\\:%8.0lf",
+                     f"GPRINT:{var}_raw:AVERAGE:Average\\:%8.2lf",
+                     f"GPRINT:{var}_raw:MAX:Maximum\\:%8.0lf\\n"]
+            continue
+        if trouble_mixed and ds == "oom_kills":
+            args += [f"DEF:{var}_raw={rrd}:{ds}:{cf}",
+                     f"CDEF:{var}={var}_raw,60,*",
+                     f"AREA:{var}#DC262680:{label}",
+                     f"GPRINT:{var}:LAST:Current\\:%8.2lf",
+                     f"GPRINT:{var}:AVERAGE:Average\\:%8.2lf",
+                     f"GPRINT:{var}:MAX:Maximum\\:%8.2lf\\n"]
+            continue
+        if storage_status and ds.startswith("readonly_"):
+            args += [f"DEF:{var}_raw={rrd}:{ds}:{cf}",
+                     f"CDEF:{var}={var}_raw,100,*",
+                     f"LINE2:{var}{COLORS[index % len(COLORS)]}:{label}",
+                     f"GPRINT:{var}_raw:LAST:Current\\:%8.0lf",
+                     f"GPRINT:{var}_raw:AVERAGE:Average\\:%8.2lf",
+                     f"GPRINT:{var}_raw:MAX:Maximum\\:%8.0lf\\n"]
+            continue
         if network_mixed and ds == "net_out":
             drawing = f"AREA:{var}#86EFAC:{label}"
         elif network_mixed and ds == "net_in":
@@ -69,17 +100,23 @@ def main() -> None:
     if not mounts or mounts[0] != "/":
         mounts.insert(0, "/")
     definitions = [
-        ("load", "Processor load", "load", [("load15", "15 minutes", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load1", "1 minute", "AVERAGE")], False, True, False, False),
+        ("load", "Processor load", "load", [("load15", "15 minutes", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load1", "1 minute", "AVERAGE")], False, True, False, False, False, False, False),
-        ("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE")], False, True, False, False),
+        ("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE"), ("io_wait", "I/O wait", "AVERAGE")], False, True, False, False, False, False, False),
-        ("network", "Network throughput", "bytes/s", [("net_out", "sent", "AVERAGE"), ("net_in", "received", "AVERAGE")], False, False, False, True),
+        ("thermal", "CPU temperature and throttling", "°C", [("temperature", "temperature", "AVERAGE"), ("throttled", "throttled", "MAX")], False, False, False, False, True, False, False),
-        ("disk", "Disk and memory usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)] + [("memory", "memory", "AVERAGE")], False, False, True, False),
+        ("trouble", "Failed services and OOM kills", "count", [("failed_services", "failed services", "MAX"), ("oom_kills", "OOM kills/min", "MAX")], False, False, False, False, False, True, False),
-        ("uptime", "System uptime", "days", [("uptime", "uptime", "AVERAGE")], False, False, False, False),
+        ("network", "Network throughput", "bytes/s", [("net_out", "sent", "AVERAGE"), ("net_in", "received", "AVERAGE")], False, False, False, True, False, False, False),
+        ("disk", "Disk and memory usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)] + [("memory", "memory", "AVERAGE")], False, False, True, False, False, False, False),
+        ("disk-free", "Available filesystem space", "GiB", [(f"free_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)], False, False, False, False, False, False, False),
+        ("storage-status", "Inode usage and read-only filesystems", "%", [(f"inode_{'root' if i == 0 else i}", f"{mount} inodes", "MAX") for i, mount in enumerate(mounts)] + [(f"readonly_{'root' if i == 0 else i}", f"{mount} read-only", "MAX") for i, mount in enumerate(mounts)], False, False, False, False, False, False, True),
+        ("storage-health", "Storage operational health", "0–10", [("storage_health", "health score", "MIN")], False, False, False, False, False, False, False),
+        ("uptime", "System uptime", "days", [("uptime", "uptime", "AVERAGE")], False, False, False, False, False, False, False),
     ]
     for period_name, start in PERIODS.items():
-        for filename, title, unit, series, logarithmic, stacked, disk_memory, network_mixed in definitions:
+        for filename, title, unit, series, logarithmic, stacked, disk_memory, network_mixed, thermal_mixed, trouble_mixed, storage_status in definitions:
             graph(output_dir / f"{filename}-{period_name}.png", rrd, start,
                   f"{title} — {period_name}", unit, series, logarithmic,
-                  stacked, disk_memory, network_mixed)
+                  stacked, disk_memory, network_mixed, thermal_mixed,
+                  trouble_mixed, storage_status)
     page_files = {
         "3hours": "index.html",
         "1day": "1day.html",

system_monitor.py

@@ -14,7 +14,13 @@ import time
 DEFAULT_CONFIG = "/opt/cheap_system_monitor/monitor.conf"
 DS_NAMES = (
     "load1", "load5", "load15", "cpu_user", "cpu_system", "cpu_nice",
+    "io_wait",
     "net_in", "net_out", "disk_root", "disk_1", "disk_2", "memory", "uptime",
+    "temperature", "throttled",
+    "failed_services", "oom_kills",
+    "free_root", "free_1", "free_2", "inode_root", "inode_1", "inode_2",
+    "readonly_root", "readonly_1", "readonly_2",
+    "storage_health",
 )
 
 
@@ -46,40 +52,53 @@ def create_rrd(path: Path) -> None:
         "DS:load1:GAUGE:180:0:U", "DS:load5:GAUGE:180:0:U",
         "DS:load15:GAUGE:180:0:U", "DS:cpu_user:GAUGE:180:0:100",
         "DS:cpu_system:GAUGE:180:0:100", "DS:cpu_nice:GAUGE:180:0:100",
+        "DS:io_wait:GAUGE:180:0:100",
         "DS:net_in:DERIVE:180:0:U", "DS:net_out:DERIVE:180:0:U",
         "DS:disk_root:GAUGE:180:0:100", "DS:disk_1:GAUGE:180:0:100",
         "DS:disk_2:GAUGE:180:0:100", "DS:memory:GAUGE:180:0:100",
         "DS:uptime:GAUGE:180:0:U",
+        "DS:temperature:GAUGE:180:-50:200", "DS:throttled:GAUGE:180:0:1",
+        "DS:failed_services:GAUGE:180:0:U", "DS:oom_kills:DERIVE:180:0:U",
+        "DS:free_root:GAUGE:180:0:U", "DS:free_1:GAUGE:180:0:U",
+        "DS:free_2:GAUGE:180:0:U", "DS:inode_root:GAUGE:180:0:100",
+        "DS:inode_1:GAUGE:180:0:100", "DS:inode_2:GAUGE:180:0:100",
+        "DS:readonly_root:GAUGE:180:0:1", "DS:readonly_1:GAUGE:180:0:1",
+        "DS:readonly_2:GAUGE:180:0:1",
+        "DS:storage_health:GAUGE:180:0:10",
     ]
     # 1-minute data for a day, 5-minute data for two weeks, hourly for a year.
     archives = [
         "RRA:AVERAGE:0.5:1:1440", "RRA:MAX:0.5:1:1440",
+        "RRA:MIN:0.5:1:1440",
         "RRA:AVERAGE:0.5:5:4032", "RRA:MAX:0.5:5:4032",
+        "RRA:MIN:0.5:5:4032",
         "RRA:AVERAGE:0.5:60:8784", "RRA:MAX:0.5:60:8784",
+        "RRA:MIN:0.5:60:8784",
     ]
     run_rrdtool("create", str(path), "--step", "60", *ds, *archives)
 
 
-def read_cpu() -> tuple[int, int, int, int]:
+def read_cpu() -> tuple[int, int, int, int, int]:
     fields = Path("/proc/stat").read_text().splitlines()[0].split()
     if fields[0] != "cpu" or len(fields) < 8:
         raise RuntimeError("unexpected /proc/stat format")
     values = [int(value) for value in fields[1:]]
     user, nice, system, idle = values[:4]
-    idle += values[4] if len(values) > 4 else 0  # include iowait
+    iowait = values[4] if len(values) > 4 else 0
+    idle += iowait
     # Linux reports guest time as already included in user/nice, so only the
     # first eight fields belong in the total (through steal time).
-    return user, system, nice, sum(values[:8])
+    return user, system, nice, iowait, sum(values[:8])
 
 
-def cpu_percentages(sample_seconds: float) -> tuple[float, float, float]:
+def cpu_percentages(sample_seconds: float) -> tuple[float, float, float, float]:
     before = read_cpu()
     time.sleep(sample_seconds)
     after = read_cpu()
-    total = after[3] - before[3]
+    total = after[4] - before[4]
     if total <= 0:
-        return 0.0, 0.0, 0.0
+        return 0.0, 0.0, 0.0, 0.0
-    return tuple(100.0 * (after[i] - before[i]) / total for i in range(3))
+    return tuple(100.0 * (after[i] - before[i]) / total for i in range(4))
 
 
 def memory_percent() -> float:
@@ -97,11 +116,115 @@ def uptime_days() -> float:
     return seconds / 86400.0
 
 
-def disk_percent(mountpoint: str) -> float:
+def cpu_temperature() -> float | str:
+    """Return the CPU temperature in Celsius, or an RRD unknown value."""
+    thermal_root = Path("/sys/class/thermal")
+    if not thermal_root.exists():
+        return "U"
+
+    zones = sorted(thermal_root.glob("thermal_zone*"))
+    preferred = []
+    others = []
+    for zone in zones:
+        try:
+            zone_type = (zone / "type").read_text().strip().lower()
+        except OSError:
+            zone_type = ""
+        target = preferred if any(name in zone_type for name in ("cpu", "soc", "package")) else others
+        target.append(zone)
+
+    for zone in preferred + others:
+        try:
+            value = float((zone / "temp").read_text().strip())
+            # Linux thermal-zone temperatures are normally millidegrees.
+            return value / 1000.0 if abs(value) >= 1000 else value
+        except (OSError, ValueError):
+            continue
+    return "U"
+
+
+def throttle_state() -> int | str:
+    """Return 1 during active Raspberry Pi throttling, 0 otherwise."""
+    try:
+        result = subprocess.run(
+            ["vcgencmd", "get_throttled"], check=True, capture_output=True,
+            text=True, timeout=2,
+        )
+    except (FileNotFoundError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
+        return "U"
+
+    try:
+        flags = int(result.stdout.strip().split("=", 1)[1], 16)
+    except (IndexError, ValueError):
+        return "U"
+    # Bits 1, 2, and 3 mean frequency capped, throttled, and soft temperature
+    # limit active. Ignore the corresponding sticky "has occurred" bits.
+    return int(bool(flags & 0x0E))
+
+
+def failed_service_count() -> int | str:
+    """Return the number of failed systemd service units."""
+    try:
+        result = subprocess.run(
+            ["systemctl", "list-units", "--state=failed", "--type=service",
+             "--no-legend", "--plain"],
+            check=True, capture_output=True, text=True, timeout=5,
+        )
+    except (FileNotFoundError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
+        return "U"
+    return sum(bool(line.strip()) for line in result.stdout.splitlines())
+
+
+def oom_kill_count() -> int | str:
+    """Return the kernel's cumulative OOM-kill counter since boot."""
+    try:
+        for line in Path("/proc/vmstat").read_text().splitlines():
+            name, value = line.split()
+            if name == "oom_kill":
+                return int(value)
+    except (OSError, ValueError):
+        pass
+    return "U"
+
+
+def disk_metrics(mountpoint: str) -> tuple[float, float, float | str, int]:
+    """Return used %, available GiB, inode used %, and read-only status."""
     stats = os.statvfs(mountpoint)
     used = stats.f_blocks - stats.f_bfree
     available = stats.f_bavail
-    return 100.0 * used / (used + available) if used + available else 0.0
+    used_percent = 100.0 * used / (used + available) if used + available else 0.0
+    block_size = stats.f_frsize or stats.f_bsize
+    free_gib = available * block_size / (1024 ** 3)
+    used_inodes = stats.f_files - stats.f_ffree
+    available_inodes = stats.f_favail
+    inode_total = used_inodes + available_inodes
+    inode_percent: float | str = (
+        100.0 * used_inodes / inode_total if inode_total else "U"
+    )
+    read_only = int(bool(stats.f_flag & os.ST_RDONLY))
+    return used_percent, free_gib, inode_percent, read_only
+
+
+def storage_health_score(
+        storage: list[tuple[float, float, float | str, int]],
+        io_wait: float) -> float:
+    """Score current storage operability from 0 (unusable) to 10 (healthy)."""
+    if any(read_only for _used, _free, _inodes, read_only in storage):
+        return 0.0
+
+    score = 10.0
+    for used, _free, inodes, _read_only in storage:
+        # No penalty through 75%; decline linearly to zero at 100%.
+        space_score = max(0.0, min(10.0, (100.0 - used) / 25.0 * 10.0))
+        score = min(score, space_score)
+        if isinstance(inodes, float):
+            inode_score = max(0.0, min(10.0, (100.0 - inodes) / 25.0 * 10.0))
+            score = min(score, inode_score)
+
+    # I/O wait is workload-sensitive, so it can reduce the score by at most
+    # three points: no penalty through 10%, maximum penalty at 50%.
+    io_penalty = max(0.0, min(3.0, (io_wait - 10.0) / 40.0 * 3.0))
+    return max(0.0, score - io_penalty)
 
 
 def network_counters(interface_setting: str) -> tuple[int, int]:
@@ -128,9 +251,20 @@ def collect(config: configparser.SectionProxy) -> list[float | int | str]:
         mounts.insert(0, "/")
     if len(mounts) > 3:
         raise SystemExit("At most three mountpoints are supported (including /)")
-    disks: list[float | str] = [disk_percent(item) for item in mounts]
+    storage = [disk_metrics(item) for item in mounts]
-    disks.extend(["U"] * (3 - len(disks)))
+    disks: list[float | str] = [item[0] for item in storage]
-    return [*load, *cpu, *net, *disks, memory_percent(), uptime_days()]
+    free: list[float | str] = [item[1] for item in storage]
+    inodes: list[float | str] = [item[2] for item in storage]
+    read_only: list[int | str] = [item[3] for item in storage]
+    for values in (disks, free, inodes, read_only):
+        values.extend(["U"] * (3 - len(values)))
+    return [
+        *load, *cpu, *net, *disks, memory_percent(), uptime_days(),
+        cpu_temperature(), throttle_state(),
+        failed_service_count(), oom_kill_count(),
+        *free, *inodes, *read_only,
+        storage_health_score(storage, cpu[3]),
+    ]
 
 
 def main() -> None: