ignace/cheap_system_monitor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adfded temperature, failed services and disk health

Browse Source
This commit is contained in:
ignace
2026-08-12 07:59:29 +02:00
parent 413543ca84
commit 177c7479b6
3 changed files with 215 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+24 -3
View File
@@ -12,6 +12,11 @@ It collects:
- disk usage for `/` and up to two additional mountpoints
- memory usage
- system uptime in days
- CPU temperature and active Raspberry Pi throttling
- failed systemd services and kernel out-of-memory kills
- available filesystem space, inode usage, and read-only filesystem state
- CPU I/O wait
- a combined storage operational-health score from 0 to 10
## Install
@@ -65,9 +70,25 @@ installation naturally needs time to accumulate history; RRD archives are
consolidated to five-minute samples for two weeks and hourly samples for a year.
The RRD schema is fixed when the database is created. When upgrading from a
version without uptime monitoring, move or remove the existing `system.rrd`
before the next collection if preserving its old history is not required. The
collector will then create a new database containing the uptime data source.
version without the latest monitoring data sources, move or remove the
existing `system.rrd` before the next collection if preserving its old history
is not required. The collector will then create a database with the new data
sources. Temperature is read from Linux thermal zones. Active Raspberry Pi
throttling is read with `vcgencmd`; it appears as unknown on other systems.
Failed services are counted with `systemctl`. OOM kills come from the kernel's
cumulative `/proc/vmstat` counter and are graphed as events per minute.
Storage-health graphs show available space in GiB and inode consumption for
each configured mountpoint. A read-only filesystem is shown at 100% on the
status graph. I/O wait is included in the CPU graph. These portable indicators
work with SD cards and USB drives without requiring SMART support.
The storage operational-health score summarizes the worst configured
mountpoint. A score of 10 means no current problem is visible, while a
read-only filesystem scores 0. Space and inode usage begin reducing the score
above 75%; either reaches 0 when exhausted. I/O wait above 10% can subtract up
to three points. This is an operational-risk score, not a measurement of flash
wear or remaining media life; SD cards and many USB bridges do not expose the
hardware data needed to estimate those reliably.
## Run from the source tree
generate_graphs.py
+45 -8
View File
@@ -18,16 +18,47 @@ LOAD_COLORS = ["#FACC15", "#F59E0B", "#DC2626"] # bottom to top
def graph(output: Path, rrd: Path, start: str, title: str, unit: str,
series: list[tuple[str, str, str]], logarithmic: bool = False,
stacked: bool = False, disk_memory: bool = False,
network_mixed: bool = False) -> None:
network_mixed: bool = False, thermal_mixed: bool = False,
trouble_mixed: bool = False, storage_status: bool = False) -> None:
args = ["rrdtool", "graph", str(output), "--start", f"end-{start}",
"--end", "now", "--width", "900", "--height", "260",
"--title", title, "--vertical-label", unit, "--slope-mode",
"--watermark", "cheap-system-monitor"]
if logarithmic:
args += ["--logarithmic", "--lower-limit", "1"]
if thermal_mixed:
args += ["--lower-limit", "0", "--upper-limit", "100", "--rigid"]
if storage_status:
args += ["--lower-limit", "0", "--upper-limit", "100", "--rigid"]
if any(ds == "storage_health" for ds, _label, _cf in series):
args += ["--lower-limit", "0", "--upper-limit", "10", "--rigid"]
value_suffix = "%%" if unit == "%" else ""
for index, (ds, label, cf) in enumerate(series):
var = f"v{index}"
if thermal_mixed and ds == "throttled":
args += [f"DEF:{var}_raw={rrd}:{ds}:{cf}",
f"CDEF:{var}={var}_raw,100,*",
f"AREA:{var}#DC262640:{label}",
f"GPRINT:{var}_raw:LAST:Current\\:%8.0lf",
f"GPRINT:{var}_raw:AVERAGE:Average\\:%8.2lf",
f"GPRINT:{var}_raw:MAX:Maximum\\:%8.0lf\\n"]
continue
if trouble_mixed and ds == "oom_kills":
args += [f"DEF:{var}_raw={rrd}:{ds}:{cf}",
f"CDEF:{var}={var}_raw,60,*",
f"AREA:{var}#DC262680:{label}",
f"GPRINT:{var}:LAST:Current\\:%8.2lf",
f"GPRINT:{var}:AVERAGE:Average\\:%8.2lf",
f"GPRINT:{var}:MAX:Maximum\\:%8.2lf\\n"]
continue
if storage_status and ds.startswith("readonly_"):
args += [f"DEF:{var}_raw={rrd}:{ds}:{cf}",
f"CDEF:{var}={var}_raw,100,*",
f"LINE2:{var}{COLORS[index % len(COLORS)]}:{label}",
f"GPRINT:{var}_raw:LAST:Current\\:%8.0lf",
f"GPRINT:{var}_raw:AVERAGE:Average\\:%8.2lf",
f"GPRINT:{var}_raw:MAX:Maximum\\:%8.0lf\\n"]
continue
if network_mixed and ds == "net_out":
drawing = f"AREA:{var}#86EFAC:{label}"
elif network_mixed and ds == "net_in":
@@ -69,17 +100,23 @@ def main() -> None:
if not mounts or mounts[0] != "/":
mounts.insert(0, "/")
definitions = [
("load", "Processor load", "load", [("load15", "15 minutes", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load1", "1 minute", "AVERAGE")], False, True, False, False),
("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE")], False, True, False, False),
("network", "Network throughput", "bytes/s", [("net_out", "sent", "AVERAGE"), ("net_in", "received", "AVERAGE")], False, False, False, True),
("disk", "Disk and memory usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)] + [("memory", "memory", "AVERAGE")], False, False, True, False),
("uptime", "System uptime", "days", [("uptime", "uptime", "AVERAGE")], False, False, False, False),
("load", "Processor load", "load", [("load15", "15 minutes", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load1", "1 minute", "AVERAGE")], False, True, False, False, False, False, False),
("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE"), ("io_wait", "I/O wait", "AVERAGE")], False, True, False, False, False, False, False),
("thermal", "CPU temperature and throttling", "°C", [("temperature", "temperature", "AVERAGE"), ("throttled", "throttled", "MAX")], False, False, False, False, True, False, False),
("trouble", "Failed services and OOM kills", "count", [("failed_services", "failed services", "MAX"), ("oom_kills", "OOM kills/min", "MAX")], False, False, False, False, False, True, False),
("network", "Network throughput", "bytes/s", [("net_out", "sent", "AVERAGE"), ("net_in", "received", "AVERAGE")], False, False, False, True, False, False, False),
("disk", "Disk and memory usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)] + [("memory", "memory", "AVERAGE")], False, False, True, False, False, False, False),
("disk-free", "Available filesystem space", "GiB", [(f"free_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)], False, False, False, False, False, False, False),
("storage-status", "Inode usage and read-only filesystems", "%", [(f"inode_{'root' if i == 0 else i}", f"{mount} inodes", "MAX") for i, mount in enumerate(mounts)] + [(f"readonly_{'root' if i == 0 else i}", f"{mount} read-only", "MAX") for i, mount in enumerate(mounts)], False, False, False, False, False, False, True),
("storage-health", "Storage operational health", "0–10", [("storage_health", "health score", "MIN")], False, False, False, False, False, False, False),
("uptime", "System uptime", "days", [("uptime", "uptime", "AVERAGE")], False, False, False, False, False, False, False),
]
for period_name, start in PERIODS.items():
for filename, title, unit, series, logarithmic, stacked, disk_memory, network_mixed in definitions:
for filename, title, unit, series, logarithmic, stacked, disk_memory, network_mixed, thermal_mixed, trouble_mixed, storage_status in definitions:
graph(output_dir / f"{filename}-{period_name}.png", rrd, start,
f"{title} — {period_name}", unit, series, logarithmic,
stacked, disk_memory, network_mixed)
stacked, disk_memory, network_mixed, thermal_mixed,
trouble_mixed, storage_status)
page_files = {
"3hours": "index.html",
"1day": "1day.html",
system_monitor.py
+146 -12
View File
@@ -14,7 +14,13 @@ import time
DEFAULT_CONFIG = "/opt/cheap_system_monitor/monitor.conf"
DS_NAMES = (
"load1", "load5", "load15", "cpu_user", "cpu_system", "cpu_nice",
"io_wait",
"net_in", "net_out", "disk_root", "disk_1", "disk_2", "memory", "uptime",
"temperature", "throttled",
"failed_services", "oom_kills",
"free_root", "free_1", "free_2", "inode_root", "inode_1", "inode_2",
"readonly_root", "readonly_1", "readonly_2",
"storage_health",
)
@@ -46,40 +52,53 @@ def create_rrd(path: Path) -> None:
"DS:load1:GAUGE:180:0:U", "DS:load5:GAUGE:180:0:U",
"DS:load15:GAUGE:180:0:U", "DS:cpu_user:GAUGE:180:0:100",
"DS:cpu_system:GAUGE:180:0:100", "DS:cpu_nice:GAUGE:180:0:100",
"DS:io_wait:GAUGE:180:0:100",
"DS:net_in:DERIVE:180:0:U", "DS:net_out:DERIVE:180:0:U",
"DS:disk_root:GAUGE:180:0:100", "DS:disk_1:GAUGE:180:0:100",
"DS:disk_2:GAUGE:180:0:100", "DS:memory:GAUGE:180:0:100",
"DS:uptime:GAUGE:180:0:U",
"DS:temperature:GAUGE:180:-50:200", "DS:throttled:GAUGE:180:0:1",
"DS:failed_services:GAUGE:180:0:U", "DS:oom_kills:DERIVE:180:0:U",
"DS:free_root:GAUGE:180:0:U", "DS:free_1:GAUGE:180:0:U",
"DS:free_2:GAUGE:180:0:U", "DS:inode_root:GAUGE:180:0:100",
"DS:inode_1:GAUGE:180:0:100", "DS:inode_2:GAUGE:180:0:100",
"DS:readonly_root:GAUGE:180:0:1", "DS:readonly_1:GAUGE:180:0:1",
"DS:readonly_2:GAUGE:180:0:1",
"DS:storage_health:GAUGE:180:0:10",
]
# 1-minute data for a day, 5-minute data for two weeks, hourly for a year.
archives = [
"RRA:AVERAGE:0.5:1:1440", "RRA:MAX:0.5:1:1440",
"RRA:MIN:0.5:1:1440",
"RRA:AVERAGE:0.5:5:4032", "RRA:MAX:0.5:5:4032",
"RRA:MIN:0.5:5:4032",
"RRA:AVERAGE:0.5:60:8784", "RRA:MAX:0.5:60:8784",
"RRA:MIN:0.5:60:8784",
]
run_rrdtool("create", str(path), "--step", "60", *ds, *archives)
def read_cpu() -> tuple[int, int, int, int]:
def read_cpu() -> tuple[int, int, int, int, int]:
fields = Path("/proc/stat").read_text().splitlines()[0].split()
if fields[0] != "cpu" or len(fields) < 8:
raise RuntimeError("unexpected /proc/stat format")
values = [int(value) for value in fields[1:]]
user, nice, system, idle = values[:4]
idle += values[4] if len(values) > 4 else 0 # include iowait
iowait = values[4] if len(values) > 4 else 0
idle += iowait
# Linux reports guest time as already included in user/nice, so only the
# first eight fields belong in the total (through steal time).
return user, system, nice, sum(values[:8])
return user, system, nice, iowait, sum(values[:8])
def cpu_percentages(sample_seconds: float) -> tuple[float, float, float]:
def cpu_percentages(sample_seconds: float) -> tuple[float, float, float, float]:
before = read_cpu()
time.sleep(sample_seconds)
after = read_cpu()
total = after[3] - before[3]
total = after[4] - before[4]
if total <= 0:
return 0.0, 0.0, 0.0
return tuple(100.0 * (after[i] - before[i]) / total for i in range(3))
return 0.0, 0.0, 0.0, 0.0
return tuple(100.0 * (after[i] - before[i]) / total for i in range(4))
def memory_percent() -> float:
@@ -97,11 +116,115 @@ def uptime_days() -> float:
return seconds / 86400.0
def disk_percent(mountpoint: str) -> float:
def cpu_temperature() -> float | str:
"""Return the CPU temperature in Celsius, or an RRD unknown value."""
thermal_root = Path("/sys/class/thermal")
if not thermal_root.exists():
return "U"
zones = sorted(thermal_root.glob("thermal_zone*"))
preferred = []
others = []
for zone in zones:
try:
zone_type = (zone / "type").read_text().strip().lower()
except OSError:
zone_type = ""
target = preferred if any(name in zone_type for name in ("cpu", "soc", "package")) else others
target.append(zone)
for zone in preferred + others:
try:
value = float((zone / "temp").read_text().strip())
# Linux thermal-zone temperatures are normally millidegrees.
return value / 1000.0 if abs(value) >= 1000 else value
except (OSError, ValueError):
continue
return "U"
def throttle_state() -> int | str:
"""Return 1 during active Raspberry Pi throttling, 0 otherwise."""
try:
result = subprocess.run(
["vcgencmd", "get_throttled"], check=True, capture_output=True,
text=True, timeout=2,
)
except (FileNotFoundError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
return "U"
try:
flags = int(result.stdout.strip().split("=", 1)[1], 16)
except (IndexError, ValueError):
return "U"
# Bits 1, 2, and 3 mean frequency capped, throttled, and soft temperature
# limit active. Ignore the corresponding sticky "has occurred" bits.
return int(bool(flags & 0x0E))
def failed_service_count() -> int | str:
"""Return the number of failed systemd service units."""
try:
result = subprocess.run(
["systemctl", "list-units", "--state=failed", "--type=service",
"--no-legend", "--plain"],
check=True, capture_output=True, text=True, timeout=5,
)
except (FileNotFoundError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
return "U"
return sum(bool(line.strip()) for line in result.stdout.splitlines())
def oom_kill_count() -> int | str:
"""Return the kernel's cumulative OOM-kill counter since boot."""
try:
for line in Path("/proc/vmstat").read_text().splitlines():
name, value = line.split()
if name == "oom_kill":
return int(value)
except (OSError, ValueError):
pass
return "U"
def disk_metrics(mountpoint: str) -> tuple[float, float, float | str, int]:
"""Return used %, available GiB, inode used %, and read-only status."""
stats = os.statvfs(mountpoint)
used = stats.f_blocks - stats.f_bfree
available = stats.f_bavail
return 100.0 * used / (used + available) if used + available else 0.0
used_percent = 100.0 * used / (used + available) if used + available else 0.0
block_size = stats.f_frsize or stats.f_bsize
free_gib = available * block_size / (1024 ** 3)
used_inodes = stats.f_files - stats.f_ffree
available_inodes = stats.f_favail
inode_total = used_inodes + available_inodes
inode_percent: float | str = (
100.0 * used_inodes / inode_total if inode_total else "U"
)
read_only = int(bool(stats.f_flag & os.ST_RDONLY))
return used_percent, free_gib, inode_percent, read_only
def storage_health_score(
storage: list[tuple[float, float, float | str, int]],
io_wait: float) -> float:
"""Score current storage operability from 0 (unusable) to 10 (healthy)."""
if any(read_only for _used, _free, _inodes, read_only in storage):
return 0.0
score = 10.0
for used, _free, inodes, _read_only in storage:
# No penalty through 75%; decline linearly to zero at 100%.
space_score = max(0.0, min(10.0, (100.0 - used) / 25.0 * 10.0))
score = min(score, space_score)
if isinstance(inodes, float):
inode_score = max(0.0, min(10.0, (100.0 - inodes) / 25.0 * 10.0))
score = min(score, inode_score)
# I/O wait is workload-sensitive, so it can reduce the score by at most
# three points: no penalty through 10%, maximum penalty at 50%.
io_penalty = max(0.0, min(3.0, (io_wait - 10.0) / 40.0 * 3.0))
return max(0.0, score - io_penalty)
def network_counters(interface_setting: str) -> tuple[int, int]:
@@ -128,9 +251,20 @@ def collect(config: configparser.SectionProxy) -> list[float | int | str]:
mounts.insert(0, "/")
if len(mounts) > 3:
raise SystemExit("At most three mountpoints are supported (including /)")
disks: list[float | str] = [disk_percent(item) for item in mounts]
disks.extend(["U"] * (3 - len(disks)))
return [*load, *cpu, *net, *disks, memory_percent(), uptime_days()]
storage = [disk_metrics(item) for item in mounts]
disks: list[float | str] = [item[0] for item in storage]
free: list[float | str] = [item[1] for item in storage]
inodes: list[float | str] = [item[2] for item in storage]
read_only: list[int | str] = [item[3] for item in storage]
for values in (disks, free, inodes, read_only):
values.extend(["U"] * (3 - len(values)))
return [
*load, *cpu, *net, *disks, memory_percent(), uptime_days(),
cpu_temperature(), throttle_state(),
failed_service_count(), oom_kill_count(),
*free, *inodes, *read_only,
storage_health_score(storage, cpu[3]),
]
def main() -> None: