graphing optimized
This commit is contained in:
@@ -77,18 +77,19 @@ sources. Temperature is read from Linux thermal zones. Active Raspberry Pi
|
||||
throttling is read with `vcgencmd`; it appears as unknown on other systems.
|
||||
Failed services are counted with `systemctl`. OOM kills come from the kernel's
|
||||
cumulative `/proc/vmstat` counter and are graphed as events per minute.
|
||||
Storage-health graphs show available space in GiB and inode consumption for
|
||||
each configured mountpoint. A read-only filesystem is shown at 100% on the
|
||||
status graph. I/O wait is included in the CPU graph. These portable indicators
|
||||
work with SD cards and USB drives without requiring SMART support.
|
||||
Available space, inode consumption, and read-only state are retained as inputs
|
||||
to the per-mountpoint health score rather than displayed as separate graphs.
|
||||
I/O wait is included in the CPU graph. These portable indicators work with SD
|
||||
cards and USB drives without requiring SMART support.
|
||||
|
||||
The storage operational-health score summarizes the worst configured
|
||||
mountpoint. A score of 10 means no current problem is visible, while a
|
||||
read-only filesystem scores 0. Space and inode usage begin reducing the score
|
||||
above 75%; either reaches 0 when exhausted. I/O wait above 10% can subtract up
|
||||
to three points. This is an operational-risk score, not a measurement of flash
|
||||
wear or remaining media life; SD cards and many USB bridges do not expose the
|
||||
hardware data needed to estimate those reliably.
|
||||
The storage operational-health graph shows one score for each configured
|
||||
mountpoint. A score of 10 means no current problem is visible, while a read-only
|
||||
filesystem scores 0. Space and inode usage begin reducing that filesystem's
|
||||
score above 75%; either reaches 0 when exhausted. Host-wide I/O wait above 10%
|
||||
can subtract up to three points from every score because Linux's CPU wait metric
|
||||
cannot attribute the delay to one mountpoint. This is an operational-risk score,
|
||||
not a measurement of flash wear or remaining media life; SD cards and many USB
|
||||
bridges do not expose the hardware data needed to estimate those reliably.
|
||||
|
||||
## Run from the source tree
|
||||
|
||||
|
||||
+12
-24
@@ -19,7 +19,7 @@ def graph(output: Path, rrd: Path, start: str, title: str, unit: str,
|
||||
series: list[tuple[str, str, str]], logarithmic: bool = False,
|
||||
stacked: bool = False, disk_memory: bool = False,
|
||||
network_mixed: bool = False, thermal_mixed: bool = False,
|
||||
trouble_mixed: bool = False, storage_status: bool = False) -> None:
|
||||
trouble_mixed: bool = False) -> None:
|
||||
args = ["rrdtool", "graph", str(output), "--start", f"end-{start}",
|
||||
"--end", "now", "--width", "900", "--height", "260",
|
||||
"--title", title, "--vertical-label", unit, "--slope-mode",
|
||||
@@ -28,9 +28,7 @@ def graph(output: Path, rrd: Path, start: str, title: str, unit: str,
|
||||
args += ["--logarithmic", "--lower-limit", "1"]
|
||||
if thermal_mixed:
|
||||
args += ["--lower-limit", "0", "--upper-limit", "100", "--rigid"]
|
||||
if storage_status:
|
||||
args += ["--lower-limit", "0", "--upper-limit", "100", "--rigid"]
|
||||
if any(ds == "storage_health" for ds, _label, _cf in series):
|
||||
if any(ds.startswith("health_") for ds, _label, _cf in series):
|
||||
args += ["--lower-limit", "0", "--upper-limit", "10", "--rigid"]
|
||||
value_suffix = "%%" if unit == "%" else ""
|
||||
for index, (ds, label, cf) in enumerate(series):
|
||||
@@ -51,14 +49,6 @@ def graph(output: Path, rrd: Path, start: str, title: str, unit: str,
|
||||
f"GPRINT:{var}:AVERAGE:Average\\:%8.2lf",
|
||||
f"GPRINT:{var}:MAX:Maximum\\:%8.2lf\\n"]
|
||||
continue
|
||||
if storage_status and ds.startswith("readonly_"):
|
||||
args += [f"DEF:{var}_raw={rrd}:{ds}:{cf}",
|
||||
f"CDEF:{var}={var}_raw,100,*",
|
||||
f"LINE2:{var}{COLORS[index % len(COLORS)]}:{label}",
|
||||
f"GPRINT:{var}_raw:LAST:Current\\:%8.0lf",
|
||||
f"GPRINT:{var}_raw:AVERAGE:Average\\:%8.2lf",
|
||||
f"GPRINT:{var}_raw:MAX:Maximum\\:%8.0lf\\n"]
|
||||
continue
|
||||
if network_mixed and ds == "net_out":
|
||||
drawing = f"AREA:{var}#86EFAC:{label}"
|
||||
elif network_mixed and ds == "net_in":
|
||||
@@ -100,23 +90,21 @@ def main() -> None:
|
||||
if not mounts or mounts[0] != "/":
|
||||
mounts.insert(0, "/")
|
||||
definitions = [
|
||||
("load", "Processor load", "load", [("load15", "15 minutes", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load1", "1 minute", "AVERAGE")], False, True, False, False, False, False, False),
|
||||
("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE"), ("io_wait", "I/O wait", "AVERAGE")], False, True, False, False, False, False, False),
|
||||
("thermal", "CPU temperature and throttling", "°C", [("temperature", "temperature", "AVERAGE"), ("throttled", "throttled", "MAX")], False, False, False, False, True, False, False),
|
||||
("trouble", "Failed services and OOM kills", "count", [("failed_services", "failed services", "MAX"), ("oom_kills", "OOM kills/min", "MAX")], False, False, False, False, False, True, False),
|
||||
("network", "Network throughput", "bytes/s", [("net_out", "sent", "AVERAGE"), ("net_in", "received", "AVERAGE")], False, False, False, True, False, False, False),
|
||||
("disk", "Disk and memory usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)] + [("memory", "memory", "AVERAGE")], False, False, True, False, False, False, False),
|
||||
("disk-free", "Available filesystem space", "GiB", [(f"free_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)], False, False, False, False, False, False, False),
|
||||
("storage-status", "Inode usage and read-only filesystems", "%", [(f"inode_{'root' if i == 0 else i}", f"{mount} inodes", "MAX") for i, mount in enumerate(mounts)] + [(f"readonly_{'root' if i == 0 else i}", f"{mount} read-only", "MAX") for i, mount in enumerate(mounts)], False, False, False, False, False, False, True),
|
||||
("storage-health", "Storage operational health", "0–10", [("storage_health", "health score", "MIN")], False, False, False, False, False, False, False),
|
||||
("uptime", "System uptime", "days", [("uptime", "uptime", "AVERAGE")], False, False, False, False, False, False, False),
|
||||
("load", "Processor load", "load", [("load15", "15 minutes", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load1", "1 minute", "AVERAGE")], False, True, False, False, False, False),
|
||||
("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE"), ("io_wait", "I/O wait", "AVERAGE")], False, True, False, False, False, False),
|
||||
("thermal", "CPU temperature and throttling", "°C", [("temperature", "temperature", "AVERAGE"), ("throttled", "throttled", "MAX")], False, False, False, False, True, False),
|
||||
("trouble", "Failed services and OOM kills", "count", [("failed_services", "failed services", "MAX"), ("oom_kills", "OOM kills/min", "MAX")], False, False, False, False, False, True),
|
||||
("network", "Network throughput", "bytes/s", [("net_out", "sent", "AVERAGE"), ("net_in", "received", "AVERAGE")], False, False, False, True, False, False),
|
||||
("disk", "Disk and memory usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)] + [("memory", "memory", "AVERAGE")], False, False, True, False, False, False),
|
||||
("storage-health", "Storage operational health", "0–10", [(f"health_{'root' if i == 0 else i}", mount, "MIN") for i, mount in enumerate(mounts)], False, False, False, False, False, False),
|
||||
("uptime", "System uptime", "days", [("uptime", "uptime", "AVERAGE")], False, False, False, False, False, False),
|
||||
]
|
||||
for period_name, start in PERIODS.items():
|
||||
for filename, title, unit, series, logarithmic, stacked, disk_memory, network_mixed, thermal_mixed, trouble_mixed, storage_status in definitions:
|
||||
for filename, title, unit, series, logarithmic, stacked, disk_memory, network_mixed, thermal_mixed, trouble_mixed in definitions:
|
||||
graph(output_dir / f"{filename}-{period_name}.png", rrd, start,
|
||||
f"{title} — {period_name}", unit, series, logarithmic,
|
||||
stacked, disk_memory, network_mixed, thermal_mixed,
|
||||
trouble_mixed, storage_status)
|
||||
trouble_mixed)
|
||||
page_files = {
|
||||
"3hours": "index.html",
|
||||
"1day": "1day.html",
|
||||
|
||||
+13
-10
@@ -20,7 +20,7 @@ DS_NAMES = (
|
||||
"failed_services", "oom_kills",
|
||||
"free_root", "free_1", "free_2", "inode_root", "inode_1", "inode_2",
|
||||
"readonly_root", "readonly_1", "readonly_2",
|
||||
"storage_health",
|
||||
"health_root", "health_1", "health_2",
|
||||
)
|
||||
|
||||
|
||||
@@ -64,7 +64,8 @@ def create_rrd(path: Path) -> None:
|
||||
"DS:inode_1:GAUGE:180:0:100", "DS:inode_2:GAUGE:180:0:100",
|
||||
"DS:readonly_root:GAUGE:180:0:1", "DS:readonly_1:GAUGE:180:0:1",
|
||||
"DS:readonly_2:GAUGE:180:0:1",
|
||||
"DS:storage_health:GAUGE:180:0:10",
|
||||
"DS:health_root:GAUGE:180:0:10", "DS:health_1:GAUGE:180:0:10",
|
||||
"DS:health_2:GAUGE:180:0:10",
|
||||
]
|
||||
# 1-minute data for a day, 5-minute data for two weeks, hourly for a year.
|
||||
archives = [
|
||||
@@ -206,17 +207,15 @@ def disk_metrics(mountpoint: str) -> tuple[float, float, float | str, int]:
|
||||
|
||||
|
||||
def storage_health_score(
|
||||
storage: list[tuple[float, float, float | str, int]],
|
||||
storage: tuple[float, float, float | str, int],
|
||||
io_wait: float) -> float:
|
||||
"""Score current storage operability from 0 (unusable) to 10 (healthy)."""
|
||||
if any(read_only for _used, _free, _inodes, read_only in storage):
|
||||
"""Score one filesystem's operability from 0 (unusable) to 10 (healthy)."""
|
||||
used, _free, inodes, read_only = storage
|
||||
if read_only:
|
||||
return 0.0
|
||||
|
||||
score = 10.0
|
||||
for used, _free, inodes, _read_only in storage:
|
||||
# No penalty through 75%; decline linearly to zero at 100%.
|
||||
space_score = max(0.0, min(10.0, (100.0 - used) / 25.0 * 10.0))
|
||||
score = min(score, space_score)
|
||||
score = max(0.0, min(10.0, (100.0 - used) / 25.0 * 10.0))
|
||||
if isinstance(inodes, float):
|
||||
inode_score = max(0.0, min(10.0, (100.0 - inodes) / 25.0 * 10.0))
|
||||
score = min(score, inode_score)
|
||||
@@ -258,12 +257,16 @@ def collect(config: configparser.SectionProxy) -> list[float | int | str]:
|
||||
read_only: list[int | str] = [item[3] for item in storage]
|
||||
for values in (disks, free, inodes, read_only):
|
||||
values.extend(["U"] * (3 - len(values)))
|
||||
health: list[float | str] = [
|
||||
storage_health_score(item, cpu[3]) for item in storage
|
||||
]
|
||||
health.extend(["U"] * (3 - len(health)))
|
||||
return [
|
||||
*load, *cpu, *net, *disks, memory_percent(), uptime_days(),
|
||||
cpu_temperature(), throttle_state(),
|
||||
failed_service_count(), oom_kill_count(),
|
||||
*free, *inodes, *read_only,
|
||||
storage_health_score(storage, cpu[3]),
|
||||
*health,
|
||||
]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user