diff --git a/README.md b/README.md index 1c3ddd5..e0d36b3 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ It collects: - total network receive/transmit throughput - disk usage for `/` and up to two additional mountpoints - memory usage +- system uptime in days ## Install @@ -62,6 +63,11 @@ The first meaningful network rate appears after two collection samples. A new installation naturally needs time to accumulate history; RRD archives are consolidated to five-minute samples for two weeks and hourly samples for a year. +The RRD schema is fixed when the database is created. When upgrading from a +version without uptime monitoring, move or remove the existing `system.rrd` +before the next collection if preserving its old history is not required. The +collector will then create a new database containing the uptime data source. + ## Run from the source tree For development or a non-system installation, copy the example configuration, diff --git a/generate_graphs.py b/generate_graphs.py index 7f7adb2..08c7d1e 100755 --- a/generate_graphs.py +++ b/generate_graphs.py @@ -12,23 +12,43 @@ import subprocess PERIODS = {"3hours": "3h", "1day": "1d", "2weeks": "2w", "1year": "1y"} COLORS = ["#2563EB", "#DC2626", "#16A34A", "#9333EA"] +LOAD_COLORS = ["#DC2626", "#F59E0B", "#FACC15"] # red, amber, yellow def graph(output: Path, rrd: Path, start: str, title: str, unit: str, - series: list[tuple[str, str, str]], logarithmic: bool = False) -> None: + series: list[tuple[str, str, str]], logarithmic: bool = False, + stacked: bool = False, disk_memory: bool = False, + network_mixed: bool = False) -> None: args = ["rrdtool", "graph", str(output), "--start", f"end-{start}", "--end", "now", "--width", "900", "--height", "260", "--title", title, "--vertical-label", unit, "--slope-mode", "--watermark", "cheap-system-monitor"] if logarithmic: args += ["--logarithmic", "--lower-limit", "1"] + value_suffix = "%%" if unit == "%" else "" for index, (ds, label, cf) in enumerate(series): var = f"v{index}" + if network_mixed and ds == "net_out": + drawing = f"AREA:{var}#86EFAC:{label}" + elif network_mixed and ds == "net_in": + drawing = f"LINE3:{var}#2563EB:{label}" + elif disk_memory and ds == "memory": + drawing = f"LINE3:{var}#DC2626:{label}" + elif disk_memory: + # Transparency keeps multiple mountpoints visible when their + # percentage areas overlap. + drawing = f"AREA:{var}{COLORS[index % len(COLORS)]}80:{label}" + elif stacked: + stack = ":STACK" if index else "" + stacked_colors = LOAD_COLORS if title.startswith("Processor load") else COLORS + drawing = f"AREA:{var}{stacked_colors[index]}:{label}{stack}" + else: + drawing = f"LINE{2 if index < 3 else 1}:{var}{COLORS[index % len(COLORS)]}:{label}" args += [f"DEF:{var}={rrd}:{ds}:{cf}", - f"LINE{2 if index < 3 else 1}:{var}{COLORS[index % len(COLORS)]}:{label}", - f"GPRINT:{var}:LAST:Current\\:%8.2lf%s", - f"GPRINT:{var}:AVERAGE:Average\\:%8.2lf%s", - f"GPRINT:{var}:MAX:Maximum\\:%8.2lf%s\\n"] + drawing, + f"GPRINT:{var}:LAST:Current\\:%8.2lf%s{value_suffix}", + f"GPRINT:{var}:AVERAGE:Average\\:%8.2lf%s{value_suffix}", + f"GPRINT:{var}:MAX:Maximum\\:%8.2lf%s{value_suffix}\\n"] subprocess.run(args, check=True) @@ -49,23 +69,24 @@ def main() -> None: if not mounts or mounts[0] != "/": mounts.insert(0, "/") definitions = [ - ("load", "Processor load", "load", [("load1", "1 minute", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load15", "15 minutes", "AVERAGE")], False), - ("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE")], False), - ("network", "Network throughput", "bytes/s", [("net_in", "received", "AVERAGE"), ("net_out", "sent", "AVERAGE")], False), - ("disk", "Disk usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)], False), - ("memory", "Memory usage", "%", [("memory", "used", "AVERAGE")], False), + ("load", "Processor load", "load", [("load1", "1 minute", "AVERAGE"), ("load5", "5 minutes", "AVERAGE"), ("load15", "15 minutes", "AVERAGE")], False, True, False, False), + ("cpu", "CPU usage", "%", [("cpu_user", "user", "AVERAGE"), ("cpu_system", "system", "AVERAGE"), ("cpu_nice", "nice", "AVERAGE")], False, True, False, False), + ("network", "Network throughput", "bytes/s", [("net_out", "sent", "AVERAGE"), ("net_in", "received", "AVERAGE")], False, False, False, True), + ("disk", "Disk and memory usage", "%", [(f"disk_{'root' if i == 0 else i}", mount, "AVERAGE") for i, mount in enumerate(mounts)] + [("memory", "memory", "AVERAGE")], False, False, True, False), + ("uptime", "System uptime", "days", [("uptime", "uptime", "AVERAGE")], False, False, False, False), ] for period_name, start in PERIODS.items(): - for filename, title, unit, series, logarithmic in definitions: + for filename, title, unit, series, logarithmic, stacked, disk_memory, network_mixed in definitions: graph(output_dir / f"{filename}-{period_name}.png", rrd, start, - f"{title} — {period_name}", unit, series, logarithmic) + f"{title} — {period_name}", unit, series, logarithmic, + stacked, disk_memory, network_mixed) cards = "\n".join( f'

{html.escape(title)}

' + "".join( f'
{period}
' f'{html.escape(title)}, {period}
' for period in PERIODS ) + "
" - for filename, title, _unit, _series, _logarithmic in definitions + for filename, title, _unit, _series, _logarithmic, _stacked, _disk_memory, _network_mixed in definitions ) page = f""" diff --git a/system_monitor.py b/system_monitor.py index a8406be..d544326 100755 --- a/system_monitor.py +++ b/system_monitor.py @@ -14,7 +14,7 @@ import time DEFAULT_CONFIG = "/opt/cheap_system_monitor/monitor.conf" DS_NAMES = ( "load1", "load5", "load15", "cpu_user", "cpu_system", "cpu_nice", - "net_in", "net_out", "disk_root", "disk_1", "disk_2", "memory", + "net_in", "net_out", "disk_root", "disk_1", "disk_2", "memory", "uptime", ) @@ -49,6 +49,7 @@ def create_rrd(path: Path) -> None: "DS:net_in:DERIVE:180:0:U", "DS:net_out:DERIVE:180:0:U", "DS:disk_root:GAUGE:180:0:100", "DS:disk_1:GAUGE:180:0:100", "DS:disk_2:GAUGE:180:0:100", "DS:memory:GAUGE:180:0:100", + "DS:uptime:GAUGE:180:0:U", ] # 1-minute data for a day, 5-minute data for two weeks, hourly for a year. archives = [ @@ -91,6 +92,11 @@ def memory_percent() -> float: return 100.0 * (total - available) / total +def uptime_days() -> float: + seconds = float(Path("/proc/uptime").read_text().split()[0]) + return seconds / 86400.0 + + def disk_percent(mountpoint: str) -> float: stats = os.statvfs(mountpoint) used = stats.f_blocks - stats.f_bfree @@ -124,7 +130,7 @@ def collect(config: configparser.SectionProxy) -> list[float | int | str]: raise SystemExit("At most three mountpoints are supported (including /)") disks: list[float | str] = [disk_percent(item) for item in mounts] disks.extend(["U"] * (3 - len(disks))) - return [*load, *cpu, *net, *disks, memory_percent()] + return [*load, *cpu, *net, *disks, memory_percent(), uptime_days()] def main() -> None: