diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a230a78 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.venv/ +__pycache__/ diff --git a/HOWTO.md b/HOWTO.md index 5ecacc9..c0016a2 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -324,3 +324,132 @@ change, rather than continuously. The SNZB-02P publishes `temperature` in °C and `humidity` as a percentage. Reference: [Zigbee2MQTT MQTT topics](https://www.zigbee2mqtt.io/guide/usage/mqtt_topics_and_messages.html) + +## 7, reading with curl +```bash +curl --silent --max-time 2 "mqtt://192.168.178.247/$topic" | dd bs=1 skip=$((2 + ${#topic})) status=none +{"battery":100,"humidity":56.3,"humidity_calibration":0,"temperature":27.1,"temperature_calibration":0,"update":{"installed_version":8704,"latest_release_notes":null,"latest_source":"https://raw.githubusercontent.com/Koenkk/zigbee-OTA/master/images/Sonoff/snzb-02p_v2.2.0.ota","latest_version":8704,"state":"idle"}} +``` + +## 8. Store six sensors in RRD files and create graphs + +Zigbee2MQTT must retain each sensor's state so the collector can obtain the +latest reading without waiting for a sleeping sensor: + +```yaml +device_options: + retain: true +``` + +Install RRDtool and create a Python virtual environment in this project: + +```bash +sudo apt install -y rrdtool python3-venv +cd "$HOME/service_zigbee" +python3 -m venv .venv +.venv/bin/pip install -r requirements.txt +``` + +Edit the supplied `config.yaml`. MQTT settings, collection timing, storage +paths, logging, all six sensors, RRD archives, and graph presentation are +configured in this one file. Replace all five `CHANGE_ME` topics with the +friendly names shown by Zigbee2MQTT: + +```bash +nano config.yaml +``` + +By default, the databases are written to `/var/lib/rrd` and the generated +graphs to `/var/www/html/sensors`. Create those directories and grant the user +running the collector ownership: + +```bash +sudo install -d -o "$USER" -g "$USER" /var/lib/rrd +sudo install -d -o "$USER" -g "$USER" /var/www/html/sensors +``` + +Both paths can be changed under `storage` in `config.yaml`: + +```yaml +storage: + data_directory: /var/lib/rrd + graph_directory: /var/www/html/sensors +``` + +Test one collection cycle: + +```bash +.venv/bin/python read_temperature.py --once +``` + +The script creates one RRD file per sensor in the configured data directory and +three PNG files per sensor in the configured graph directory: + +- `*_day.png` covers the last 24 hours. +- `*_month.png` covers the last 31 days. +- `*_two_years.png` covers the last two years. + +Each graph contains a filled temperature area, a solid humidity line, and a +dotted battery line. Initially, most of each graph is blank because no historic +samples existed before the RRD was created. + +The collector also generates `/var/www/html/sensors/index.html`. It displays +six graphs at a time and has controls for switching between the daily, monthly, +and two-year periods. It remembers the selected period and refreshes the PNGs +at the interval configured under `html.refresh_seconds`. + +With a web server serving `/var/www/html`, open: + +```text +http://RASPBERRY_PI_IP/sensors/ +``` + +The page settings are configurable in `config.yaml`: + +```yaml +html: + filename: index.html + title: Zigbee sensor history + default_period: day + refresh_seconds: 600 +``` + +RRD structure settings are used only when each `.rrd` file is first created. +Changing `rrd.step_seconds`, data sources, or archives does not rewrite an +existing database; preserve or remove the old RRD deliberately before creating +a replacement. + +Run the ten-minute collection loop in the foreground with: + +```bash +.venv/bin/python read_temperature.py +``` + +To run it automatically as a user service, copy the supplied service file. It +assumes this repository is located at `$HOME/service_zigbee`: + +```bash +mkdir -p "$HOME/.config/systemd/user" +cp zigbee-rrd.service.example "$HOME/.config/systemd/user/zigbee-rrd.service" +systemctl --user daemon-reload +systemctl --user enable --now zigbee-rrd.service +sudo loginctl enable-linger "$USER" +``` + +Inspect its status and live logs with: + +```bash +systemctl --user status zigbee-rrd.service +journalctl --user -u zigbee-rrd.service -f +``` + +Graphs can be regenerated without collecting a new MQTT reading: + +```bash +.venv/bin/python read_temperature.py --graph-only +``` + +References: + +- [RRDtool database creation](https://oss.oetiker.ch/rrdtool/doc/rrdcreate.en.html) +- [RRDtool graph elements](https://oss.oetiker.ch/rrdtool/doc/rrdgraph_graph.en.html) diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..05ddb69 --- /dev/null +++ b/config.yaml @@ -0,0 +1,107 @@ +mqtt: + host: 192.168.178.247 + port: 1883 + username: null + password: null + +collection: + interval_seconds: 600 + mqtt_timeout_seconds: 15 + +storage: + data_directory: /var/lib/rrd + graph_directory: /var/www/html/sensors + +logging: + level: INFO + format: "%(asctime)s %(levelname)s %(message)s" + +sensors: + - name: Server room + topic: zigbee2mqtt/sensor_server_room + - name: Sensor 2 + topic: zigbee2mqtt/CHANGE_ME_SENSOR_2 + - name: Sensor 3 + topic: zigbee2mqtt/CHANGE_ME_SENSOR_3 + - name: Sensor 4 + topic: zigbee2mqtt/CHANGE_ME_SENSOR_4 + - name: Sensor 5 + topic: zigbee2mqtt/CHANGE_ME_SENSOR_5 + - name: Sensor 6 + topic: zigbee2mqtt/CHANGE_ME_SENSOR_6 + +rrd: + # This must normally match collection.interval_seconds. + step_seconds: 600 + start: now-600 + data_sources: + temperature: + type: GAUGE + heartbeat_seconds: 1200 + minimum: -50 + maximum: 80 + humidity: + type: GAUGE + heartbeat_seconds: 1200 + minimum: 0 + maximum: 100 + battery: + type: GAUGE + heartbeat_seconds: 1200 + minimum: 0 + maximum: 100 + archives: + # Ten-minute values for 8 days. + - consolidation_function: AVERAGE + xfiles_factor: 0.5 + steps: 1 + rows: 1152 + # Hourly averages for 62 days. + - consolidation_function: AVERAGE + xfiles_factor: 0.5 + steps: 6 + rows: 1488 + # Daily averages for just over two years. + - consolidation_function: AVERAGE + xfiles_factor: 0.5 + steps: 144 + rows: 740 + +graph: + width: 1000 + height: 360 + end: now + vertical_label: degrees C / percent + slope_mode: true + alt_autoscale: true + periods: + day: + start: end-1d + label: last 24 hours + month: + start: end-31d + label: last month + two_years: + start: end-2y + label: last two years + temperature: + color: E4575680 + legend: Temperature (C) + last_value_format: "Temperature last\\: %5.1lf C" + humidity: + color: 4C78A8 + line_width: 2 + legend: Humidity (%) + last_value_format: "Humidity last\\: %5.1lf %%" + battery: + color: 54A24B + line_width: 2 + dashes: 3,3 + legend: Battery (%) + last_value_format: "Battery last\\: %5.1lf %%\\n" + +html: + filename: index.html + title: Zigbee sensors + default_period: day + refresh_seconds: 600 diff --git a/read_temperature.py b/read_temperature.py new file mode 100644 index 0000000..b5b4b59 --- /dev/null +++ b/read_temperature.py @@ -0,0 +1,454 @@ +#!/usr/bin/env python3 +"""Collect Zigbee2MQTT readings in RRD files and render graphs.""" + +from __future__ import annotations + +import argparse +import html +import json +import logging +import math +import re +import shutil +import signal +import subprocess +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import paho.mqtt.client as mqtt +import yaml + + +LOG = logging.getLogger("zigbee-rrd") +BASE_DIR = Path(__file__).resolve().parent +STOP = threading.Event() + + +@dataclass(frozen=True) +class Sensor: + name: str + topic: str + slug: str + + +@dataclass(frozen=True) +class Config: + mqtt_host: str + mqtt_port: int + mqtt_username: str | None + mqtt_password: str | None + interval: int + timeout: int + data_dir: Path + graph_dir: Path + sensors: tuple[Sensor, ...] + rrd: dict[str, Any] + graph: dict[str, Any] + html: dict[str, Any] + log_level: str + log_format: str + + +def safe_slug(name: str) -> str: + slug = re.sub(r"[^a-zA-Z0-9_-]+", "_", name.strip()).strip("_") + if not slug: + raise ValueError(f"Invalid sensor name {name!r}") + return slug + + +def load_config(path: Path) -> Config: + try: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + except FileNotFoundError as error: + raise SystemExit( + f"Configuration file {path} was not found" + ) from error + except yaml.YAMLError as error: + raise SystemExit(f"Invalid YAML in {path}: {error}") from error + + if not isinstance(raw, dict): + raise SystemExit("The YAML root must be a mapping") + + mqtt_config = raw.get("mqtt", {}) + configured_sensors = raw.get("sensors", []) + if not isinstance(configured_sensors, list) or not configured_sensors: + raise SystemExit("The configuration requires a non-empty sensors list") + + sensors: list[Sensor] = [] + slugs: set[str] = set() + topics: set[str] = set() + for item in configured_sensors: + try: + name = str(item["name"]).strip() + topic = str(item["topic"]).strip().strip("/") + except (KeyError, TypeError) as error: + raise SystemExit("Every sensor requires a name and topic") from error + if not name or not topic: + raise SystemExit("Sensor names and topics cannot be empty") + slug = safe_slug(name) + if slug in slugs or topic in topics: + raise SystemExit(f"Duplicate sensor slug or topic: {name!r}, {topic!r}") + slugs.add(slug) + topics.add(topic) + sensors.append(Sensor(name, topic, slug)) + + collection = raw.get("collection", {}) + storage = raw.get("storage", {}) + logging_config = raw.get("logging", {}) + interval = int(collection.get("interval_seconds", 600)) + timeout = int(collection.get("mqtt_timeout_seconds", 15)) + if interval < 60 or timeout < 1: + raise SystemExit("interval_seconds must be >= 60 and timeout must be positive") + + def config_path(key: str, default: str) -> Path: + value = Path(storage.get(key, default)).expanduser() + return value if value.is_absolute() else path.parent / value + + return Config( + mqtt_host=str(mqtt_config.get("host", "127.0.0.1")), + mqtt_port=int(mqtt_config.get("port", 1883)), + mqtt_username=mqtt_config.get("username"), + mqtt_password=mqtt_config.get("password"), + interval=interval, + timeout=timeout, + data_dir=config_path("data_directory", "/var/lib/rrd"), + graph_dir=config_path("graph_directory", "/var/www/html/sensors"), + sensors=tuple(sensors), + rrd=raw.get("rrd", {}), + graph=raw.get("graph", {}), + html=raw.get("html", {}), + log_level=str(logging_config.get("level", "INFO")), + log_format=str(logging_config.get("format", "%(asctime)s %(levelname)s %(message)s")), + ) + + +def run_rrdtool(*arguments: str) -> None: + try: + subprocess.run(["rrdtool", *arguments], check=True, text=True) + except subprocess.CalledProcessError as error: + raise RuntimeError(f"rrdtool {' '.join(arguments[:2])} failed") from error + + +def create_rrd(path: Path, config: Config) -> None: + if path.exists(): + return + rrd = config.rrd + data_sources = rrd.get("data_sources", {}) + archives = rrd.get("archives", []) + if not data_sources or not archives: + raise SystemExit("rrd.data_sources and rrd.archives must be configured") + + def data_source(name: str) -> str: + settings = data_sources[name] + minimum = settings.get("minimum", "U") + maximum = settings.get("maximum", "U") + return ( + f"DS:{name}:{settings['type']}:{settings['heartbeat_seconds']}:" + f"{minimum}:{maximum}" + ) + + archive_arguments = [ + "RRA:{consolidation_function}:{xfiles_factor}:{steps}:{rows}".format(**archive) + for archive in archives + ] + run_rrdtool( + "create", str(path), "--start", str(rrd["start"]), + "--step", str(rrd.get("step_seconds", config.interval)), + data_source("temperature"), data_source("humidity"), data_source("battery"), + *archive_arguments, + ) + LOG.info("Created %s", path) + + +def number_or_unknown(reading: dict[str, Any], key: str) -> str: + value = reading.get(key) + if isinstance(value, bool) or not isinstance(value, (int, float)): + return "U" + number = float(value) + return str(number) if math.isfinite(number) else "U" + + +def update_rrd(path: Path, reading: dict[str, Any]) -> None: + values = ":".join( + number_or_unknown(reading, key) + for key in ("temperature", "humidity", "battery") + ) + run_rrdtool( + "update", str(path), "--template", "temperature:humidity:battery", f"N:{values}" + ) + + +def render_graphs(sensor: Sensor, rrd_path: Path, config: Config) -> None: + graph = config.graph + temperature = graph["temperature"] + humidity = graph["humidity"] + battery = graph["battery"] + optional_flags = [] + if graph.get("slope_mode", False): + optional_flags.append("--slope-mode") + if graph.get("alt_autoscale", False): + optional_flags.append("--alt-autoscale") + + for suffix, period in graph["periods"].items(): + output = config.graph_dir / f"{sensor.slug}_{suffix}.png" + run_rrdtool( + "graph", str(output), "--start", str(period["start"]), + "--end", str(graph["end"]), + "--title", f"{sensor.name} - {period['label']}", + "--vertical-label", str(graph["vertical_label"]), + "--width", str(graph["width"]), "--height", str(graph["height"]), + *optional_flags, + f"DEF:temp={rrd_path}:temperature:AVERAGE", + f"DEF:humidity={rrd_path}:humidity:AVERAGE", + f"DEF:battery={rrd_path}:battery:AVERAGE", + f"AREA:temp#{temperature['color']}:{temperature['legend']}", + f"LINE{humidity['line_width']}:humidity#{humidity['color']}:{humidity['legend']}", + f"LINE{battery['line_width']}:battery#{battery['color']}:{battery['legend']}:dashes={battery['dashes']}", + f"GPRINT:temp:LAST:{temperature['last_value_format']}", + f"GPRINT:humidity:LAST:{humidity['last_value_format']}", + f"GPRINT:battery:LAST:{battery['last_value_format']}", + ) + + +def render_dashboard(config: Config) -> None: + settings = config.html + filename = str(settings.get("filename", "index.html")) + if Path(filename).name != filename: + raise SystemExit("html.filename must be a filename, not a path") + + title = str(settings.get("title", "Zigbee sensor history")) + periods = config.graph["periods"] + default_period = str(settings.get("default_period", next(iter(periods)))) + if default_period not in periods: + raise SystemExit(f"html.default_period {default_period!r} is not a graph period") + refresh_seconds = int(settings.get("refresh_seconds", config.interval)) + if refresh_seconds < 0: + raise SystemExit("html.refresh_seconds cannot be negative") + + buttons = "\n".join( + f'' + for key, period in periods.items() + ) + panels: list[str] = [] + for period_key, period in periods.items(): + cards = "\n".join( + "
" + f"

{html.escape(sensor.name)}

" + f'' + f'' + "
" + for sensor in config.sensors + ) + panels.append( + f'' + ) + + document = f""" + + + + + {html.escape(title)} + + + +
+

{html.escape(title)}

+ +
+
{''.join(panels)}
+ + + +""" + output = config.graph_dir / filename + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(document, encoding="utf-8") + temporary.replace(output) + LOG.info("Generated dashboard %s", output) + + +def read_sensors(config: Config) -> dict[str, dict[str, Any]]: + readings: dict[str, dict[str, Any]] = {} + complete = threading.Event() + topics = {sensor.topic for sensor in config.sensors} + lock = threading.Lock() + + def on_connect(client, userdata, flags, reason_code, properties): + if reason_code != 0: + LOG.error("MQTT connection failed: %s", reason_code) + complete.set() + return + for topic in topics: + client.subscribe(topic) + + def on_message(client, userdata, message): + if message.topic not in topics: + return + try: + payload = json.loads(message.payload.decode("utf-8")) + if not isinstance(payload, dict): + raise ValueError("payload is not an object") + except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as error: + LOG.warning("Ignoring invalid message on %s: %s", message.topic, error) + return + with lock: + readings[message.topic] = payload + if readings.keys() >= topics: + complete.set() + + client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2) + if config.mqtt_username is not None: + client.username_pw_set(config.mqtt_username, config.mqtt_password) + client.on_connect = on_connect + client.on_message = on_message + try: + client.connect(config.mqtt_host, config.mqtt_port) + client.loop_start() + complete.wait(config.timeout) + except OSError as error: + LOG.error("Could not connect to %s:%s: %s", config.mqtt_host, config.mqtt_port, error) + finally: + client.disconnect() + client.loop_stop() + + missing = topics - readings.keys() + if missing: + LOG.warning("No reading received for: %s", ", ".join(sorted(missing))) + return readings + + +def collect_once(config: Config) -> None: + config.data_dir.mkdir(parents=True, exist_ok=True) + config.graph_dir.mkdir(parents=True, exist_ok=True) + readings = read_sensors(config) + for sensor in config.sensors: + rrd_path = config.data_dir / f"{sensor.slug}.rrd" + create_rrd(rrd_path, config) + reading = readings.get(sensor.topic) + if reading is not None: + update_rrd(rrd_path, reading) + LOG.info( + "%s: temperature=%s C humidity=%s %% battery=%s %%", + sensor.name, reading.get("temperature", "unknown"), + reading.get("humidity", "unknown"), reading.get("battery", "unknown"), + ) + render_graphs(sensor, rrd_path, config) + render_dashboard(config) + + +def render_only(config: Config) -> None: + config.graph_dir.mkdir(parents=True, exist_ok=True) + for sensor in config.sensors: + rrd_path = config.data_dir / f"{sensor.slug}.rrd" + if rrd_path.exists(): + render_graphs(sensor, rrd_path, config) + else: + LOG.warning("Skipping %s; %s does not exist", sensor.name, rrd_path) + render_dashboard(config) + + +def handle_signal(signum, frame) -> None: + LOG.info("Stopping") + STOP.set() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=BASE_DIR / "config.yaml") + modes = parser.add_mutually_exclusive_group() + modes.add_argument("--once", action="store_true", help="collect and graph once") + modes.add_argument("--graph-only", action="store_true", help="only regenerate graphs") + parser.add_argument("--verbose", action="store_true") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if shutil.which("rrdtool") is None: + raise SystemExit("rrdtool is not installed; run: sudo apt install rrdtool") + config = load_config(args.config.resolve()) + logging.basicConfig( + level=logging.DEBUG if args.verbose else config.log_level.upper(), + format=config.log_format, + ) + signal.signal(signal.SIGINT, handle_signal) + signal.signal(signal.SIGTERM, handle_signal) + + if args.graph_only: + render_only(config) + return 0 + while not STOP.is_set(): + started = time.monotonic() + try: + collect_once(config) + except RuntimeError: + LOG.exception("Collection cycle failed") + if args.once: + return 1 + if args.once: + break + STOP.wait(max(0.0, config.interval - (time.monotonic() - started))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..28c73fd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +paho-mqtt>=2.1,<3 +PyYAML>=6.0.3,<7 diff --git a/zigbee-rrd.service.example b/zigbee-rrd.service.example new file mode 100644 index 0000000..ab04684 --- /dev/null +++ b/zigbee-rrd.service.example @@ -0,0 +1,14 @@ +[Unit] +Description=Zigbee temperature sensor RRD collector +Wants=network-online.target +After=network-online.target + +[Service] +Type=simple +WorkingDirectory=%h/service_zigbee +ExecStart=%h/service_zigbee/.venv/bin/python %h/service_zigbee/read_temperature.py +Restart=on-failure +RestartSec=15 + +[Install] +WantedBy=default.target