1st POC

netatmo_service/__init__.py

@@ -0,0 +1,6 @@
+"""Netatmo to RRDtool web service."""
+
+from .app import create_app
+
+__all__ = ["create_app"]
+

netatmo_service/app.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import logging
+import os
+
+from flask import Flask, Response, abort, jsonify, render_template
+
+from .collector import Collector
+from .config import Config
+from .netatmo import NetatmoClient
+from .rrd import ALIASES, PERIODS, RRD_ERROR, RRDStore, SCHEMAS
+
+
+def create_app(test_config: dict | None = None) -> Flask:
+    app = Flask(__name__)
+    app.config.from_object(Config)
+    if test_config:
+        app.config.update(test_config)
+
+    logging.basicConfig(level=app.config.get("LOG_LEVEL", "INFO"))
+    store = app.config.get("RRD_STORE") or RRDStore(
+        app.config["RRD_FOLDER"], app.config["GRAPH_WIDTH"], app.config["GRAPH_HEIGHT"]
+    )
+    store.ensure_all()
+    client = app.config.get("NETATMO_CLIENT") or NetatmoClient(app.config)
+    names = {key: app.config[f"MODULE_{key.upper()}"] for key in SCHEMAS}
+    collector = Collector(client, store, names, app.config["POLL_INTERVAL"])
+    app.extensions["rrd_store"] = store
+    app.extensions["netatmo_collector"] = collector
+
+    # Flask's debug reloader imports twice. Only its serving child starts a poller.
+    should_start = app.config["START_COLLECTOR"] and client.configured
+    if should_start and (not app.debug or os.environ.get("WERKZEUG_RUN_MAIN") == "true"):
+        collector.start()
+    elif app.config["START_COLLECTOR"] and not client.configured:
+        app.logger.warning("Collector disabled: configure NETATMO_ACCESS_TOKEN or OAuth refresh credentials")
+
+    @app.get("/")
+    def dashboard():
+        return render_template("dashboard.html", rrd_names=list(SCHEMAS), periods=list(PERIODS))
+
+    @app.get("/health")
+    def health():
+        return jsonify(status="ok", collector_configured=client.configured)
+
+    @app.get("/last/<rrd_name>/<data_point>")
+    def last(rrd_name: str, data_point: str):
+        try:
+            return jsonify(store.last(rrd_name.lower(), data_point.lower()))
+        except KeyError:
+            abort(404, description="Unknown RRD or data point")
+        except (OSError, RuntimeError, RRD_ERROR):
+            app.logger.exception("Could not read RRD")
+            abort(503, description="RRD data is unavailable")
+
+    @app.get("/graph/<rrd_name>/<period>")
+    def graph(rrd_name: str, period: str):
+        try:
+            image = store.graph(rrd_name.lower(), ALIASES.get(period.lower(), period.lower()))
+            return Response(image, mimetype="image/png", headers={"Cache-Control": "no-cache, max-age=0"})
+        except KeyError:
+            abort(404, description="Unknown RRD or period")
+        except (OSError, RuntimeError, RRD_ERROR):
+            app.logger.exception("Could not render graph")
+            abort(503, description="RRD graph is unavailable")
+
+    # Compatibility with the compact form requested as /graph/[rrdname][period].
+    @app.get("/graph/<rrd_and_period>")
+    def compact_graph(rrd_and_period: str):
+        for name in SCHEMAS:
+            if rrd_and_period.startswith(name):
+                return graph(name, rrd_and_period[len(name):].lstrip("-_"))
+        abort(404, description="Unknown RRD or period")
+
+    @app.cli.command("collect-now")
+    def collect_now():
+        """Fetch Netatmo and update every RRD immediately."""
+        count = collector.collect_once()
+        print(f"Updated {count} RRDs")
+
+    return app

netatmo_service/collector.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import logging
+import threading
+import time
+
+from .data import extract_samples
+
+LOG = logging.getLogger(__name__)
+
+
+class Collector:
+    def __init__(self, client, store, module_names: dict[str, str], interval: int = 600):
+        self.client = client
+        self.store = store
+        self.module_names = module_names
+        self.interval = interval
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+
+    def collect_once(self) -> int:
+        samples = extract_samples(self.client.stations_data(), self.module_names)
+        for name, sample in samples.items():
+            try:
+                self.store.update(name, sample)
+            except Exception:
+                # One stale/duplicate module must not discard the other modules.
+                LOG.exception("Could not update %s RRD", name)
+        LOG.info("Collected %d Netatmo modules", len(samples))
+        return len(samples)
+
+    def start(self) -> None:
+        if self._thread and self._thread.is_alive():
+            return
+        self._thread = threading.Thread(target=self._loop, name="netatmo-collector", daemon=True)
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._stop.set()
+
+    def _loop(self) -> None:
+        # Collect immediately, then align roughly to the configured cadence.
+        while not self._stop.is_set():
+            started = time.monotonic()
+            try:
+                self.collect_once()
+            except Exception:
+                LOG.exception("Netatmo collection failed")
+            remaining = max(1, self.interval - (time.monotonic() - started))
+            self._stop.wait(remaining)
+

netatmo_service/config.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+def _bool(name: str, default: bool) -> bool:
+    value = os.getenv(name)
+    return default if value is None else value.lower() in {"1", "true", "yes", "on"}
+
+
+class Config:
+    RRD_FOLDER = Path(os.getenv("RRD_FOLDER", "./rrd")).expanduser().resolve()
+    GRAPH_WIDTH = int(os.getenv("GRAPH_WIDTH", "900"))
+    GRAPH_HEIGHT = int(os.getenv("GRAPH_HEIGHT", "240"))
+    POLL_INTERVAL = int(os.getenv("POLL_INTERVAL", "600"))
+    START_COLLECTOR = _bool("START_COLLECTOR", True)
+
+    NETATMO_CLIENT_ID = os.getenv("NETATMO_CLIENT_ID", "")
+    NETATMO_CLIENT_SECRET = os.getenv("NETATMO_CLIENT_SECRET", "")
+    NETATMO_REFRESH_TOKEN = os.getenv("NETATMO_REFRESH_TOKEN", "")
+    NETATMO_ACCESS_TOKEN = os.getenv("NETATMO_ACCESS_TOKEN", "")
+    NETATMO_DEVICE_ID = os.getenv("NETATMO_DEVICE_ID", "")
+    NETATMO_TOKEN_URL = os.getenv("NETATMO_TOKEN_URL", "https://api.netatmo.com/oauth2/token")
+    NETATMO_STATIONS_URL = os.getenv(
+        "NETATMO_STATIONS_URL", "https://api.netatmo.com/api/getstationsdata"
+    )
+
+    MODULE_OUTDOOR = os.getenv("MODULE_OUTDOOR", "Outdoor")
+    MODULE_WIND = os.getenv("MODULE_WIND", "Wind")
+    MODULE_BEDROOM = os.getenv("MODULE_BEDROOM", "Bedroom")
+    MODULE_STUDY = os.getenv("MODULE_STUDY", "Study")
+    MODULE_LIVING = os.getenv("MODULE_LIVING", "Living")

netatmo_service/data.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Sample:
+    timestamp: int
+    values: dict[str, float | None]
+
+
+def _dashboard(item: dict[str, Any]) -> dict[str, Any]:
+    return item.get("dashboard_data") or {}
+
+
+def extract_samples(payload: dict, names: dict[str, str]) -> dict[str, Sample]:
+    """Translate a getstationsdata response into the service's five RRD samples."""
+    devices = payload.get("body", {}).get("devices", [])
+    if not devices:
+        raise ValueError("Netatmo response contains no weather station")
+
+    device = devices[0]
+    items = [device, *device.get("modules", [])]
+    by_name = {item.get("module_name"): item for item in items}
+    result: dict[str, Sample] = {}
+
+    def sample(key: str, fields: dict[str, str]) -> None:
+        item = by_name.get(names[key])
+        if not item or not _dashboard(item):
+            return
+        data = _dashboard(item)
+        result[key] = Sample(
+            int(data["time_utc"]),
+            {target: _number(data.get(source)) for target, source in fields.items()},
+        )
+
+    sample("bedroom", {"temperature": "Temperature", "co2": "CO2", "humidity": "Humidity"})
+    sample("study", {"temperature": "Temperature", "co2": "CO2", "humidity": "Humidity"})
+    sample("living", {"temperature": "Temperature", "co2": "CO2", "humidity": "Humidity"})
+    sample("wind", {"gust": "GustStrength", "average": "WindStrength", "angle": "WindAngle"})
+
+    outdoor = by_name.get(names["outdoor"])
+    if outdoor and _dashboard(outdoor):
+        data = _dashboard(outdoor)
+        # Pressure is measured by the main indoor station; associate it with the
+        # outdoor/weather graph while preserving the outdoor module timestamp.
+        pressure = _dashboard(device).get("Pressure")
+        result["outdoor"] = Sample(
+            int(data["time_utc"]),
+            {
+                "min_temp": _number(data.get("min_temp", data.get("Temperature"))),
+                "max_temp": _number(data.get("max_temp", data.get("Temperature"))),
+                "humidity": _number(data.get("Humidity")),
+                "pressure": _number(pressure),
+            },
+        )
+    return result
+
+
+def _number(value: Any) -> float | None:
+    if value is None:
+        return None
+    return float(value)
+

netatmo_service/netatmo.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import json
+import logging
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+LOG = logging.getLogger(__name__)
+
+
+class NetatmoError(RuntimeError):
+    pass
+
+
+class NetatmoClient:
+    """Small dependency-free Netatmo OAuth and Weather API client."""
+
+    def __init__(self, config):
+        self.client_id = config["NETATMO_CLIENT_ID"]
+        self.client_secret = config["NETATMO_CLIENT_SECRET"]
+        self.refresh_token = config["NETATMO_REFRESH_TOKEN"]
+        self.access_token = config["NETATMO_ACCESS_TOKEN"]
+        self.device_id = config["NETATMO_DEVICE_ID"]
+        self.token_url = config["NETATMO_TOKEN_URL"]
+        self.stations_url = config["NETATMO_STATIONS_URL"]
+        self._expires_at = 0.0
+
+    @property
+    def configured(self) -> bool:
+        return bool(self.access_token or (self.client_id and self.client_secret and self.refresh_token))
+
+    def _request(self, request: urllib.request.Request) -> dict:
+        try:
+            with urllib.request.urlopen(request, timeout=30) as response:
+                return json.load(response)
+        except (urllib.error.URLError, urllib.error.HTTPError, ValueError) as exc:
+            detail = getattr(exc, "read", lambda: b"")().decode(errors="replace")
+            raise NetatmoError(f"Netatmo request failed: {exc}; {detail}") from exc
+
+    def _refresh(self) -> None:
+        data = urllib.parse.urlencode(
+            {
+                "grant_type": "refresh_token",
+                "refresh_token": self.refresh_token,
+                "client_id": self.client_id,
+                "client_secret": self.client_secret,
+            }
+        ).encode()
+        result = self._request(urllib.request.Request(self.token_url, data=data, method="POST"))
+        self.access_token = result["access_token"]
+        # Netatmo may rotate refresh tokens; retain the new value for this process.
+        self.refresh_token = result.get("refresh_token", self.refresh_token)
+        self._expires_at = time.time() + int(result.get("expires_in", 10800)) - 60
+
+    def stations_data(self) -> dict:
+        if not self.configured:
+            raise NetatmoError("Netatmo credentials are not configured")
+        if not self.access_token or (self.refresh_token and time.time() >= self._expires_at):
+            self._refresh()
+        query = {"get_favorites": "false"}
+        if self.device_id:
+            query["device_id"] = self.device_id
+        url = f"{self.stations_url}?{urllib.parse.urlencode(query)}"
+        request = urllib.request.Request(url, headers={"Authorization": f"Bearer {self.access_token}"})
+        try:
+            return self._request(request)
+        except NetatmoError:
+            # A statically supplied access token may expire. Refresh once when possible.
+            if not self.refresh_token:
+                raise
+            LOG.info("Access token rejected; refreshing and retrying once")
+            self._refresh()
+            request.headers["Authorization"] = f"Bearer {self.access_token}"
+            return self._request(request)
+

netatmo_service/rrd.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import colorsys
+import re
+import tempfile
+from pathlib import Path
+
+import rrdtool
+
+from .data import Sample
+
+RRD_ERROR = rrdtool.OperationalError
+
+STEP = 600
+HEARTBEAT = 1500
+SAFE_NAME = re.compile(r"^[a-z][a-z0-9_]*$")
+
+SCHEMAS = {
+    "outdoor": {"min_temp": "GAUGE:-60:70", "max_temp": "GAUGE:-60:70", "humidity": "GAUGE:0:100", "pressure": "GAUGE:800:1200"},
+    "wind": {"gust": "GAUGE:0:300", "average": "GAUGE:0:300", "angle": "GAUGE:0:360"},
+    "bedroom": {"temperature": "GAUGE:-20:60", "co2": "GAUGE:0:10000", "humidity": "GAUGE:0:100"},
+    "study": {"temperature": "GAUGE:-20:60", "co2": "GAUGE:0:10000", "humidity": "GAUGE:0:100"},
+    "living": {"temperature": "GAUGE:-20:60", "co2": "GAUGE:0:10000", "humidity": "GAUGE:0:100"},
+}
+
+PERIODS = {"day": ("1 day", "-1d"), "2weeks": ("2 weeks", "-14d"), "year": ("1 year", "-1y")}
+ALIASES = {"1day": "day", "14days": "2weeks", "2week": "2weeks", "1year": "year"}
+
+
+class RRDStore:
+    def __init__(self, folder: Path, width: int = 900, height: int = 240):
+        self.folder = Path(folder)
+        self.width = width
+        self.height = height
+
+    def path(self, name: str) -> Path:
+        if name not in SCHEMAS:
+            raise KeyError(name)
+        return self.folder / f"{name}.rrd"
+
+    def ensure_all(self) -> None:
+        self.folder.mkdir(parents=True, exist_ok=True)
+        for name, schema in SCHEMAS.items():
+            path = self.path(name)
+            if path.exists():
+                continue
+            # Leave room for a station that has been temporarily offline when the
+            # RRD is first created; RRDtool rejects samples older than --start.
+            args = ["--step", str(STEP), "--start", "now-1d"]
+            args += [f"DS:{field}:{definition.split(':', 1)[0]}:{HEARTBEAT}:{definition.split(':', 1)[1]}" for field, definition in schema.items()]
+            # 10-minute points for 2 weeks; 6-hour averages for over a year.
+            args += ["RRA:AVERAGE:0.5:1:2016", "RRA:MIN:0.5:1:2016", "RRA:MAX:0.5:1:2016",
+                     "RRA:AVERAGE:0.5:36:1464", "RRA:MIN:0.5:36:1464", "RRA:MAX:0.5:36:1464"]
+            rrdtool.create(str(path), *args)
+
+    def update(self, name: str, sample: Sample) -> None:
+        schema = SCHEMAS[name]
+        fields = list(schema)
+        values = ["U" if sample.values.get(field) is None else str(sample.values[field]) for field in fields]
+        rrdtool.update(
+            str(self.path(name)),
+            "--template", ":".join(fields),
+            f"{sample.timestamp}:{':'.join(values)}",
+        )
+
+    def last(self, name: str, field: str) -> dict:
+        if name not in SCHEMAS or field not in SCHEMAS[name]:
+            raise KeyError(f"{name}/{field}")
+        latest = rrdtool.lastupdate(str(self.path(name)))
+        value = latest["ds"][field]
+        return {
+            "rrd": name,
+            "data_point": field,
+            "timestamp": int(latest["date"].timestamp()),
+            "value": None if value is None else float(value),
+        }
+
+    def graph(self, name: str, period: str) -> bytes:
+        period = ALIASES.get(period, period)
+        if name not in SCHEMAS or period not in PERIODS:
+            raise KeyError(f"{name}/{period}")
+        title, start = PERIODS[period]
+        common = ["--imgformat", "PNG", "--start", start, "--end", "now",
+                  "--width", str(self.width), "--height", str(self.height), "--title", f"{name.title()} - {title}",
+                  "--slope-mode", "--watermark", "GetNetatmoData v2"]
+        definitions = [f"DEF:{field}={self.path(name)}:{field}:AVERAGE" for field in SCHEMAS[name]]
+        if name == "wind":
+            drawings = self._wind_drawings()
+        elif name == "outdoor":
+            drawings = ["LINE2:min_temp#3488DB:Minimum temperature (C)", "LINE2:max_temp#E74C3C:Maximum temperature (C)",
+                        "LINE1:humidity#27AE60:Humidity (%)", "LINE1:pressure#8E44AD:Pressure (hPa):dashes"]
+        else:
+            drawings = ["LINE2:temperature#E74C3C:Temperature (C)", "LINE1:co2#8E44AD:CO2 (ppm)", "LINE1:humidity#27AE60:Humidity (%):dashes"]
+        # The Python binding returns graph metadata, not the encoded image, so
+        # render to an isolated temporary file and return its contents.
+        with tempfile.TemporaryDirectory(prefix="netatmo-graph-") as folder:
+            image_path = Path(folder) / f"{name}-{period}.png"
+            rrdtool.graph(str(image_path), *(common + definitions + drawings))
+            return image_path.read_bytes()
+
+    @staticmethod
+    def _wind_drawings() -> list[str]:
+        result: list[str] = []
+        # Split average wind into 12 angle bands. Each AREA starts at zero and
+        # only exists for its direction, producing rainbow-colored bars/areas.
+        for index in range(12):
+            low, high = index * 30, (index + 1) * 30
+            color = "#%02X%02X%02X80" % tuple(round(channel * 255) for channel in colorsys.hsv_to_rgb(index / 12, .85, .9))
+            result += [f"CDEF:dir{index}=angle,{low},GE,angle,{high},LT,*,average,UNKN,IF", f"AREA:dir{index}{color}:{low:03d}-{high:03d} degrees"]
+        result += ["LINE2:gust#111111:Gust", "LINE1:average#555555:Average"]
+        return result

netatmo_service/static/dashboard.css

@@ -0,0 +1,15 @@
+:root { color-scheme: dark; --ink:#eef3ec; --muted:#a7b3a8; --panel:#17221e; --accent:#b6e36f; }
+* { box-sizing:border-box; }
+body { margin:0; padding:2rem; background:#0c1411; color:var(--ink); font:16px/1.5 system-ui,sans-serif; }
+header { max-width:1200px; margin:auto auto 2rem; display:flex; align-items:end; justify-content:space-between; gap:1.5rem; }
+h1 { margin:.1rem 0 0; font-size:clamp(2rem,5vw,4rem); letter-spacing:-.05em; }
+.eyebrow { color:var(--accent); text-transform:uppercase; letter-spacing:.15em; font-size:.75rem; }
+nav { display:flex; padding:.3rem; border:1px solid #304139; border-radius:999px; background:#111b17; }
+button { border:0; border-radius:999px; padding:.65rem 1rem; color:var(--muted); background:transparent; cursor:pointer; }
+button.active { color:#172012; background:var(--accent); }
+main { max-width:1200px; margin:auto; display:grid; grid-template-columns:repeat(auto-fit,minmax(min(100%,480px),1fr)); gap:1rem; }
+.card { overflow:hidden; padding:1rem; border:1px solid #26362f; border-radius:1rem; background:var(--panel); box-shadow:0 18px 45px #0004; }
+h2 { margin:0 0 .75rem; font-size:1rem; color:var(--muted); font-weight:600; }
+img { display:block; width:100%; min-height:180px; object-fit:contain; background:white; border-radius:.45rem; }
+@media (max-width:700px) { body{padding:1rem} header{align-items:start;flex-direction:column} nav{width:100%} button{flex:1} }
+

netatmo_service/templates/dashboard.html

@@ -0,0 +1,37 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <title>Netatmo weather graphs</title>
+  <link rel="stylesheet" href="{{ url_for('static', filename='dashboard.css') }}">
+</head>
+<body>
+  <header>
+    <div><span class="eyebrow">GetNetatmoData v2</span><h1>Weather at home</h1></div>
+    <nav aria-label="Graph period">
+      <button data-period="day" class="active">1 day</button>
+      <button data-period="2weeks">2 weeks</button>
+      <button data-period="year">1 year</button>
+    </nav>
+  </header>
+  <main>
+    {% for name in rrd_names %}
+    <section class="card">
+      <h2>{{ name|title }}</h2>
+      <img src="{{ url_for('graph', rrd_name=name, period='day') }}" alt="{{ name|title }} measurements" data-name="{{ name }}">
+    </section>
+    {% endfor %}
+  </main>
+  <script>
+    const buttons = document.querySelectorAll('button[data-period]');
+    buttons.forEach(button => button.addEventListener('click', () => {
+      buttons.forEach(item => item.classList.toggle('active', item === button));
+      document.querySelectorAll('img[data-name]').forEach(image => {
+        image.src = `/graph/${image.dataset.name}/${button.dataset.period}?t=${Date.now()}`;
+      });
+    }));
+  </script>
+</body>
+</html>
+