more robust import script

README.md

@@ -52,7 +52,26 @@ Preview an import without changing the database:
 python scripts/import_excel.py /path/to/fatima.xlsx
 ```
 
-Add `--apply` to import. The command backs up the existing SQLite file before updating it.
+The preview prints the absolute workbook and database paths, current database
+row counts and date range, discovered workbook values, and overwrite counts.
+Check that the printed database path is the same file used by the production
+app before applying the import.
+
+Add `--apply` to import:
+
+```sh
+python scripts/import_excel.py /path/to/fatima.xlsx --apply
+```
+
+The importer refuses to write if the database is missing, has the wrong schema,
+or the workbook contains no importable data. It creates a consistent SQLite
+backup before writing, reads every imported value back after committing, and
+prints the final row counts and date range. If production uses a non-default
+database location, specify it explicitly:
+
+```sh
+python scripts/import_excel.py /path/to/fatima.xlsx --database /absolute/path/calendar.sqlite --apply
+```
 
 On the iPhone, visit `http://<your-computer-on-the-local-network>:9009/fatima/`.
 Waitress listens on all network interfaces with the command above. For anything

scripts/import_excel.py

@@ -1,5 +1,4 @@
 import argparse
-import shutil
 import sqlite3
 from datetime import date, datetime, timedelta
 from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
@@ -13,6 +12,9 @@ YELLOW = "FFFFFF00"
 ORANGE_THEME = 5
 MONTH_HEADER_ROWS = (2, 11, 20)
 MONTH_START_COLUMNS = (2, 10, 18, 26)
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_DATABASE = PROJECT_ROOT / "instance" / "calendar.sqlite"
+REQUIRED_TABLES = {"date_choices", "date_statuses"}
 
 
 def cell_color(cell):
@@ -77,8 +79,6 @@ def read_workbook(path):
 
 
 def existing_conflicts(database_path, amounts, statuses):
-    if not database_path.exists():
-        return 0, 0
     with sqlite3.connect(database_path) as database:
         existing_amounts = dict(database.execute("SELECT choice_date, amount_cents FROM date_choices"))
         existing_statuses = dict(database.execute("SELECT status_date, day_status FROM date_statuses"))
@@ -87,11 +87,59 @@ def existing_conflicts(database_path, amounts, statuses):
     return amount_conflicts, status_conflicts
 
 
+def inspect_database(database_path):
+    if not database_path.is_file():
+        raise RuntimeError(
+            f"Database does not exist: {database_path}\n"
+            "Start the app once to initialize it, or pass the correct path with --database."
+        )
+
+    with sqlite3.connect(database_path) as database:
+        tables = {
+            row[0]
+            for row in database.execute("SELECT name FROM sqlite_master WHERE type = 'table'")
+        }
+        missing = REQUIRED_TABLES - tables
+        if missing:
+            raise RuntimeError(
+                f"Database has the wrong schema; missing tables: {', '.join(sorted(missing))}"
+            )
+        amount_count = database.execute("SELECT COUNT(*) FROM date_choices").fetchone()[0]
+        status_count = database.execute("SELECT COUNT(*) FROM date_statuses").fetchone()[0]
+        dates = database.execute(
+            """
+            SELECT MIN(recorded_date), MAX(recorded_date) FROM (
+                SELECT choice_date AS recorded_date FROM date_choices
+                UNION ALL
+                SELECT status_date AS recorded_date FROM date_statuses
+            )
+            """
+        ).fetchone()
+    return amount_count, status_count, dates[0], dates[1]
+
+
+def verify_import(database_path, amounts, statuses):
+    with sqlite3.connect(database_path) as database:
+        stored_amounts = dict(
+            database.execute("SELECT choice_date, amount_cents FROM date_choices")
+        )
+        stored_statuses = dict(
+            database.execute("SELECT status_date, day_status FROM date_statuses")
+        )
+    bad_amounts = [key for key, value in amounts.items() if stored_amounts.get(key) != value]
+    bad_statuses = [key for key, value in statuses.items() if stored_statuses.get(key) != value]
+    if bad_amounts or bad_statuses:
+        raise RuntimeError(
+            "Post-import verification failed: "
+            f"{len(bad_amounts)} amounts and {len(bad_statuses)} statuses do not match."
+        )
+
+
 def import_data(database_path, amounts, statuses):
-    timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+    timestamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f")
     backup_path = database_path.with_name(f"{database_path.name}.backup-{timestamp}")
-    if database_path.exists():
+    with sqlite3.connect(database_path) as source, sqlite3.connect(backup_path) as backup:
-        shutil.copy2(database_path, backup_path)
+        source.backup(backup)
 
     with sqlite3.connect(database_path) as database:
         database.executemany(
@@ -112,26 +160,56 @@ def import_data(database_path, amounts, statuses):
             """,
             statuses.items(),
         )
-    return backup_path if backup_path.exists() else None
+    verify_import(database_path, amounts, statuses)
+    return backup_path
 
 
 def main():
     parser = argparse.ArgumentParser(description="Import Fatima calendar data from Excel.")
     parser.add_argument("workbook", type=Path)
-    parser.add_argument("--database", type=Path, default=Path("instance/calendar.sqlite"))
+    parser.add_argument(
+        "--database",
+        type=Path,
+        default=DEFAULT_DATABASE,
+        help=f"SQLite database used by the app (default: {DEFAULT_DATABASE})",
+    )
     parser.add_argument("--apply", action="store_true", help="Write the data; otherwise perform a dry run.")
     args = parser.parse_args()
+    database_path = args.database.expanduser().resolve()
+    workbook_path = args.workbook.expanduser().resolve()
 
-    amounts, statuses = read_workbook(args.workbook)
+    if not workbook_path.is_file():
-    amount_conflicts, status_conflicts = existing_conflicts(args.database, amounts, statuses)
+        parser.error(f"Workbook does not exist: {workbook_path}")
+    try:
+        before_amounts, before_statuses, first_date, last_date = inspect_database(database_path)
+    except RuntimeError as error:
+        parser.error(str(error))
+
+    print(f"Workbook: {workbook_path}")
+    print(f"Database: {database_path}")
+    print(
+        f"Database before import: {before_amounts} amounts, {before_statuses} statuses"
+        + (f", date range {first_date} through {last_date}." if first_date else ", no recorded dates.")
+    )
+
+    amounts, statuses = read_workbook(workbook_path)
+    if not amounts and not statuses:
+        parser.error("Workbook produced no importable amounts or statuses; refusing to continue.")
+    amount_conflicts, status_conflicts = existing_conflicts(database_path, amounts, statuses)
     print(f"Found {len(amounts)} amounts, {sum(v == 'vacation' for v in statuses.values())} vacation days, and {sum(v == 'sick' for v in statuses.values())} sick days.")
     print(f"Existing differing values to overwrite: {amount_conflicts} amounts, {status_conflicts} statuses.")
     if not args.apply:
         print("Dry run only; no database changes made.")
         return
 
-    backup = import_data(args.database, amounts, statuses)
+    backup = import_data(database_path, amounts, statuses)
-    print(f"Import complete. Backup: {backup}" if backup else "Import complete; no previous database existed.")
+    after_amounts, after_statuses, first_date, last_date = inspect_database(database_path)
+    print(f"Import complete and verified: all {len(amounts)} amounts and {len(statuses)} statuses match.")
+    print(f"Backup: {backup}")
+    print(
+        f"Database after import: {after_amounts} amounts, {after_statuses} statuses, "
+        f"date range {first_date} through {last_date}."
+    )
 
 
 if __name__ == "__main__":

tests/test_import_excel.py

@@ -0,0 +1,65 @@
+import sqlite3
+import tempfile
+from pathlib import Path
+
+import pytest
+
+from scripts.import_excel import DEFAULT_DATABASE, import_data, inspect_database, verify_import
+
+
+def database_file():
+    directory = tempfile.TemporaryDirectory()
+    path = Path(directory.name) / "calendar.sqlite"
+    with sqlite3.connect(path) as database:
+        database.execute(
+            "CREATE TABLE date_choices (choice_date TEXT PRIMARY KEY, amount_cents INTEGER, updated_at TEXT DEFAULT CURRENT_TIMESTAMP)"
+        )
+        database.execute(
+            "CREATE TABLE date_statuses (status_date TEXT PRIMARY KEY, day_status TEXT, updated_at TEXT DEFAULT CURRENT_TIMESTAMP)"
+        )
+    return directory, path
+
+
+def test_default_database_is_anchored_to_project_directory():
+    assert DEFAULT_DATABASE.is_absolute()
+    assert DEFAULT_DATABASE.name == "calendar.sqlite"
+    assert DEFAULT_DATABASE.parent.name == "instance"
+
+
+def test_database_inspection_rejects_missing_or_wrong_database():
+    with tempfile.TemporaryDirectory() as directory:
+        missing = Path(directory) / "missing.sqlite"
+        with pytest.raises(RuntimeError, match="does not exist"):
+            inspect_database(missing)
+
+        wrong = Path(directory) / "wrong.sqlite"
+        with sqlite3.connect(wrong) as database:
+            database.execute("CREATE TABLE something_else (value TEXT)")
+        with pytest.raises(RuntimeError, match="wrong schema"):
+            inspect_database(wrong)
+
+
+def test_import_creates_backup_and_verifies_committed_values():
+    directory, path = database_file()
+    try:
+        backup = import_data(
+            path,
+            {"2026-07-15": 2750},
+            {"2026-07-16": "vacation"},
+        )
+
+        assert backup.is_file()
+        assert inspect_database(path) == (1, 1, "2026-07-15", "2026-07-16")
+        verify_import(path, {"2026-07-15": 2750}, {"2026-07-16": "vacation"})
+        assert inspect_database(backup) == (0, 0, None, None)
+    finally:
+        directory.cleanup()
+
+
+def test_verification_detects_values_that_do_not_match():
+    directory, path = database_file()
+    try:
+        with pytest.raises(RuntimeError, match="verification failed"):
+            verify_import(path, {"2026-07-15": 5500}, {})
+    finally:
+        directory.cleanup()