From 64f1993a84e87948b6569090463ad37ff2397411 Mon Sep 17 00:00:00 2001 From: Ignace Date: Fri, 17 Jul 2026 19:43:19 +0200 Subject: [PATCH] more robust import script --- .DS_Store | Bin 0 -> 10244 bytes README.md | 21 +++- instance/.DS_Store | Bin 0 -> 6148 bytes instance/calendar.sqlite | Bin 20480 -> 20480 bytes .../calendar.sqlite.backup-20260717-192703 | Bin 0 -> 20480 bytes scripts/import_excel.py | 102 +++++++++++++++--- tests/test_import_excel.py | 65 +++++++++++ 7 files changed, 175 insertions(+), 13 deletions(-) create mode 100644 .DS_Store create mode 100644 instance/.DS_Store create mode 100644 instance/calendar.sqlite.backup-20260717-192703 create mode 100644 tests/test_import_excel.py diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..2870ea5648a9680cc00890dbe6901a1c924bc78b GIT binary patch literal 10244 zcmeHMTWl0n7(V~>c7_h@6pCC1+-(BZIwbxYqzBrF5BImfey^hl%3ga zv0$o+Ml^VPk$55D$#~USF7!-ZfioqBkOf>2X7=1Az(f`buZMNI`LexkyCpqUo zm;d|EIp6;O%sI1!5Qt{gjf8{h1I2`*!o?gV0Q(SU50q^nw-pt_LP{5rX?T1`- z0ue6#(r3VDU@ikJTq^Ol9w%eu*sS||yu+|kNyAE_2GfdSuK&|bSR-T!YFor42Z>G& zk$SxIPvGr8jdl#I*P}JoGK#FcQ)H#G$jZDZs4YSTa8?8D+vEtY{2sQlJ1X< zTH`S%;biS-BD0sZ`Yg*x=Ie>%9$g!c%hj!x=_GVh%eVkt)lg;MfNrYRc+|?8s_kN9 ze1M8njLXX=CpS0Og_@e0Pt=7bn>RIL{q8L%PKb1AxM5pw{D_e@t&_L~OyE5&$wJ59c{v4V{?WE zhixn4w7SLyb+pg5DlQuAbQ*uYh=N(X!y0bsIKq-5zQ0=z3^6AXhD3 zQY}ftNT})P2X!seKd!6J=upB=X{M^1hX!)GttSnw&rGDXDyS31>E$a{29#CG?fIc% z2$jQhFq_nmW_9P-^kTUxAk`>gQGSo^Q*m8gmda*Q-4UfmSxf0&*58`Zu;ks+dZkX3 z6SU7p0=S6=X`?9HV&4(;1`chKwoq9W`;A0WGk90Cw2jKU#DN_D4R~j@bUzgbQ&u{S zo3wdfRB5B~WqN>3+?lYQ?lH~GFDa_*pmf+CO=Pt0arDJJqLWhhN5y^yX;*c}%5XMn zlpacln6VaBMLU5NE0_1i6eTXpx|!3_^YJI;0U&=tDGld;ns8)2uZAxsN8FlAbU4z; zyG3-%)npTCBb{V~WXL2rMNX5mGjKgtw9G-wD;Uqi*FT!bf30{V?@CLjIZ^Jn_4FKiWd3Y|iiuv_RB28D!hSg?c% zz5yX0a{GBZUkb;tHebkjPZHtW{2>>7a7V|^UAIoiU0uY3Z0T*wLU*ihXu9XVmKj&% zo||{p!XR+{epa-&e((B5vCm>SDyvpY!F~)3ZK60B^Fbvbg~#A&I0etbbMQQR`z!D&oPpQib@ceR(C5#?1$ZAmfREu4dS1)?kW>By2T`Q7a z(oR}1{<~1t<2zlQjd0H)X9~_CGCn0Va+GAzKC7iMn>e41Xa~kd5WwQ@r zAxqT)Ge}mybM5$D2D{zrcX`1k*R?US2- K|Gzcv|9=7B)LV=I literal 0 HcmV?d00001 diff --git a/README.md b/README.md index 7382145..ab31761 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,26 @@ Preview an import without changing the database: python scripts/import_excel.py /path/to/fatima.xlsx ``` -Add `--apply` to import. The command backs up the existing SQLite file before updating it. +The preview prints the absolute workbook and database paths, current database +row counts and date range, discovered workbook values, and overwrite counts. +Check that the printed database path is the same file used by the production +app before applying the import. + +Add `--apply` to import: + +```sh +python scripts/import_excel.py /path/to/fatima.xlsx --apply +``` + +The importer refuses to write if the database is missing, has the wrong schema, +or the workbook contains no importable data. It creates a consistent SQLite +backup before writing, reads every imported value back after committing, and +prints the final row counts and date range. If production uses a non-default +database location, specify it explicitly: + +```sh +python scripts/import_excel.py /path/to/fatima.xlsx --database /absolute/path/calendar.sqlite --apply +``` On the iPhone, visit `http://:9009/fatima/`. Waitress listens on all network interfaces with the command above. For anything diff --git a/instance/.DS_Store b/instance/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0{M8(Hn3ZhmJ1@Vc0YDEwfU-$sQ+3VfSZ_G^epQ3-vlB4_0 zeKIrq-JRL)_jkANs-daMq-6SAA>ZJ1up={|$qE z-EMPeZ5Nvu3oyG> zyryLD9o;ot7(HO`E*|KeLr;`X)H=NgkL)SgBm0Mj?7`xW!v3L>ow{x2=%LAp)SSNz zx*fQ(II!F9294|*vAgSk(rc%Vm&ePMshQ~%-=CctKa}d}nLCDKw*-To7~@Zk)d9~L z!@&O0(c(yHth8&mxUW1sYVRmcz{Pj|?FlF!nK?FH85^IRuFTrI zMoPt<#ZkNAK+RwY>f375OW(evm*bPyHL3hP(vL+d%~erw$7vz8@)PJWO)m%J}| zV{%J!N#cjZdx^7&Q;F*nxkOw1yZF2Dr{j0WuZ{ch1+i~qZ^zEWPR6c|xiJ*|Ci-Ud z$>@n_DY_||h+K%g5qTnVJaQEuh}nz-#sTAiao~UFK<8jfN0aALH{;N(!~FhAV?S%N z>}EH+nR=ZA5Td!oCsojP31VufAfSxnrqxhCgjiM$ZId9XhWa2xJvFpdK)&N<+wNB# z-4zl<+iy`pTOhe1~zL(mVF5eCmM^(_G`-5~R7tQxMWksdWP4K4sMNt_wmeueM;V z1i5Oc6GBvTWv&qrb=-gmY;){J|#8zD@%OOPb>Z4mGK^mWy3WzWE8lT!B#5DJoB@(2$w=9Mb)l8{H0^*Ck zX2EEaAkBht8HAW-!B_|(s+o5S1eA6>&AcNJay7Ry3?Y_L&lUtBs+o6|1Zn17Qb7EU zpy@6FA>A~OOOU3!7=)sKZZ}^JMm$BJs!c^FvT7CGTe-QMi zSie|bTc29zt=Ft`))Ur4)?L=LHD>L#3YKeiT8sEV%w`-g4j2cF1I7X4fN{V$U>q0)$RI*n2PMmQNzz>lCG|Un45F?zl5{(uWWEh0&8?OsTh)-& z@M)!xLCmy5$RJR<97>vBE=hNpkU<=@6iOc5N|G&slBO368N@b=B+1&KWNH4~TD|yv zhw`87LLq~AWdW3~N1)`F6_hlOB`yg(i#6J#q{g1^V-@fmy^@4;))@92I0 zw%LpW#sTAialklW954jo#YahGI@z3qlKc3%6Y{ZA}dU#D= zpwyoa<(2(9n&nT8^P#-H577Ar(8a6#i)&-$F-<=B`haSjyv|>5(Q{~ep3b{?wZGb8 zH5c)iro}E^@2_Dzdtg+D@ru8|sHekt&A*QE=tA2Xulfs&XzL~>-I)mL|Ag58Q2)R57rE)dIA9zw4j2cF1I7X4fN{V$U>q;F|r{d)bs`h7^ZQUAXv8})xKgD9k4|K~D@JVgDUe-q1N up#C4k7oh&1=RBzY2XO?b|MO9Kp#Gn8Xa>~(gBStS|9Q^if%<7y^Mp9whlQO$vpQ&@?82LOXNb)%?b{ zQ~F2U_K#T*+OPI=^Z4$}?CtM2I{n4@wa%*hwT*i(v^%aB+ZKz*W0S5Mi^Yb-&!G4T zk3MlS5DM`-4$e)dL$Sk`b|UgwYyi<1eww_9D1N{M+rL`}tOM2o>wtB@I$#~J4p;}Q z1J;56CkGy$?N5&6^YKScblP{FUtMW;R$H5$cIV>e>ZT?Jrk5I1&4$~YIzHcUwM*`J z#=YjaT0~1?ZnN>_rh95>ZeeQatb3wy_D~PK(taSQ^eVh~vgt0KnV)xO8naVp=9_N* z8yokYTV2Wbd{S0BeMe*ZggXujnOk(n!;c(t^Y^uv+nu$I^}IOWTw6YuFBE#)aPc15 z*p-%eYwR|-rw!9*mX;ce%~o@6p>eu7wQx#nOrbWA7+J{2V{7XxtM_ky{k(V!ZM83U zHoCfN#Fnourr9_6a4Mc085@g#_q4hP%Xe?AEeE$CIPdGd1HqMu8*uf?zk37P7d9@g zcUsG<>zz$^Zn4?;LSxC@av(5RgN{zP^g6g$=}U~982gtl27L}p3{=0EQz<#G5P3c} zlSs$Nd*sJtoy?Jg_#gOP{0#mPK8I)VDEd2k8$E@7i0($yXrJ@B^M>=JbIDnCCY?Rm zE7?C}f1mwc_O9$ic6a8J%wtCOzv)1(F)-Ta zamvd@S}D@%@I&D_^IgvB)T>jTo6#WF1M#|F%S|h&T;!z)s8q`xhY+hoL4IyZgJ=}w z(f33RmCDU*@+qjdlghEvht!E1>FQ8T8r-89t|@1bfbdA zVsG&21_-g>{5`GY$$6vvV_!y>m6z{U`+D?Ob+VaDM0f+&m_m9TIHwId5Xp+q+DZ{tZP_oK#P0}eSsW+*l46`27Bs~fx z^CqCA<=d5%;n3TlWPTk=TCHi4Rh5)s%LVxLCO3B zP}1tCCYie>yJ4#RO3HxLJ|$)7X)lztc9SOg9wlYq=td}6^#)C{-B60at&%dpvrCg~ z7)n;!38e_nC@BLm;@!177yKd>qt4q7B`a@(lGcbOIaX4JMi7*&=4g^b)CdR<($hjqX@U>&dySO=^F)&c8)b-+4c9k32ue+LR8m%lei^;X6H zpKyD=dWj46=XvFV$mlDS`ew9L5PAI&t%!F@Gs+7hyAM!v19){z&dySO=^F)&c8)b>M&LK#)rpTj*uzCCvYKCH2Gne|LjD zuZ8*ltCENLe<5WkB+UN{DMKDA|1bWd%VQw_FT)og|6db4$p6c51jzr3R(T-*UoFxy Y$p6bQLciJ$AmTh;nEx019b{PGpGxFOb^rhX diff --git a/instance/calendar.sqlite.backup-20260717-192703 b/instance/calendar.sqlite.backup-20260717-192703 new file mode 100644 index 0000000000000000000000000000000000000000..03e65c71af3e1b7cef1672627142f982e99315c1 GIT binary patch literal 20480 zcmeI%O;6h}7zc2tw1AKb9FTf_E>&6yp&|}w2PU#&plX_x)Ro{Ac}dVBpc%yF-yfAOHafKmY;|fB*y_0D=EW z;PG%_siN!Jb6d$Tm%b;J??g(fVdO`dMzLvgLvRwtCzDgAi)t(Mri^dlBw-}@si3ai z>KOJJwfWg*F4~h1=|jiC)@MOh-!$of9~ymAkbW25T=<@zyE1uf^O!f=RGYd<%c9!k zl11GB~|4>}4 zbaagc1JA#Yt}o+Ns3V6e9JNOfuddE^hOhLvtSwclRqd(wI|uI9FmTfpr1e5R18L7Y z2^jbMZ35&~I2}VmL>QRx9L?N1w literal 0 HcmV?d00001 diff --git a/scripts/import_excel.py b/scripts/import_excel.py index b0c53a6..5a0b896 100644 --- a/scripts/import_excel.py +++ b/scripts/import_excel.py @@ -1,5 +1,4 @@ import argparse -import shutil import sqlite3 from datetime import date, datetime, timedelta from decimal import Decimal, InvalidOperation, ROUND_HALF_UP @@ -13,6 +12,9 @@ YELLOW = "FFFFFF00" ORANGE_THEME = 5 MONTH_HEADER_ROWS = (2, 11, 20) MONTH_START_COLUMNS = (2, 10, 18, 26) +PROJECT_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_DATABASE = PROJECT_ROOT / "instance" / "calendar.sqlite" +REQUIRED_TABLES = {"date_choices", "date_statuses"} def cell_color(cell): @@ -77,8 +79,6 @@ def read_workbook(path): def existing_conflicts(database_path, amounts, statuses): - if not database_path.exists(): - return 0, 0 with sqlite3.connect(database_path) as database: existing_amounts = dict(database.execute("SELECT choice_date, amount_cents FROM date_choices")) existing_statuses = dict(database.execute("SELECT status_date, day_status FROM date_statuses")) @@ -87,11 +87,59 @@ def existing_conflicts(database_path, amounts, statuses): return amount_conflicts, status_conflicts +def inspect_database(database_path): + if not database_path.is_file(): + raise RuntimeError( + f"Database does not exist: {database_path}\n" + "Start the app once to initialize it, or pass the correct path with --database." + ) + + with sqlite3.connect(database_path) as database: + tables = { + row[0] + for row in database.execute("SELECT name FROM sqlite_master WHERE type = 'table'") + } + missing = REQUIRED_TABLES - tables + if missing: + raise RuntimeError( + f"Database has the wrong schema; missing tables: {', '.join(sorted(missing))}" + ) + amount_count = database.execute("SELECT COUNT(*) FROM date_choices").fetchone()[0] + status_count = database.execute("SELECT COUNT(*) FROM date_statuses").fetchone()[0] + dates = database.execute( + """ + SELECT MIN(recorded_date), MAX(recorded_date) FROM ( + SELECT choice_date AS recorded_date FROM date_choices + UNION ALL + SELECT status_date AS recorded_date FROM date_statuses + ) + """ + ).fetchone() + return amount_count, status_count, dates[0], dates[1] + + +def verify_import(database_path, amounts, statuses): + with sqlite3.connect(database_path) as database: + stored_amounts = dict( + database.execute("SELECT choice_date, amount_cents FROM date_choices") + ) + stored_statuses = dict( + database.execute("SELECT status_date, day_status FROM date_statuses") + ) + bad_amounts = [key for key, value in amounts.items() if stored_amounts.get(key) != value] + bad_statuses = [key for key, value in statuses.items() if stored_statuses.get(key) != value] + if bad_amounts or bad_statuses: + raise RuntimeError( + "Post-import verification failed: " + f"{len(bad_amounts)} amounts and {len(bad_statuses)} statuses do not match." + ) + + def import_data(database_path, amounts, statuses): - timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") + timestamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") backup_path = database_path.with_name(f"{database_path.name}.backup-{timestamp}") - if database_path.exists(): - shutil.copy2(database_path, backup_path) + with sqlite3.connect(database_path) as source, sqlite3.connect(backup_path) as backup: + source.backup(backup) with sqlite3.connect(database_path) as database: database.executemany( @@ -112,26 +160,56 @@ def import_data(database_path, amounts, statuses): """, statuses.items(), ) - return backup_path if backup_path.exists() else None + verify_import(database_path, amounts, statuses) + return backup_path def main(): parser = argparse.ArgumentParser(description="Import Fatima calendar data from Excel.") parser.add_argument("workbook", type=Path) - parser.add_argument("--database", type=Path, default=Path("instance/calendar.sqlite")) + parser.add_argument( + "--database", + type=Path, + default=DEFAULT_DATABASE, + help=f"SQLite database used by the app (default: {DEFAULT_DATABASE})", + ) parser.add_argument("--apply", action="store_true", help="Write the data; otherwise perform a dry run.") args = parser.parse_args() + database_path = args.database.expanduser().resolve() + workbook_path = args.workbook.expanduser().resolve() - amounts, statuses = read_workbook(args.workbook) - amount_conflicts, status_conflicts = existing_conflicts(args.database, amounts, statuses) + if not workbook_path.is_file(): + parser.error(f"Workbook does not exist: {workbook_path}") + try: + before_amounts, before_statuses, first_date, last_date = inspect_database(database_path) + except RuntimeError as error: + parser.error(str(error)) + + print(f"Workbook: {workbook_path}") + print(f"Database: {database_path}") + print( + f"Database before import: {before_amounts} amounts, {before_statuses} statuses" + + (f", date range {first_date} through {last_date}." if first_date else ", no recorded dates.") + ) + + amounts, statuses = read_workbook(workbook_path) + if not amounts and not statuses: + parser.error("Workbook produced no importable amounts or statuses; refusing to continue.") + amount_conflicts, status_conflicts = existing_conflicts(database_path, amounts, statuses) print(f"Found {len(amounts)} amounts, {sum(v == 'vacation' for v in statuses.values())} vacation days, and {sum(v == 'sick' for v in statuses.values())} sick days.") print(f"Existing differing values to overwrite: {amount_conflicts} amounts, {status_conflicts} statuses.") if not args.apply: print("Dry run only; no database changes made.") return - backup = import_data(args.database, amounts, statuses) - print(f"Import complete. Backup: {backup}" if backup else "Import complete; no previous database existed.") + backup = import_data(database_path, amounts, statuses) + after_amounts, after_statuses, first_date, last_date = inspect_database(database_path) + print(f"Import complete and verified: all {len(amounts)} amounts and {len(statuses)} statuses match.") + print(f"Backup: {backup}") + print( + f"Database after import: {after_amounts} amounts, {after_statuses} statuses, " + f"date range {first_date} through {last_date}." + ) if __name__ == "__main__": diff --git a/tests/test_import_excel.py b/tests/test_import_excel.py new file mode 100644 index 0000000..f87f11f --- /dev/null +++ b/tests/test_import_excel.py @@ -0,0 +1,65 @@ +import sqlite3 +import tempfile +from pathlib import Path + +import pytest + +from scripts.import_excel import DEFAULT_DATABASE, import_data, inspect_database, verify_import + + +def database_file(): + directory = tempfile.TemporaryDirectory() + path = Path(directory.name) / "calendar.sqlite" + with sqlite3.connect(path) as database: + database.execute( + "CREATE TABLE date_choices (choice_date TEXT PRIMARY KEY, amount_cents INTEGER, updated_at TEXT DEFAULT CURRENT_TIMESTAMP)" + ) + database.execute( + "CREATE TABLE date_statuses (status_date TEXT PRIMARY KEY, day_status TEXT, updated_at TEXT DEFAULT CURRENT_TIMESTAMP)" + ) + return directory, path + + +def test_default_database_is_anchored_to_project_directory(): + assert DEFAULT_DATABASE.is_absolute() + assert DEFAULT_DATABASE.name == "calendar.sqlite" + assert DEFAULT_DATABASE.parent.name == "instance" + + +def test_database_inspection_rejects_missing_or_wrong_database(): + with tempfile.TemporaryDirectory() as directory: + missing = Path(directory) / "missing.sqlite" + with pytest.raises(RuntimeError, match="does not exist"): + inspect_database(missing) + + wrong = Path(directory) / "wrong.sqlite" + with sqlite3.connect(wrong) as database: + database.execute("CREATE TABLE something_else (value TEXT)") + with pytest.raises(RuntimeError, match="wrong schema"): + inspect_database(wrong) + + +def test_import_creates_backup_and_verifies_committed_values(): + directory, path = database_file() + try: + backup = import_data( + path, + {"2026-07-15": 2750}, + {"2026-07-16": "vacation"}, + ) + + assert backup.is_file() + assert inspect_database(path) == (1, 1, "2026-07-15", "2026-07-16") + verify_import(path, {"2026-07-15": 2750}, {"2026-07-16": "vacation"}) + assert inspect_database(backup) == (0, 0, None, None) + finally: + directory.cleanup() + + +def test_verification_detects_values_that_do_not_match(): + directory, path = database_file() + try: + with pytest.raises(RuntimeError, match="verification failed"): + verify_import(path, {"2026-07-15": 5500}, {}) + finally: + directory.cleanup()