From a5ef20dd82455ca6b69c6a17a3492251b5d48600 Mon Sep 17 00:00:00 2001
From: ParticularlyPythonicBS <anil@anilr.dev>
Date: Tue, 1 Jul 2025 19:31:16 -0400
Subject: [PATCH 1/2]  more robust publish script

---
 .github/scripts/publish_script.py | 61 +++++++++++++++++--------------
 manifest.json                     | 30 ---------------
 2 files changed, 34 insertions(+), 57 deletions(-)

diff --git a/.github/scripts/publish_script.py b/.github/scripts/publish_script.py
index e1e9b58..599da39 100644
--- a/.github/scripts/publish_script.py
+++ b/.github/scripts/publish_script.py
@@ -27,39 +27,46 @@
 
 needs_update = False
 for dataset in manifest_data:
-    latest_entry = dataset["history"][0]
-    if "staging_key" in latest_entry and latest_entry["staging_key"]:
-        needs_update = True
-        staging_key = latest_entry.pop("staging_key")  # Remove the key
-        final_key = latest_entry["r2_object_key"]
-        commit_hash = (
-            subprocess.check_output(
-                ["git", "log", "-1", "--pretty=%h", "--", MANIFEST_FILE]
+    # Check the entire history for a staged entry, not just the latest
+    for i, entry in enumerate(dataset["history"]):
+        if "staging_key" in entry and entry["staging_key"]:
+            needs_update = True
+            staging_key = entry.pop("staging_key")  # Remove the key
+            final_key = entry["r2_object_key"]
+            commit_hash = (
+                subprocess.check_output(
+                    ["git", "log", "-1", "--pretty=%h", "--", MANIFEST_FILE]
+                )
+                .decode()
+                .strip()
             )
-            .decode()
-            .strip()
-        )
-        latest_entry["commit"] = commit_hash
+            entry["commit"] = commit_hash
+            dataset["history"][i] = entry  # Update the entry in the list
 
-        print(
-            f"Publishing: {dataset['fileName']} v{latest_entry['version'].lstrip('v')}"
-        )
-        try:
-            copy_source = {"Bucket": STAGING_BUCKET, "Key": staging_key}
-            client.copy_object(
-                CopySource=copy_source, Bucket=PROD_BUCKET, Key=final_key
-            )
-            print("  ✅ Server-side copy successful.")
-            client.delete_object(Bucket=STAGING_BUCKET, Key=staging_key)
-            print("  ✅ Staging object deleted.")
-        except ClientError as e:
-            print(f"  ❌ ERROR: Could not process object. Reason: {e}")
-            exit(1)
+            print(f"Publishing: {dataset['fileName']} v{entry['version']}")
+            try:
+                copy_source = {"Bucket": STAGING_BUCKET, "Key": staging_key}
+                client.copy_object(
+                    CopySource=copy_source, Bucket=PROD_BUCKET, Key=final_key
+                )
+                print("  ✅ Server-side copy successful.")
+                client.delete_object(Bucket=STAGING_BUCKET, Key=staging_key)
+                print("  ✅ Staging object deleted.")
+                # Break the inner loop after processing one entry
+                break
+            except ClientError as e:
+                print(f"  ❌ ERROR: Could not process object. Reason: {e}")
+                exit(1)
+
+    # Break the outer loop as well to ensure only one dataset is processed per run
+    if needs_update:
+        break
 
 if needs_update:
     print("\nFinalizing manifest file with new commit hash...")
     with open(MANIFEST_FILE, "w") as f:
-        json.dump(manifest_data, f, indent=2)
+        json.dump(manifest_data, f, indent=2, ensure_ascii=False)
+        f.write("\n")  # Add a trailing newline for linters
 
     print("Committing and pushing finalized manifest...")
     subprocess.run(["git", "config", "user.name", "github-actions[bot]"])
diff --git a/manifest.json b/manifest.json
index c39b485..12b4a94 100644
--- a/manifest.json
+++ b/manifest.json
@@ -86,35 +86,5 @@
         "commit": "f253df8"
       }
     ]
-  },
-  {
-    "fileName": "ci_test_data.sqlite",
-    "latestVersion": "v1",
-    "history": [
-      {
-        "version": "v1",
-        "timestamp": "2025-07-01T23:01:39.880086Z",
-        "sha256": "c27a67ce47c2988d2c18010c1e2f140c3e6c0b6ce09da05a80d643fb58f9abd9",
-        "r2_object_key": "ci_test_data/v1-c27a67ce47c2988d2c18010c1e2f140c3e6c0b6ce09da05a80d643fb58f9abd9.sqlite",
-        "staging_key": "staging-uploads/c27a67ce47c2988d2c18010c1e2f140c3e6c0b6ce09da05a80d643fb58f9abd9.sqlite",
-        "diffFromPrevious": null,
-        "commit": "pending-merge"
-      }
-    ]
-  },
-  {
-    "fileName": "ci_test_data_new.sqlite",
-    "latestVersion": "v1",
-    "history": [
-      {
-        "version": "v1",
-        "timestamp": "2025-07-01T23:23:42.912726Z",
-        "sha256": "c27a67ce47c2988d2c18010c1e2f140c3e6c0b6ce09da05a80d643fb58f9abd9",
-        "r2_object_key": "ci_test_data_new/v1-c27a67ce47c2988d2c18010c1e2f140c3e6c0b6ce09da05a80d643fb58f9abd9.sqlite",
-        "staging_key": "staging-uploads/c27a67ce47c2988d2c18010c1e2f140c3e6c0b6ce09da05a80d643fb58f9abd9.sqlite",
-        "diffFromPrevious": null,
-        "commit": "pending-merge"
-      }
-    ]
   }
 ]

From 5191f3c334a083bc97a690f3f96a92cd5a5442bc Mon Sep 17 00:00:00 2001
From: ParticularlyPythonicBS <anil@anilr.dev>
Date: Tue, 1 Jul 2025 19:33:03 -0400
Subject: [PATCH 2/2]  Prepare for ci data test

---
 manifest.json | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/manifest.json b/manifest.json
index 12b4a94..403a55c 100644
--- a/manifest.json
+++ b/manifest.json
@@ -86,5 +86,20 @@
         "commit": "f253df8"
       }
     ]
+  },
+  {
+    "fileName": "ci_test_data.sqlite",
+    "latestVersion": "v1",
+    "history": [
+      {
+        "version": "v1",
+        "timestamp": "2025-07-01T23:31:59.727048Z",
+        "sha256": "c27a67ce47c2988d2c18010c1e2f140c3e6c0b6ce09da05a80d643fb58f9abd9",
+        "r2_object_key": "ci_test_data/v1-c27a67ce47c2988d2c18010c1e2f140c3e6c0b6ce09da05a80d643fb58f9abd9.sqlite",
+        "staging_key": "staging-uploads/c27a67ce47c2988d2c18010c1e2f140c3e6c0b6ce09da05a80d643fb58f9abd9.sqlite",
+        "diffFromPrevious": null,
+        "commit": "pending-merge"
+      }
+    ]
   }
 ]