Fix weekly advisory duplicate publish

Author: Pigbibi

.github/workflows/publish_advisory_site.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       as_of:
-        description: "Report date. Defaults to current UTC date when empty."
+        description: "Report date. Defaults to the most recent Saturday when empty."
         required: false
         type: string
       site_url:
@@ -111,7 +111,11 @@ jobs:
           MARKET_DATA_PROXY_POOL_URL: ${{ github.event.inputs.market_data_proxy_pool_url || vars.MARKET_DATA_PROXY_POOL_URL || '' }}
         run: |
           set -euo pipefail
-          AS_OF="${INPUT_AS_OF:-$(date -u +%F)}"
+          if [ -n "${INPUT_AS_OF}" ]; then
+            AS_OF="${INPUT_AS_OF}"
+          else
+            AS_OF="$(python -c 'from quant_advisor_research.build_pipeline import default_weekly_as_of; print(default_weekly_as_of().isoformat())')"
+          fi
           mkdir -p .cache/market-data
           ARGS=(
             --as-of "${AS_OF}"

src/quant_advisor_research/archive_backfill.py

@@ -7,7 +7,7 @@
 from typing import Any
 
 from .build_pipeline import DEFAULT_FEED_TITLE, DEFAULT_SITE_URL, copy_if_different
-from .publisher import publish_reports
+from .publisher import publish_reports, unique_report_paths_by_content
 
 
 REPORT_JSON_PATTERN = re.compile(r"advisory_report_\d{4}-\d{2}-\d{2}\.json")
@@ -58,6 +58,7 @@ def backfill_site_archive(
 ) -> list[Path]:
     if not report_paths:
         raise ValueError("No advisory_report_YYYY-MM-DD.json files found for backfill.")
+    report_paths = unique_report_paths_by_content(report_paths)
     output = Path(output_dir)
     output.mkdir(parents=True, exist_ok=True)
     written = publish_reports(report_paths, output, site_url=site_url, feed_title=feed_title)

src/quant_advisor_research/build_pipeline.py

@@ -8,7 +8,6 @@
 import shutil
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any
 from urllib.parse import quote
 from urllib.request import urlopen
 
@@ -22,7 +21,7 @@
     write_market_confirmation_csv,
 )
 from .monthly_review import build_monthly_review, render_monthly_review_markdown
-from .publisher import publish_reports
+from .publisher import publish_reports, unique_report_paths_by_content
 from .recommendation_review import build_recommendation_review, render_recommendation_review_markdown
 
 
@@ -51,6 +50,12 @@ def parse_date(value: str) -> dt.date:
     return dt.date.fromisoformat(value.strip())
 
 
+def default_weekly_as_of(today: dt.date | None = None) -> dt.date:
+    current = today or dt.datetime.now(dt.UTC).date()
+    days_since_saturday = (current.weekday() - 5) % 7
+    return current - dt.timedelta(days=days_since_saturday)
+
+
 def existing_optional_path(value: str | Path | None) -> Path | None:
     if value in {None, ""}:
         return None
@@ -280,6 +285,7 @@ def build_advisory_artifacts(
         report_paths = [report_json, *recovered_report_paths]
     else:
         report_paths = [report_json]
+    report_paths = unique_report_paths_by_content(report_paths)
 
     recommendation_review_json: Path | None = None
     recommendation_review_md: Path | None = None

src/quant_advisor_research/publisher.py

@@ -59,6 +59,46 @@ def load_report(path: str | Path) -> dict[str, Any]:
         return json.load(handle)
 
 
+def report_content_fingerprint(report: dict[str, Any]) -> str:
+    ignored_top_level_keys = {"as_of", "generated_at", "source_artifacts"}
+    normalized = {key: value for key, value in report.items() if key not in ignored_top_level_keys}
+    return json.dumps(normalized, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+
+
+def report_as_of_date(report: dict[str, Any]) -> dt.date | None:
+    try:
+        return dt.date.fromisoformat(str(report.get("as_of", "")))
+    except ValueError:
+        return None
+
+
+def is_same_period_duplicate(report: dict[str, Any], previous_report: dict[str, Any]) -> bool:
+    if str(report.get("cadence", "")) != str(previous_report.get("cadence", "")):
+        return False
+    as_of = report_as_of_date(report)
+    previous_as_of = report_as_of_date(previous_report)
+    if as_of is None or previous_as_of is None:
+        return True
+    if str(report.get("cadence", "")) == "weekly":
+        return abs((as_of - previous_as_of).days) < 7
+    return as_of == previous_as_of
+
+
+def unique_report_paths_by_content(report_paths: list[str | Path]) -> list[Path]:
+    unique_paths: list[Path] = []
+    seen_reports_by_fingerprint: dict[str, list[dict[str, Any]]] = {}
+    for report_path in report_paths:
+        path = Path(report_path)
+        report = load_report(path)
+        fingerprint = report_content_fingerprint(report)
+        previous_reports = seen_reports_by_fingerprint.setdefault(fingerprint, [])
+        if any(is_same_period_duplicate(report, previous) for previous in previous_reports):
+            continue
+        previous_reports.append(report)
+        unique_paths.append(path)
+    return unique_paths
+
+
 def format_datetime(value: str) -> str:
     normalized = value[:-1] + "+00:00" if value.endswith("Z") else value
     parsed = dt.datetime.fromisoformat(normalized)
@@ -1191,6 +1231,7 @@ def render_feed_xml(reports: list[dict[str, Any]], *, site_url: str, feed_title:
 def publish_reports(report_paths: list[str | Path], output_dir: str | Path, *, site_url: str, feed_title: str) -> list[Path]:
     output = Path(output_dir)
     output.mkdir(parents=True, exist_ok=True)
+    report_paths = unique_report_paths_by_content(report_paths)
     reports = [load_report(path) for path in report_paths]
     written: list[Path] = []
     for report in reports:

tests/test_build_pipeline.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 
 from quant_advisor_research.archive_backfill import backfill_site_archive, discover_report_paths
-from quant_advisor_research.build_pipeline import build_advisory_artifacts
+from quant_advisor_research.build_pipeline import build_advisory_artifacts, default_weekly_as_of
 from quant_advisor_research.cross_repo_smoke import run_cross_repo_smoke
 
 
@@ -40,6 +40,12 @@ def build_fixture_report(tmp_path: Path, as_of: dt.date) -> Path:
     return result.report_json
 
 
+def test_default_weekly_as_of_uses_most_recent_saturday() -> None:
+    assert default_weekly_as_of(dt.date(2026, 6, 20)) == dt.date(2026, 6, 20)
+    assert default_weekly_as_of(dt.date(2026, 6, 21)) == dt.date(2026, 6, 20)
+    assert default_weekly_as_of(dt.date(2026, 6, 24)) == dt.date(2026, 6, 20)
+
+
 def test_build_advisory_artifacts_builds_market_report_and_site(tmp_path: Path) -> None:
     result = build_advisory_artifacts(
         as_of=dt.date(2026, 5, 31),

tests/test_publisher.py

@@ -11,6 +11,7 @@
     render_index_html,
     render_report_html,
     render_reports_index_json,
+    unique_report_paths_by_content,
 )
 
 
@@ -134,6 +135,24 @@ def make_report_for_date(base_report: dict, as_of: str) -> dict:
     return report
 
 
+def test_unique_report_paths_by_content_keeps_first_duplicate_report(tmp_path: Path) -> None:
+    base = build_sample_report()
+    current = make_report_for_date(base, "2026-06-20")
+    duplicate = make_report_for_date(base, "2026-06-21")
+    older = make_report_for_date(base, "2026-06-13")
+
+    current_path = tmp_path / "advisory_report_2026-06-20.json"
+    duplicate_path = tmp_path / "advisory_report_2026-06-21.json"
+    older_path = tmp_path / "advisory_report_2026-06-13.json"
+    write_json(current_path, current)
+    write_json(duplicate_path, duplicate)
+    write_json(older_path, older)
+
+    reports = unique_report_paths_by_content([current_path, duplicate_path, older_path])
+
+    assert reports == [current_path, older_path]
+
+
 def test_index_limits_recent_history_and_archive_keeps_all_reports() -> None:
     base = build_sample_report()
     reports = [make_report_for_date(base, f"2026-05-{day:02d}") for day in range(1, 23)]