Skip to content

Commit d828439

Browse files
Pigbibicodex
andcommitted
Add concrete weekly producer artifact
Co-Authored-By: Codex <noreply@openai.com>
1 parent cf0dcaf commit d828439

5 files changed

Lines changed: 641 additions & 1 deletion

File tree

.github/workflows/rss_source_pipeline.yml

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,16 @@ on:
2323
required: false
2424
default: "50"
2525
type: string
26+
period_start:
27+
description: "Manual completed ISO-week Monday date; required for manual runs."
28+
required: false
29+
default: ""
30+
type: string
31+
as_of:
32+
description: "Manual completed ISO-week Sunday date; required for manual runs."
33+
required: false
34+
default: ""
35+
type: string
2636
commit_outputs:
2737
description: "Commit generated live CSV outputs back to data/live."
2838
required: false
@@ -32,7 +42,7 @@ on:
3242
- "false"
3343
- "true"
3444
schedule:
35-
- cron: "15 12 * * 6"
45+
- cron: "15 12 * * 1"
3646

3747
permissions:
3848
contents: write
@@ -75,6 +85,32 @@ jobs:
7585
--watchlist "${WATCHLIST_PATH}" \
7686
--events data/output/rss_source_pipeline/source_events.csv \
7787
--output data/output/rss_source_pipeline/source_tracker.csv
88+
- name: Build completed weekly producer artifact
89+
env:
90+
PERIOD_START: ${{ github.event.inputs.period_start || '' }}
91+
AS_OF: ${{ github.event.inputs.as_of || '' }}
92+
WATCHLIST_PATH: ${{ github.event.inputs.watchlist_path || 'data/live/political_watchlist.csv' }}
93+
run: |
94+
set -euo pipefail
95+
test "${GITHUB_RUN_ATTEMPT}" = "1"
96+
mkdir -p data/output/political-event-weekly-v1
97+
ARGS=(
98+
--generated-at "$(date -u +%Y-%m-%dT%H:%M:%S.000000Z)"
99+
--workflow-ref "${GITHUB_WORKFLOW_REF}"
100+
--source-run-id "${GITHUB_RUN_ID}"
101+
--producer-ref "${GITHUB_SHA}"
102+
--source-events data/output/rss_source_pipeline/source_events.csv
103+
--watchlist "${WATCHLIST_PATH}"
104+
--feed-status data/output/rss_source_pipeline/source_fetch_status.json
105+
--output-dir data/output/political-event-weekly-v1
106+
)
107+
if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then
108+
ARGS+=(--scheduled-today "$(date -u +%F)" --run-mode scheduled)
109+
else
110+
test -n "${PERIOD_START}" && test -n "${AS_OF}"
111+
ARGS+=(--period-start "${PERIOD_START}" --as-of "${AS_OF}" --run-mode manual)
112+
fi
113+
python scripts/build_weekly_artifact.py "${ARGS[@]}"
78114
- name: Publish live CSV outputs to repository
79115
env:
80116
COMMIT_OUTPUTS: ${{ github.event_name == 'schedule' && 'true' || github.event.inputs.commit_outputs || 'false' }}
@@ -113,3 +149,10 @@ jobs:
113149
name: rss-source-pipeline
114150
path: data/output/rss_source_pipeline/
115151
if-no-files-found: error
152+
- name: Upload political weekly artifact
153+
uses: actions/upload-artifact@v7
154+
with:
155+
name: political-event-weekly-v1
156+
path: data/output/political-event-weekly-v1/
157+
if-no-files-found: error
158+
retention-days: 30
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Political Event Weekly Producer Artifact
2+
3+
`political-event-weekly-v1` is the producer-owned, complete-UTC-ISO-week artifact
4+
emitted by the RSS source pipeline.
5+
6+
## Fixed contract
7+
8+
- Scheduled execution is Monday `12:15 UTC` and covers the immediately preceding
9+
complete Monday–Sunday UTC ISO week.
10+
- Manual execution must provide matching `period_start` (Monday) and `as_of`
11+
(Sunday). No latest-source or wall-clock fallback is allowed for period identity.
12+
- `generated_at` is the actual UTC build time and is separate from period identity.
13+
- `GITHUB_RUN_ATTEMPT` must be exactly `1`; a rerun is fail-closed rather than a
14+
new version of the same snapshot.
15+
- Artifact retention is exactly 30 days.
16+
17+
The dedicated artifact contains exactly:
18+
19+
```text
20+
period_lock.json
21+
political_events.csv
22+
political_watchlist.csv
23+
political_event_weekly.json
24+
weekly_manifest.json
25+
```
26+
27+
The manifest binds the first four files by name, byte length and SHA-256. It also
28+
records CSV headers/row counts, period/as-of/generated-at, producer SHA,
29+
workflow ref, run id/attempt, source snapshot digest/provenance, and complete
30+
feed counters. Any partial/failed/stale/missing feed, source mismatch, unsafe
31+
wire, or readback mismatch prevents dedicated artifact upload.
32+
33+
This contract is producer-side only. QAR consumer acquisition/readback is a
34+
separate later slice; no legacy compatibility, identity store, Pages, publisher,
35+
or workflow permission expansion is implied.

scripts/build_weekly_artifact.py

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
#!/usr/bin/env python3
2+
"""Build the dedicated producer-owned political-event weekly artifact."""
3+
4+
from __future__ import annotations
5+
6+
import argparse
7+
import json
8+
import sys
9+
from datetime import date, datetime, timezone
10+
from pathlib import Path
11+
12+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
13+
14+
from political_event_tracking_research.weekly_artifact import ( # noqa: E402
15+
WeeklyArtifactError,
16+
build_weekly_artifact,
17+
completed_week_period,
18+
)
19+
20+
21+
def _date(value: str) -> date:
22+
try:
23+
return date.fromisoformat(value)
24+
except ValueError:
25+
raise argparse.ArgumentTypeError("invalid ISO date") from None
26+
27+
28+
def _generated_at(value: str) -> datetime:
29+
try:
30+
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
31+
except ValueError:
32+
raise argparse.ArgumentTypeError("invalid UTC timestamp") from None
33+
if parsed.tzinfo != timezone.utc:
34+
raise argparse.ArgumentTypeError("generated_at must be UTC")
35+
return parsed
36+
37+
38+
def main() -> None:
39+
parser = argparse.ArgumentParser(description=__doc__)
40+
parser.add_argument("--period-start", type=_date)
41+
parser.add_argument("--as-of", type=_date)
42+
parser.add_argument("--scheduled-today", type=_date)
43+
parser.add_argument("--generated-at", required=True, type=_generated_at)
44+
parser.add_argument("--workflow-ref", required=True)
45+
parser.add_argument("--source-run-id", required=True)
46+
parser.add_argument("--producer-ref", required=True)
47+
parser.add_argument("--source-events", required=True, type=Path)
48+
parser.add_argument("--watchlist", required=True, type=Path)
49+
parser.add_argument("--feed-status", required=True, type=Path)
50+
parser.add_argument("--output-dir", required=True, type=Path)
51+
parser.add_argument("--run-mode", choices=("scheduled", "manual"), required=True)
52+
args = parser.parse_args()
53+
54+
if args.scheduled_today is not None:
55+
if args.run_mode != "scheduled" or args.period_start is not None or args.as_of is not None:
56+
parser.error("scheduled mode requires only --scheduled-today")
57+
period_start, period_end = completed_week_period(args.scheduled_today)
58+
as_of = period_end - date.resolution
59+
elif args.run_mode == "manual" and args.period_start is not None and args.as_of is not None:
60+
period_start, as_of = args.period_start, args.as_of
61+
else:
62+
parser.error("manual mode requires --period-start and --as-of")
63+
64+
try:
65+
feed_status = json.loads(args.feed_status.read_text(encoding="utf-8"))
66+
files = build_weekly_artifact(
67+
period_start=period_start,
68+
as_of=as_of,
69+
generated_at=args.generated_at,
70+
workflow_ref=args.workflow_ref,
71+
source_run_id=args.source_run_id,
72+
producer_ref=args.producer_ref,
73+
source_events=args.source_events.read_bytes(),
74+
watchlist=args.watchlist.read_bytes(),
75+
feed_status=feed_status,
76+
source_provenance="official_rss_source_pipeline_v1",
77+
run_mode=args.run_mode,
78+
)
79+
args.output_dir.mkdir(parents=True, exist_ok=True)
80+
if any(args.output_dir.iterdir()):
81+
raise WeeklyArtifactError("artifact_output_not_empty")
82+
for name, content in files.items():
83+
(args.output_dir / name).write_bytes(content)
84+
except WeeklyArtifactError as exc:
85+
raise SystemExit(exc.code) from None
86+
except (OSError, UnicodeError, json.JSONDecodeError, TypeError, ValueError):
87+
raise SystemExit("weekly_artifact_input_invalid") from None
88+
89+
90+
if __name__ == "__main__":
91+
main()

0 commit comments

Comments
 (0)