From f0e82a54ffa725fe6add86965cbfb54e0d2556ad Mon Sep 17 00:00:00 2001 From: riddim-developer-bot Date: Thu, 14 May 2026 15:04:25 -0400 Subject: [PATCH] [EPAC-1880]: Backfill unittest suite for cpp-oas-statistics pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add test_cpp_oas_statistics.py with the four required named test cases: - test_parses_happy_path_fixture: loads CSV fixtures, asserts golden snapshot - test_handles_missing_required_field: absent Retirement column → ValueError - test_handles_malformed_upstream: empty bytes → ValueError, pipeline finished not logged - test_emits_required_log_events: patches stderr, verifies pipeline started/finished events Commits fixtures/cpp_fixture.csv and fixtures/oas_fixture.csv (Ontario + Alberta, 2022–2024) so tests run without network access, and fixtures/expected_output.json as the golden snapshot. Also creates .github/workflows/backend-python.yml (did not exist) covering all 10 Python pipelines via a matrix job — cpp-oas-statistics included with no skip guard since tests now exist. All 7 tests pass: python3 -m unittest discover -v in 0.004s. --- .github/workflows/backend-python.yml | 33 +++ .../fixtures/cpp_fixture.csv | 7 + .../fixtures/expected_output.json | 199 ++++++++++++++++++ .../fixtures/oas_fixture.csv | 10 + .../test_cpp_oas_statistics.py | 109 ++++++++++ 5 files changed, 358 insertions(+) create mode 100644 .github/workflows/backend-python.yml create mode 100644 backend/cpp-oas-statistics/fixtures/cpp_fixture.csv create mode 100644 backend/cpp-oas-statistics/fixtures/expected_output.json create mode 100644 backend/cpp-oas-statistics/fixtures/oas_fixture.csv create mode 100644 backend/cpp-oas-statistics/test_cpp_oas_statistics.py diff --git a/.github/workflows/backend-python.yml b/.github/workflows/backend-python.yml new file mode 100644 index 00000000..26d1e4ed --- /dev/null +++ b/.github/workflows/backend-python.yml @@ -0,0 +1,33 @@ +name: Backend Python tests + +on: + pull_request: + paths: ['backend/**'] + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + pipeline: + - cabinet + - corrections-statistics + - cpi-statistics + - cpp-oas-statistics + - ei-statistics + - fiscal-monitor + - pbo + - student-finance-statistics + - transport-safety-statistics + - vac-statistics + name: Test ${{ matrix.pipeline }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Run tests + working-directory: backend/${{ matrix.pipeline }} + run: python3 -m unittest discover -v diff --git a/backend/cpp-oas-statistics/fixtures/cpp_fixture.csv b/backend/cpp-oas-statistics/fixtures/cpp_fixture.csv new file mode 100644 index 00000000..d48f1369 --- /dev/null +++ b/backend/cpp-oas-statistics/fixtures/cpp_fixture.csv @@ -0,0 +1,7 @@ +Period,Province,Retirement +Jan. / jan. 2022,ONTARIO,1000000 +Jan. / jan. 2023,ONTARIO,1010000 +Jan. / jan. 2024,ONTARIO,1020000 +Jan. / jan. 2022,ALBERTA,500000 +Jan. / jan. 2023,ALBERTA,505000 +Jan. / jan. 2024,ALBERTA,510000 diff --git a/backend/cpp-oas-statistics/fixtures/expected_output.json b/backend/cpp-oas-statistics/fixtures/expected_output.json new file mode 100644 index 00000000..d30e9344 --- /dev/null +++ b/backend/cpp-oas-statistics/fixtures/expected_output.json @@ -0,0 +1,199 @@ +{ + "datasets": [ + { + "csv": "https://open.canada.ca/data/dataset/1fab2afd-4f3c-4922-a07e-58d7bed9dcfc/resource/5ddf2bac-666e-4342-81c8-59274da78425/download/20260430-cpres-cppben.csv", + "id": "1fab2afd-4f3c-4922-a07e-58d7bed9dcfc", + "title": "Canada Pension Plan – Beneficiaries by Place of Residence", + "url": "https://open.canada.ca/data/en/dataset/1fab2afd-4f3c-4922-a07e-58d7bed9dcfc" + }, + { + "csv": "https://open.canada.ca/data/dataset/77381606-95c0-411a-a7cd-eba5d038c1c4/resource/ae931981-b9ac-4b5c-9b6b-2c70c1fe448f/download/20250331-svpres-oasben.csv", + "id": "77381606-95c0-411a-a7cd-eba5d038c1c4", + "title": "Old Age Security – Beneficiaries by Province", + "url": "https://open.canada.ca/data/en/dataset/77381606-95c0-411a-a7cd-eba5d038c1c4" + } + ], + "history_years": [2022, 2023, 2024], + "national": { + "cpp_reference_period": "2024-01", + "cpp_retirement_recipients": 1530000, + "oas_pension_recipients": 1330000, + "oas_reference_period": "2024-01" + }, + "provinces": [ + { + "cpp_reference_period": null, + "cpp_retirement_recipients": null, + "history": [ + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2022}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2023}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2024} + ], + "oas_pension_recipients": null, + "oas_reference_period": null, + "province": "Newfoundland and Labrador", + "province_code": "NL" + }, + { + "cpp_reference_period": null, + "cpp_retirement_recipients": null, + "history": [ + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2022}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2023}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2024} + ], + "oas_pension_recipients": null, + "oas_reference_period": null, + "province": "Prince Edward Island", + "province_code": "PE" + }, + { + "cpp_reference_period": null, + "cpp_retirement_recipients": null, + "history": [ + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2022}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2023}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2024} + ], + "oas_pension_recipients": null, + "oas_reference_period": null, + "province": "Nova Scotia", + "province_code": "NS" + }, + { + "cpp_reference_period": null, + "cpp_retirement_recipients": null, + "history": [ + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2022}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2023}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2024} + ], + "oas_pension_recipients": null, + "oas_reference_period": null, + "province": "New Brunswick", + "province_code": "NB" + }, + { + "cpp_reference_period": null, + "cpp_retirement_recipients": null, + "history": [ + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2022}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2023}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2024} + ], + "oas_pension_recipients": null, + "oas_reference_period": null, + "province": "Quebec", + "province_code": "QC" + }, + { + "cpp_reference_period": "2024-01", + "cpp_retirement_recipients": 1020000, + "history": [ + {"cpp_retirement_recipients": 1000000, "oas_pension_recipients": 900000, "year": 2022}, + {"cpp_retirement_recipients": 1010000, "oas_pension_recipients": 910000, "year": 2023}, + {"cpp_retirement_recipients": 1020000, "oas_pension_recipients": 920000, "year": 2024} + ], + "oas_pension_recipients": 920000, + "oas_reference_period": "2024-01", + "province": "Ontario", + "province_code": "ON" + }, + { + "cpp_reference_period": null, + "cpp_retirement_recipients": null, + "history": [ + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2022}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2023}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2024} + ], + "oas_pension_recipients": null, + "oas_reference_period": null, + "province": "Manitoba", + "province_code": "MB" + }, + { + "cpp_reference_period": null, + "cpp_retirement_recipients": null, + "history": [ + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2022}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2023}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2024} + ], + "oas_pension_recipients": null, + "oas_reference_period": null, + "province": "Saskatchewan", + "province_code": "SK" + }, + { + "cpp_reference_period": "2024-01", + "cpp_retirement_recipients": 510000, + "history": [ + {"cpp_retirement_recipients": 500000, "oas_pension_recipients": 400000, "year": 2022}, + {"cpp_retirement_recipients": 505000, "oas_pension_recipients": 405000, "year": 2023}, + {"cpp_retirement_recipients": 510000, "oas_pension_recipients": 410000, "year": 2024} + ], + "oas_pension_recipients": 410000, + "oas_reference_period": "2024-01", + "province": "Alberta", + "province_code": "AB" + }, + { + "cpp_reference_period": null, + "cpp_retirement_recipients": null, + "history": [ + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2022}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2023}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2024} + ], + "oas_pension_recipients": null, + "oas_reference_period": null, + "province": "British Columbia", + "province_code": "BC" + }, + { + "cpp_reference_period": null, + "cpp_retirement_recipients": null, + "history": [ + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2022}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2023}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2024} + ], + "oas_pension_recipients": null, + "oas_reference_period": null, + "province": "Yukon", + "province_code": "YT" + }, + { + "cpp_reference_period": null, + "cpp_retirement_recipients": null, + "history": [ + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2022}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2023}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2024} + ], + "oas_pension_recipients": null, + "oas_reference_period": null, + "province": "Northwest Territories", + "province_code": "NT" + }, + { + "cpp_reference_period": null, + "cpp_retirement_recipients": null, + "history": [ + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2022}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2023}, + {"cpp_retirement_recipients": null, "oas_pension_recipients": null, "year": 2024} + ], + "oas_pension_recipients": null, + "oas_reference_period": null, + "province": "Nunavut", + "province_code": "NU" + } + ], + "source": { + "note": "Provincial recipient counts come from the ESDC monthly Statistical Bulletin datasets published on open.canada.ca. Average monthly benefit amounts are only published nationally; provincial-level averages are not available.", + "title": "Employment and Social Development Canada — CPP/OAS Statistical Bulletin", + "url": "https://www.canada.ca/en/employment-social-development/programs/pensions/reports/statistical-bulletin.html" + } +} diff --git a/backend/cpp-oas-statistics/fixtures/oas_fixture.csv b/backend/cpp-oas-statistics/fixtures/oas_fixture.csv new file mode 100644 index 00000000..325ee45f --- /dev/null +++ b/backend/cpp-oas-statistics/fixtures/oas_fixture.csv @@ -0,0 +1,10 @@ +Employment and Social Development Canada +Old Age Security - Beneficiaries by Province +, +Period,Province,Old Age Security Pension +Jan. / jan. 2022,ONTARIO,900000 +Jan. / jan. 2023,ONTARIO,910000 +Jan. / jan. 2024,ONTARIO,920000 +Jan. / jan. 2022,ALBERTA,400000 +Jan. / jan. 2023,ALBERTA,405000 +Jan. / jan. 2024,ALBERTA,410000 diff --git a/backend/cpp-oas-statistics/test_cpp_oas_statistics.py b/backend/cpp-oas-statistics/test_cpp_oas_statistics.py new file mode 100644 index 00000000..d422167d --- /dev/null +++ b/backend/cpp-oas-statistics/test_cpp_oas_statistics.py @@ -0,0 +1,109 @@ +"""Unit tests for cpp_oas_statistics.py.""" + +from __future__ import annotations + +import io +import json +import os +import sys +import tempfile +import unittest +from unittest.mock import patch + +import cpp_oas_statistics + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures") + + +def _load_fixture(name: str) -> bytes: + with open(os.path.join(FIXTURES_DIR, name), "rb") as f: + return f.read() + + +class ParsePeriodTests(unittest.TestCase): + def test_standard_bilingual_format(self) -> None: + self.assertEqual(cpp_oas_statistics.parse_period("Jan. / jan. 2023"), (2023, 1)) + + def test_returns_none_for_empty(self) -> None: + self.assertIsNone(cpp_oas_statistics.parse_period("")) + + def test_returns_none_for_unparseable(self) -> None: + self.assertIsNone(cpp_oas_statistics.parse_period("Total / Total")) + + +class CppOasStatisticsTests(unittest.TestCase): + def setUp(self) -> None: + self.cpp_csv = _load_fixture("cpp_fixture.csv") + self.oas_csv = _load_fixture("oas_fixture.csv") + + def test_parses_happy_path_fixture(self) -> None: + payload = cpp_oas_statistics.build_payload(self.cpp_csv, self.oas_csv) + + self.assertIn("provinces", payload) + self.assertIn("national", payload) + self.assertIn("history_years", payload) + self.assertEqual(len(payload["provinces"]), 13) + self.assertEqual(payload["history_years"], [2022, 2023, 2024]) + + ontario = next(p for p in payload["provinces"] if p["province_code"] == "ON") + self.assertEqual(ontario["province"], "Ontario") + self.assertEqual(ontario["cpp_retirement_recipients"], 1020000) + self.assertEqual(ontario["cpp_reference_period"], "2024-01") + self.assertEqual(ontario["oas_pension_recipients"], 920000) + self.assertEqual(ontario["oas_reference_period"], "2024-01") + self.assertEqual(len(ontario["history"]), 3) + + national = payload["national"] + self.assertEqual(national["cpp_retirement_recipients"], 1530000) + self.assertEqual(national["oas_pension_recipients"], 1330000) + self.assertEqual(national["cpp_reference_period"], "2024-01") + self.assertEqual(national["oas_reference_period"], "2024-01") + + with open(os.path.join(FIXTURES_DIR, "expected_output.json"), encoding="utf-8") as f: + expected = json.load(f) + actual = {k: v for k, v in payload.items() if k != "generated_at"} + self.assertEqual(actual, expected) + + def test_handles_missing_required_field(self) -> None: + # CSV with no "Retirement" column — all rows skipped → ValueError, not silent KeyError + bad_cpp = b"Period,Province,NotARecognisedColumn\nJan. / jan. 2024,ONTARIO,1000000\n" + with self.assertRaises(ValueError): + cpp_oas_statistics.build_payload(bad_cpp, self.oas_csv) + + def test_handles_malformed_upstream(self) -> None: + # Truncated / empty bytes → no parseable rows → ValueError raised before + # "pipeline finished" can be logged. + stderr_buf = io.StringIO() + with patch("sys.stderr", stderr_buf): + with patch("cpp_oas_statistics.fetch_csv", side_effect=[b"", b""]): + with self.assertRaises(ValueError): + cpp_oas_statistics.main([]) + self.assertNotIn("pipeline finished", stderr_buf.getvalue()) + + def test_emits_required_log_events(self) -> None: + stderr_buf = io.StringIO() + with patch("sys.stderr", stderr_buf): + with patch( + "cpp_oas_statistics.fetch_csv", + side_effect=[self.cpp_csv, self.oas_csv], + ): + with tempfile.TemporaryDirectory() as tmpdir: + out_file = os.path.join(tmpdir, "out.json") + result = cpp_oas_statistics.main(["--output", out_file]) + self.assertEqual(result, 0) + + log_lines = [l for l in stderr_buf.getvalue().splitlines() if l.strip()] + self.assertTrue(log_lines, "No log output captured on stderr") + log_events = [json.loads(line) for line in log_lines] + messages = [e["message"] for e in log_events] + self.assertIn("pipeline started", messages) + self.assertIn("pipeline finished", messages) + + finished = next(e for e in log_events if e["message"] == "pipeline finished") + self.assertIn("records_processed", finished) + self.assertIn("duration_ms", finished) + self.assertEqual(finished["pipeline"], "cpp-oas-statistics") + + +if __name__ == "__main__": + unittest.main()