From 816e846640e579e04ba23b6357ccb983b4fb5a7a Mon Sep 17 00:00:00 2001 From: "Saheed Alabi (Standard User)" Date: Thu, 11 Jun 2026 15:14:48 +0100 Subject: [PATCH] Raise missing associated entity issue for plan-timetable --- digital_land/phase/lookup.py | 27 ++++++++++++- digital_land/pipeline/main.py | 1 + digital_land/specification.py | 10 +++++ tests/unit/phase/test_lookup.py | 66 ++++++++++++++++++++++++++++++++ tests/unit/test_specification.py | 12 ++++++ 5 files changed, 114 insertions(+), 2 deletions(-) diff --git a/digital_land/phase/lookup.py b/digital_land/phase/lookup.py index 454b5f68a..758245c1d 100644 --- a/digital_land/phase/lookup.py +++ b/digital_land/phase/lookup.py @@ -172,11 +172,17 @@ def process(self, stream): class FactLookupPhase(LookupPhase): def __init__( - self, lookups={}, redirect_lookups={}, issue_log=None, odp_collections=[] + self, + lookups={}, + redirect_lookups={}, + issue_log=None, + odp_collections=None, + package_prefixes=None, ): super().__init__(lookups, redirect_lookups, issue_log) self.entity_field = "reference-entity" self.odp_collections = odp_collections + self.package_prefixes = package_prefixes or {} def process(self, stream): for block in stream: @@ -211,11 +217,28 @@ def process(self, stream): # as it is organisation specific. find_entity = self.check_associated_organisation(find_entity) + for package_prefix in self.package_prefixes.get(prefix, []): + if find_entity: + break + + find_entity = self.lookup( + prefix=package_prefix, + organisation=organisation, + reference=reference, + ) + if not find_entity: + find_entity = self.lookup( + prefix=package_prefix, reference=reference + ) + find_entity = self.check_associated_organisation(find_entity) + if not find_entity or ( str(find_entity) in self.redirect_lookups and int(self.redirect_lookups[str(find_entity)].get("status", 0)) == 410 ): - if self.odp_collections and prefix in self.odp_collections: + if (self.odp_collections and prefix in self.odp_collections) or ( + prefix in self.package_prefixes + ): self.issues.log_issue( prefix, "missing associated entity", diff --git a/digital_land/pipeline/main.py b/digital_land/pipeline/main.py index a586e5879..d320b9b2a 100644 --- a/digital_land/pipeline/main.py +++ b/digital_land/pipeline/main.py @@ -690,6 +690,7 @@ def transform( redirect_lookups=redirect_lookups, issue_log=self.issue_log, odp_collections=self.specification.get_odp_collections(), + package_prefixes=self.specification.get_package_prefixes(), ), FactPrunePhase(), SavePhase( diff --git a/digital_land/specification.py b/digital_land/specification.py index fdffa09e5..2f2ed8079 100644 --- a/digital_land/specification.py +++ b/digital_land/specification.py @@ -319,6 +319,16 @@ def get_field_typology_map(self): def get_field_prefix_map(self): return {key: self.field_prefix(key) for key in self.field.keys()} + def get_package_prefixes(self): + return { + "plan": [ + "local-plan", + "minerals-plan", + "waste-plan", + "supplementary-plan", + ], + } + def get_dataset_typology(self, dataset): if dataset: return self.dataset[dataset]["typology"] diff --git a/tests/unit/phase/test_lookup.py b/tests/unit/phase/test_lookup.py index 55f3153b1..bb38a0027 100644 --- a/tests/unit/phase/test_lookup.py +++ b/tests/unit/phase/test_lookup.py @@ -314,3 +314,69 @@ def test_missing_associated_entity_issue_no_organisation( assert output[0]["row"]["reference-entity"] == "1" assert len(issues.rows) == 0 + + def test_package_prefix_lookup_assigns_reference_entity(self): + input_stream = [ + { + "organisation": "local-authorityabc", + "row": { + "fact": "abc", + "entity": "10", + "field": "plan", + "value": "local-plan-1", + "prefix": "plan", + "reference": "local-plan-1", + "entry-number": 1, + "line-number": 2, + }, + } + ] + lookups = { + ",local-plan,local-plan-1,local-authorityabc": "1", + ",plan-timetable,1,local-authorityabc": "10", + } + issues = IssueLog() + + phase = FactLookupPhase( + lookups=lookups, + issue_log=issues, + package_prefixes={"plan": ["local-plan"]}, + ) + output = [block for block in phase.process(input_stream)] + + assert output[0]["row"]["reference-entity"] == "1" + assert len(issues.rows) == 0 + + def test_package_prefix_lookup_raises_missing_associated_entity(self): + input_stream = [ + { + "organisation": "local-authorityabc", + "row": { + "fact": "abc", + "entity": "10", + "field": "plan", + "value": "missing-plan", + "prefix": "plan", + "reference": "missing-plan", + "entry-number": 1, + "line-number": 2, + }, + } + ] + lookups = { + ",local-plan,other-plan,local-authorityabc": "1", + ",plan-timetable,1,local-authorityabc": "10", + } + issues = IssueLog() + + phase = FactLookupPhase( + lookups=lookups, + issue_log=issues, + package_prefixes={"plan": ["local-plan"]}, + ) + output = [block for block in phase.process(input_stream)] + + assert "reference-entity" not in output[0]["row"] + assert issues.rows[0]["field"] == "plan" + assert issues.rows[0]["issue-type"] == "missing associated entity" + assert issues.rows[0]["value"] == "missing-plan" diff --git a/tests/unit/test_specification.py b/tests/unit/test_specification.py index 011f2a940..ce62f495e 100644 --- a/tests/unit/test_specification.py +++ b/tests/unit/test_specification.py @@ -227,3 +227,15 @@ def test_get_field_prefix_map_returns_correct_values(input, expected): spec = Specification() spec.field = input assert expected == spec.get_field_prefix_map() + + +def test_get_package_prefixes_returns_plan_package_prefixes(): + spec = Specification() + assert spec.get_package_prefixes() == { + "plan": [ + "local-plan", + "minerals-plan", + "waste-plan", + "supplementary-plan", + ], + }