diff --git a/every_election/apps/api/tests/test_api_election_endpoint.py b/every_election/apps/api/tests/test_api_election_endpoint.py index 5fa65cab0..10eaec62e 100644 --- a/every_election/apps/api/tests/test_api_election_endpoint.py +++ b/every_election/apps/api/tests/test_api_election_endpoint.py @@ -446,7 +446,9 @@ def test_all_expected_fields_returned(self): "current": true, "poll_open_date": "2017-03-23", "timetable": { + "notice_of_election_deadline": "2017-02-16", "close_of_nominations": "2017-02-24", + "sopn_publish_deadline": "2017-02-27", "registration_deadline": "2017-03-07", "postal_vote_application_deadline": "2017-03-08", "vac_application_deadline": null diff --git a/every_election/apps/elections/migrations/0089_more_timetable_fields.py b/every_election/apps/elections/migrations/0089_more_timetable_fields.py new file mode 100644 index 000000000..6f79fcc07 --- /dev/null +++ b/every_election/apps/elections/migrations/0089_more_timetable_fields.py @@ -0,0 +1,30 @@ +# Generated by Django 5.2.15 on 2026-07-08 09:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("elections", "0088_fix_ni_timetable_fields"), + ] + + operations = [ + migrations.AddField( + model_name="election", + name="notice_of_election_deadline", + field=models.DateField( + blank=True, + help_text="Deadline to publish Notice of Election document", + null=True, + ), + ), + migrations.AddField( + model_name="election", + name="sopn_publish_deadline", + field=models.DateField( + blank=True, + help_text="Deadline to publish SOPN document", + null=True, + ), + ), + ] diff --git a/every_election/apps/elections/migrations/0090_backfill_new_timetable_fields.py b/every_election/apps/elections/migrations/0090_backfill_new_timetable_fields.py new file mode 100644 index 000000000..f6e8590da --- /dev/null +++ b/every_election/apps/elections/migrations/0090_backfill_new_timetable_fields.py @@ -0,0 +1,105 @@ +# Generated by Django 5.2.13 on 2026-06-16 10:30 + +from django.db import migrations +from django.db.models import F +from uk_election_timetables.calendars import Country +from uk_election_timetables.election_ids import from_election_id + +country_map = { + "WLS": Country.WALES, + "ENG": Country.ENGLAND, + "NIR": Country.NORTHERN_IRELAND, + "SCT": Country.SCOTLAND, + "GBN": None, +} + + +def backfill_new_timetable_fields(apps, schema_editor): + Election = apps.get_model("elections", "Election") + qs = Election.private_objects.using(schema_editor.connection.alias) + + elections_to_update = [] + city_of_london_to_update = [] + + for election in qs.iterator(): + # we can't calculate a timetable for en election with + # no polling day + if election.poll_open_date is None: + continue + + # only calculate a timetable for ballots + if election.group_type is not None: + continue + + # We don't have logic for computing timetable for + # EU Parliament elections but some do exist in the DB + # from back in the day + if election.election_id.startswith("europarl."): + continue + + # There is no nominations or SOPN for refenda + # Notice of election deadline is applicable + # but not implemented + if election.election_id.startswith("ref."): + continue + + area = election.division or election.organisation + territory_code = ( + area.territory_code or election.organisation.territory_code + ) + + timetable = from_election_id( + election.election_id, country=country_map[territory_code] + ) + + # populate the new fields we just added in 0089 + election.notice_of_election_deadline = ( + timetable.notice_of_election_deadline + ) + election.sopn_publish_deadline = timetable.sopn_publish_deadline + elections_to_update.append(election) + + # We used to make a slightly different assumption about + # City of London elections. This changed in 5.0.0 so + # update close_of_nominations for City of London only + if election.election_id.startswith("local.city-of-london."): + election.close_of_nominations = timetable.close_of_nominations + city_of_london_to_update.append(election) + + batch_size = 2000 + for i in range(0, len(elections_to_update), batch_size): + qs.bulk_update( + elections_to_update[i : i + batch_size], + ["notice_of_election_deadline", "sopn_publish_deadline"], + ) + for i in range(0, len(city_of_london_to_update), batch_size): + qs.bulk_update( + city_of_london_to_update[i : i + batch_size], + ["close_of_nominations"], + ) + + +def clear_new_timetable_fields(apps, schema_editor): + Election = apps.get_model("elections", "Election") + qs = Election.private_objects.using(schema_editor.connection.alias) + + qs.filter(election_id__startswith="local.city-of-london.").update( + close_of_nominations=F("sopn_publish_deadline") + ) + + qs.update( + notice_of_election_deadline=None, + sopn_publish_deadline=None, + ) + + +class Migration(migrations.Migration): + dependencies = [ + ("elections", "0089_more_timetable_fields"), + ] + + operations = [ + migrations.RunPython( + backfill_new_timetable_fields, clear_new_timetable_fields + ) + ] diff --git a/every_election/apps/elections/models.py b/every_election/apps/elections/models.py index 30c54eb01..ba5610309 100644 --- a/every_election/apps/elections/models.py +++ b/every_election/apps/elections/models.py @@ -191,9 +191,17 @@ class Election(TimeStampedModel): # timetable poll_open_date = models.DateField(blank=True, null=True) + notice_of_election_deadline = models.DateField( + blank=True, + null=True, + help_text="Deadline to publish Notice of Election document", + ) close_of_nominations = models.DateField( blank=True, null=True, help_text="Close of Nominations" ) + sopn_publish_deadline = models.DateField( + blank=True, null=True, help_text="Deadline to publish SOPN document" + ) registration_deadline = models.DateField( blank=True, null=True, help_text="Register to vote deadline" ) @@ -204,7 +212,9 @@ class Election(TimeStampedModel): blank=True, null=True, help_text="VAC application deadline" ) TIMETABLE_FIELDS = ( + "notice_of_election_deadline", "close_of_nominations", + "sopn_publish_deadline", "registration_deadline", "postal_vote_application_deadline", "vac_application_deadline", @@ -648,9 +658,11 @@ def clean(self): ) expected_timetable_fields = self.get_expected_timetable_fields() + optional_timetable_fields = self.get_optional_timetable_fields() for field in self.TIMETABLE_FIELDS: if ( field in expected_timetable_fields + and field not in optional_timetable_fields and getattr(self, field) is None ): raise ValidationError(f"{field} is required") @@ -664,6 +676,12 @@ def clean(self): ) for field in expected_timetable_fields: + if ( + field in optional_timetable_fields + and getattr(self, field) is None + ): + continue + if getattr(self, field) > self.poll_open_date: raise ValidationError(f"{field} must be before poll_open_date") @@ -672,6 +690,23 @@ def clean(self): f"{field} must be within 50 days of poll_open_date" ) + def get_optional_timetable_fields(self): + # timetable is only applicable to elections with + # polling day + if self.poll_open_date is None: + return [] + + # timetable is only applicable to ballots + if self.group_type is not None: + return [] + + if self.election_id.startswith("ref."): + # notice_of_election_deadline is conceptually applicable to + # referenda but is valid to leave blank + return ["notice_of_election_deadline"] + + return [] + def get_expected_timetable_fields(self): timetable_fields = [] @@ -689,9 +724,12 @@ def get_expected_timetable_fields(self): if self.election_id.startswith("europarl."): return [] - # There is no nominations or SOPN for refenda + timetable_fields.append("notice_of_election_deadline") + + # There is no nominations or SOPN for referenda if not self.election_id.startswith("ref."): timetable_fields.append("close_of_nominations") + timetable_fields.append("sopn_publish_deadline") timetable_fields.append("registration_deadline") timetable_fields.append("postal_vote_application_deadline") @@ -703,8 +741,9 @@ def get_expected_timetable_fields(self): return timetable_fields def set_timetable_fields(self): - fields = self.get_expected_timetable_fields() - if not fields: + expected_timetable_fields = self.get_expected_timetable_fields() + optional_timetable_fields = self.get_optional_timetable_fields() + if not expected_timetable_fields: return country_map = { @@ -721,10 +760,10 @@ def set_timetable_fields(self): self.election_id, country=country_map[territory_code] ) - for field in fields: + for field in expected_timetable_fields: if getattr(self, field) is None: - if field == "close_of_nominations": - setattr(self, field, timetable.sopn_publish_date) + if field in optional_timetable_fields: + setattr(self, field, None) else: setattr(self, field, getattr(timetable, field)) diff --git a/every_election/apps/elections/tests/test_election_model.py b/every_election/apps/elections/tests/test_election_model.py index 2b24d30df..24b544060 100644 --- a/every_election/apps/elections/tests/test_election_model.py +++ b/every_election/apps/elections/tests/test_election_model.py @@ -1,12 +1,16 @@ import contextlib +from datetime import date, timedelta from datetime import timezone as dt_timezone -from unittest.mock import patch +from unittest.mock import PropertyMock, patch +from django.core.exceptions import ValidationError from django.test import TestCase from django.utils import timezone as dj_timezone from elections.models import ( DEFAULT_STATUS, + ByElectionReason, Election, + ElectionCancellationReason, ModerationHistory, ModerationStatuses, ) @@ -374,8 +378,10 @@ def test_requires_id(self): election_group.refresh_from_db() org_group.refresh_from_db() - # All 4 timetable fields should be populated on the ballot + # All timetable fields should be populated on the ballot + self.assertIsNotNone(ballot.notice_of_election_deadline) self.assertIsNotNone(ballot.close_of_nominations) + self.assertIsNotNone(ballot.sopn_publish_deadline) self.assertIsNotNone(ballot.registration_deadline) self.assertIsNotNone(ballot.postal_vote_application_deadline) self.assertIsNotNone(ballot.vac_application_deadline) @@ -427,7 +433,9 @@ def test_no_id_required(self): election_group.refresh_from_db() org_group.refresh_from_db() + self.assertIsNotNone(ballot.notice_of_election_deadline) self.assertIsNotNone(ballot.close_of_nominations) + self.assertIsNotNone(ballot.sopn_publish_deadline) self.assertIsNotNone(ballot.registration_deadline) self.assertIsNotNone(ballot.postal_vote_application_deadline) @@ -438,6 +446,64 @@ def test_no_id_required(self): self.assert_timetable_fields_all_none(election_group) self.assert_timetable_fields_all_none(org_group) + def test_nothern_ireland_ballot(self): + org = OrganisationFactory(territory_code="WLS") + election_type = ElectionTypeFactory(election_type="local") + div_set = OrganisationDivisionSetFactory(organisation=org) + div = OrganisationDivisionFactory(divisionset=div_set) + + election_group = Election( + election_id=f"local.{self.POLL_DATE}", + election_title="Local elections", + election_type=election_type, + poll_open_date=self.POLL_DATE, + group_type="election", + ) + election_group.save() + + org_group = Election( + election_id=f"local.{org.slug}.{self.POLL_DATE}", + election_title=f"Local elections - {org.official_name}", + election_type=election_type, + organisation=org, + poll_open_date=self.POLL_DATE, + group=election_group, + group_type="organisation", + ) + org_group.save() + + ballot = Election( + election_id=f"local.{org.slug}.{div.slug}.{self.POLL_DATE}", + election_title=f"Local elections - {org.official_name} - {div.name}", + election_type=election_type, + organisation=org, + division=div, + poll_open_date=self.POLL_DATE, + group=org_group, + group_type=None, + requires_voter_id="EFA-2002", + ) + ballot.save() + + ballot.refresh_from_db() + election_group.refresh_from_db() + org_group.refresh_from_db() + + self.assertIsNotNone(ballot.notice_of_election_deadline) + self.assertIsNotNone(ballot.close_of_nominations) + self.assertIsNotNone(ballot.sopn_publish_deadline) + self.assertIsNotNone(ballot.registration_deadline) + self.assertIsNotNone(ballot.postal_vote_application_deadline) + + # VAC deadline should be none on the ballot + # although ID is required in NI + # VACs are GB-only + self.assertIsNone(ballot.vac_application_deadline) + + # Timetable fields should all be none on the parent groups + self.assert_timetable_fields_all_none(election_group) + self.assert_timetable_fields_all_none(org_group) + def test_referendum(self): org = OrganisationFactory(territory_code="ENG") election_type = ElectionTypeFactory(election_type="ref") @@ -483,8 +549,11 @@ def test_referendum(self): self.assertIsNotNone(ballot.registration_deadline) self.assertIsNotNone(ballot.postal_vote_application_deadline) - # Close of nominations should be none on the ballot + # notice_of_election_deadline is optional for referenda + self.assertIsNone(ballot.notice_of_election_deadline) + # There is no nominations or SOPN for referenda self.assertIsNone(ballot.close_of_nominations) + self.assertIsNone(ballot.sopn_publish_deadline) # Timetable fields should all be none on the parent groups self.assert_timetable_fields_all_none(election_group) @@ -536,3 +605,201 @@ def test_provisional_ballot(self): self.assert_timetable_fields_all_none(ballot) self.assert_timetable_fields_all_none(election_group) self.assert_timetable_fields_all_none(org_group) + + +class TestCleanMethod(TestCase): + """ + Tests for Election.clean() validation method + """ + + POLL_DATE = date(2024, 5, 2) + + def _make_ballot(self, **kwargs): + defaults = { + "election_id": f"local.test.test-div.{self.POLL_DATE}", + "election_title": "Test ballot", + "election_type": ElectionTypeFactory(election_type="local"), + "poll_open_date": self.POLL_DATE, + "group_type": None, + } + defaults.update(kwargs) + return Election(**defaults) + + def _make_group(self, group_type="election", **kwargs): + defaults = { + "election_id": f"local.{self.POLL_DATE}", + "election_title": "Local elections", + "election_type": ElectionTypeFactory(election_type="local"), + "poll_open_date": self.POLL_DATE, + "group_type": group_type, + } + defaults.update(kwargs) + return Election(**defaults) + + # --- cancellation rules --- + + def test_group_cannot_be_cancelled(self): + group = self._make_group(cancelled=True) + with self.assertRaisesRegex( + ValidationError, + "Can't set a group to cancelled", + ): + group.clean() + + def test_ballot_can_be_cancelled(self): + ballot = self._make_ballot(cancelled=True, poll_open_date=None) + # should not raise + ballot.clean() + + def test_cancellation_notice_requires_cancelled(self): + ballot = self._make_ballot(cancelled=False, poll_open_date=None) + with ( + patch.object( + type(ballot), + "cancellation_notice", + new_callable=PropertyMock, + return_value=object(), # truthy stand-in for a Document + ), + self.assertRaisesRegex( + ValidationError, + "Only a cancelled election can have a cancellation notice", + ), + ): + ballot.clean() + + def test_cancellation_reason_requires_cancelled(self): + ballot = self._make_ballot( + cancelled=False, + cancellation_reason=ElectionCancellationReason.NO_CANDIDATES, + ) + with self.assertRaisesRegex( + ValidationError, + "Only a cancelled election can have a cancellation reason", + ): + ballot.clean() + + def test_cancelled_ballot_with_reason_is_valid(self): + ballot = self._make_ballot( + cancelled=True, + cancellation_reason=ElectionCancellationReason.NO_CANDIDATES, + poll_open_date=None, + ) + # should not raise + ballot.clean() + + # --- by_election_reason rules --- + + def test_by_election_reason_requires_by_election_id(self): + ballot = self._make_ballot( + election_id=f"local.test.test-div.{self.POLL_DATE}", + by_election_reason=ByElectionReason.RESIGNATION, + ) + with self.assertRaisesRegex( + ValidationError, + "Only a by election can have a by_election_reason", + ): + ballot.clean() + + def test_by_election_reason_allowed_on_by_election(self): + ballot = self._make_ballot( + election_id=f"local.test.test-div.by.{self.POLL_DATE}", + by_election_reason=ByElectionReason.RESIGNATION, + poll_open_date=None, + ) + # should not raise + ballot.clean() + + def test_not_applicable_reason_allowed_on_non_by_election(self): + ballot = self._make_ballot( + by_election_reason=ByElectionReason.NOT_APPLICABLE, + poll_open_date=None, + ) + # should not raise + ballot.clean() + + # --- timetable field presence rules --- + + def test_timetable_field_required_raises(self): + ballot = self._make_ballot() + ballot.notice_of_election_deadline = self.POLL_DATE - timedelta(days=25) + ballot.close_of_nominations = self.POLL_DATE - timedelta(days=19) + ballot.sopn_publish_deadline = self.POLL_DATE - timedelta(days=19) + ballot.postal_vote_application_deadline = self.POLL_DATE - timedelta( + days=11 + ) + + ballot.registration_deadline = None + + with self.assertRaisesRegex( + ValidationError, "registration_deadline is required" + ): + ballot.clean() + + def test_timetable_field_set_on_group_raises(self): + group = self._make_group() + group.registration_deadline = self.POLL_DATE - timedelta(days=11) + with self.assertRaisesRegex( + ValidationError, + "registration_deadline should not be set for this election", + ): + group.clean() + + def test_timetable_field_set_without_poll_date_raises(self): + ballot = self._make_ballot(poll_open_date=None) + ballot.registration_deadline = date(2024, 4, 21) + with self.assertRaisesRegex( + ValidationError, + "registration_deadline should not be set for this election", + ): + ballot.clean() + + # --- timetable field date range rules --- + + def test_timetable_field_after_poll_date_raises(self): + ballot = self._make_ballot() + ballot.notice_of_election_deadline = self.POLL_DATE - timedelta(days=25) + ballot.close_of_nominations = self.POLL_DATE - timedelta(days=19) + ballot.sopn_publish_deadline = self.POLL_DATE - timedelta(days=19) + ballot.registration_deadline = self.POLL_DATE - timedelta(days=11) + ballot.postal_vote_application_deadline = self.POLL_DATE + timedelta( + days=1 + ) # after poll + with self.assertRaisesRegex( + ValidationError, + "postal_vote_application_deadline must be before poll_open_date", + ): + ballot.clean() + + def test_timetable_field_too_far_before_poll_date_raises(self): + ballot = self._make_ballot() + ballot.notice_of_election_deadline = self.POLL_DATE - timedelta( + days=51 + ) # > 50 days + ballot.close_of_nominations = self.POLL_DATE - timedelta(days=19) + ballot.sopn_publish_deadline = self.POLL_DATE - timedelta(days=19) + ballot.registration_deadline = self.POLL_DATE - timedelta(days=11) + ballot.postal_vote_application_deadline = self.POLL_DATE - timedelta( + days=11 + ) + with self.assertRaisesRegex( + ValidationError, + "notice_of_election_deadline must be within 50 days of poll_open_date", + ): + ballot.clean() + + def test_valid_ballot_with_timetable_fields_passes(self): + ballot = self._make_ballot() + ballot.notice_of_election_deadline = self.POLL_DATE - timedelta(days=25) + ballot.close_of_nominations = self.POLL_DATE - timedelta(days=19) + ballot.sopn_publish_deadline = self.POLL_DATE - timedelta(days=19) + ballot.registration_deadline = self.POLL_DATE - timedelta(days=11) + ballot.postal_vote_application_deadline = self.POLL_DATE - timedelta( + days=11 + ) + # should not raise + ballot.clean() + + def test_provisional_ballot_passes_without_timetable_fields(self): + ballot = self._make_ballot(poll_open_date=None) + # should not raise + ballot.clean() diff --git a/pyproject.toml b/pyproject.toml index d6b046f20..f5d18e5eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "retry2==0.9.5", "scrapy==2.16.0", "uk-election-ids==0.10.1", - "uk-election-timetables==4.4.0", + "uk-election-timetables==5.0.0", "uk-geo-utils==0.19.1", "dc-django-utils", "dc-design-system", diff --git a/uv.lock b/uv.lock index 52f41cf8c..d70d52def 100644 --- a/uv.lock +++ b/uv.lock @@ -703,7 +703,7 @@ requires-dist = [ { name = "scrapy", specifier = "==2.16.0" }, { name = "tippecanoe", specifier = "==2.72.0" }, { name = "uk-election-ids", specifier = "==0.10.1" }, - { name = "uk-election-timetables", specifier = "==4.4.0" }, + { name = "uk-election-timetables", specifier = "==5.0.0" }, { name = "uk-geo-utils", specifier = "==0.19.1" }, ] @@ -1798,11 +1798,11 @@ wheels = [ [[package]] name = "uk-election-timetables" -version = "4.4.0" +version = "5.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/24/7e/fcb9e159c3e38b38305e0d51d2737d5680a993e6180c5e1e09477efb6a25/uk_election_timetables-4.4.0.tar.gz", hash = "sha256:1462ee03dad497550078d1a3728ee88aa5c92a226fef0c9d1fddc525663658df", size = 18746, upload-time = "2026-03-31T07:12:06.506Z" } +sdist = { url = "https://files.pythonhosted.org/packages/33/06/34e1ad5f782b6e72f56d3d950d7f364dfe1554362aff7d218f494218ff7f/uk_election_timetables-5.0.0.tar.gz", hash = "sha256:f6d6db174d9e0fcc9a29a8167047530b6d7b93a27af4958c0195fa293e307fba", size = 20743, upload-time = "2026-07-08T08:14:15.173Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/6c/6351387e8e88e97d6e2bfb65bc895018199f0e59277d4e03927dd18c3ed7/uk_election_timetables-4.4.0-py3-none-any.whl", hash = "sha256:f14a84d6f95cac34835018d8292b3e664bad8cd98323795477976bf89d744973", size = 21076, upload-time = "2026-03-31T07:12:05.316Z" }, + { url = "https://files.pythonhosted.org/packages/52/8a/1220516bc7c3fe46b5dce6d7a8b67bf601e3c5d1757a9e66a2f804b8ba41/uk_election_timetables-5.0.0-py3-none-any.whl", hash = "sha256:3f85d746b7778bb8b901b1cce0dd92b16592d5fe5a3f7ce2c5c6ebd2ceb71d28", size = 23120, upload-time = "2026-07-08T08:14:14.084Z" }, ] [[package]]