From e8b5a211ba62a55f71b739605882dd858d3f91ec Mon Sep 17 00:00:00 2001 From: MizukiTemma Date: Thu, 16 Jul 2026 14:52:57 +0200 Subject: [PATCH] Fetch all available news from Tu News Co-authored-by: hannaseithe --- integreat_cms/api/v3/news.py | 5 +- integreat_cms/api/v3/social_media_headers.py | 7 +- integreat_cms/core/circleci_settings.py | 2 + integreat_cms/core/settings.py | 18 +++++ .../news_managers/abstract_news_manager.py | 12 ++- integreat_cms/news_managers/tunews_manager.py | 77 ++++++++++++------- tests/api/test_news_endpoint.py | 14 +++- tests/conftest.py | 8 ++ tests/news_managers/test_tunews_manager.py | 47 +++++++++-- tools/test.sh | 4 + 10 files changed, 149 insertions(+), 45 deletions(-) diff --git a/integreat_cms/api/v3/news.py b/integreat_cms/api/v3/news.py index 704882ab94..25ff2605ee 100644 --- a/integreat_cms/api/v3/news.py +++ b/integreat_cms/api/v3/news.py @@ -75,11 +75,14 @@ def news( result for result in sorted_result if result["source"] in sources ] - page_num = request.GET.get("page", 1) try: page_size = min(int(request.GET.get("size", 20)), 500) except (TypeError, ValueError): page_size = 20 + + if "page" not in request.GET and "size" not in request.GET: + return JsonResponse(list(sorted_result), safe=False) + page_num = request.GET.get("page", 1) paginator = Paginator(sorted_result, page_size) page = get_safe_page(paginator, page_num) diff --git a/integreat_cms/api/v3/social_media_headers.py b/integreat_cms/api/v3/social_media_headers.py index bb6a27e4e0..7fa854396d 100644 --- a/integreat_cms/api/v3/social_media_headers.py +++ b/integreat_cms/api/v3/social_media_headers.py @@ -112,7 +112,7 @@ def render_error_headers(request: HttpRequest, error: str) -> HttpResponse: @partial_html_response def root_social_media_headers( request: HttpRequest, - language_slug: str = settings.LANGUAGE_CODE, + language_slug: str | None = None, ) -> HttpResponse: """ Renders the social media headers for a root page @@ -122,7 +122,10 @@ def root_social_media_headers( :return: HTML social meta headers required by social media platforms """ - language = get_object_or_404(Language, slug=language_slug) + language = get_object_or_404( + Language, + slug=language_slug or settings.LANGUAGE_CODE, + ) title = language.social_media_webapp_title or settings.BRANDING_TITLE url = site_url(request) diff --git a/integreat_cms/core/circleci_settings.py b/integreat_cms/core/circleci_settings.py index 0f5d4fe3b1..bbc9465fc1 100644 --- a/integreat_cms/core/circleci_settings.py +++ b/integreat_cms/core/circleci_settings.py @@ -40,6 +40,8 @@ LINKCHECK_DISABLE_LISTENERS = True # Disable background tasks during testing BACKGROUND_TASKS_ENABLED = False +# Disable auto re-import of external news on demand +EXTERNALNEWS_DISABLE_AUTO_REIMPORT = True #: Enable logging of all entries from the messages framework MESSAGE_LOGGING_ENABLED = True #: Use debug logging on CircleCI diff --git a/integreat_cms/core/settings.py b/integreat_cms/core/settings.py index 36f9df023a..881b611ca2 100644 --- a/integreat_cms/core/settings.py +++ b/integreat_cms/core/settings.py @@ -281,6 +281,24 @@ 24, ) +################# +# External News # +################# + +#: How many days back news posts should be fetched from Tü News +#: Default to FCM_HISTORY_DAYS +TUNEWS_HISTORY_DAYS: Final[int] = env_int( + "INTEGREAT_CMS_TUNEWS_HISTORY_DAYS", + FCM_HISTORY_DAYS, +) + +#: Disable re-importing of external news posts on demand e.g. in test context +EXTERNALNEWS_DISABLE_AUTO_REIMPORT: bool = bool( + strtobool( + os.environ.get("INTEGREAT_CMS_EXTERNALNEWS_DISABLE_AUTO_REIMPORT", "False") + ), +) + ########### # GVZ API # ########### diff --git a/integreat_cms/news_managers/abstract_news_manager.py b/integreat_cms/news_managers/abstract_news_manager.py index c1c5c0a01d..1382cea859 100644 --- a/integreat_cms/news_managers/abstract_news_manager.py +++ b/integreat_cms/news_managers/abstract_news_manager.py @@ -5,6 +5,7 @@ from io import StringIO from typing import TYPE_CHECKING, TypedDict +from django.conf import settings from django.core.cache import cache from django.http import Http404, JsonResponse from lxml import etree @@ -75,9 +76,14 @@ def get_cached_news_items(self, language_slug: str) -> list[NewsItem]: cache_key = f"{self.short_name}:{language_slug}" posts = cache.get(cache_key, _CACHE_MISS) if posts is _CACHE_MISS: - logger.info("Cache miss for %s; importing news items on demand.", cache_key) - self.import_news_items() - posts = cache.get(cache_key, []) + if not settings.EXTERNALNEWS_DISABLE_AUTO_REIMPORT: + logger.info( + "Cache miss for %s; importing news items on demand.", cache_key + ) + self.import_news_items() + posts = cache.get(cache_key, []) + else: + return [] return posts def collect_news_items( diff --git a/integreat_cms/news_managers/tunews_manager.py b/integreat_cms/news_managers/tunews_manager.py index 75bc92eb95..71db8277ce 100644 --- a/integreat_cms/news_managers/tunews_manager.py +++ b/integreat_cms/news_managers/tunews_manager.py @@ -1,10 +1,11 @@ from __future__ import annotations import logging -from datetime import datetime +from datetime import datetime, timedelta from typing import TypedDict import requests +from django.conf import settings from django.core.cache import cache from ..cms.models import Language @@ -42,54 +43,76 @@ def import_news_items(self) -> None: """ Imports Tü News posts and save them as cache """ - for language in Language.objects.all(): - try: - response = requests.get( - f"https://tuenews.de/wp-json/wp/v2/posts/?lang={language.slug}", - timeout=10, - ) + per_page = 100 + oldest_date = ( + datetime.now() - timedelta(days=settings.TUNEWS_HISTORY_DAYS) + ).isoformat() - if response.status_code != 200: - logger.error( - "Could not find posts in %s.", - language, + for language in Language.objects.all(): + news = [] + page_count = 1 + import_failed = False + while True: + try: + response = requests.get( + f"https://tuenews.de/wp-json/wp/v2/posts/?lang={language.slug}&per_page={per_page}&page={page_count}&after={oldest_date}", + timeout=10, + ) + except requests.exceptions.RequestException: + logger.exception( + "Failed to fetch posts from TuNews in %s.", language ) - continue + import_failed = True + break - posts = response.json() + result = response.json() + if response.status_code != 200: + code = result.get("code") + if code == "rest_invalid_param" and "lang" in result.get( + "data", {} + ).get("params", {}): + # Language genuinely not served by Tü News, not a failure: + # cache the (empty) result to avoid retrying import at every request. + logger.debug( + "Tü News does not serve %s; caching empty result.", language + ) + elif code == "rest_post_invalid_page_number": + # No more pages, not a failure: keep the posts gathered so far. + logger.debug("Reached end of pagination for %s.", language) + else: + logger.error( + "Could not fetch page %s in %s.", page_count, language + ) + import_failed = True + break logger.info( - "Got %s posts in %s.", - len(posts), + "Got %s result in %s.", + len(result), language, ) - - news = [] - - for post in posts: + for post in result: try: if not post["acf"]["integreat"]: continue news.append(self.transform_post(post)) except (KeyError, TypeError, ValueError): logger.exception( - "Malformed %s post (id=%s); skipped.", - self.name, + "Malformed Tü News post (id=%s); skipped.", post.get("id"), ) + if len(result) < per_page: + break + page_count += 1 + + if not import_failed: cache.set(f"{self.short_name}:{language.slug}", news, timeout=None) - logger.info("Saving %s news in %s", len(news), language) logger.info( "Saved %s news in %s", len(cache.get(f"{self.short_name}:{language.slug}")), language, ) - for post in cache.get(f"{self.short_name}:{language.slug}"): - logger.info(post.get("title")) - - except requests.exceptions.RequestException: - logger.exception("Failed to fetch posts in %s.", language) def transform_post(self, post: _TunewsPost) -> NewsItem: """ diff --git a/tests/api/test_news_endpoint.py b/tests/api/test_news_endpoint.py index 4c8786ec1d..0d4b7dba85 100644 --- a/tests/api/test_news_endpoint.py +++ b/tests/api/test_news_endpoint.py @@ -44,7 +44,9 @@ def _create_push_notification( @pytest.mark.django_db -def test_news_endpoint(load_test_data: None, clean_news_cache: None) -> None: +def test_news_endpoint( + load_test_data: None, clean_news_cache: None, disable_auto_news_reimport: None +) -> None: """ The combined endpoint merges items from every news source and sorts them by ``display_date``, newest first. @@ -121,7 +123,9 @@ def test_news_endpoint(load_test_data: None, clean_news_cache: None) -> None: @pytest.mark.django_db -def test_news_endpoint_pagination(load_test_data: None, clean_news_cache: None) -> None: +def test_news_endpoint_pagination( + load_test_data: None, clean_news_cache: None, disable_auto_news_reimport: None +) -> None: """ The combined endpoint returns only one page of results at a time. Page 1 contains the most recent items; page 2 contains the next batch. @@ -170,7 +174,7 @@ def test_news_endpoint_pagination(load_test_data: None, clean_news_cache: None) @pytest.mark.django_db def test_news_endpoint_source_filter( - load_test_data: None, clean_news_cache: None + load_test_data: None, clean_news_cache: None, disable_auto_news_reimport: None ) -> None: """ The combined endpoint supports filtering by source. @@ -260,7 +264,9 @@ def test_news_endpoint_source_filter( @pytest.mark.django_db -def test_single_news_endpoint(load_test_data: None, clean_news_cache: None) -> None: +def test_single_news_endpoint( + load_test_data: None, clean_news_cache: None, disable_auto_news_reimport: None +) -> None: """ The single news endpoint returns one news that matches the given id. diff --git a/tests/conftest.py b/tests/conftest.py index d3d68011c2..248a03eb39 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -176,3 +176,11 @@ def clean_news_cache(load_test_data: None) -> Generator[None]: yield for key in keys: cache.delete(key) + + +@pytest.fixture() +def disable_auto_news_reimport(settings: SettingsWrapper) -> None: + """ + Disable re-import of external news on demand to avoid hitting the real APIs and getting real news posts + """ + settings.EXTERNALNEWS_DISABLE_AUTO_REIMPORT = True diff --git a/tests/news_managers/test_tunews_manager.py b/tests/news_managers/test_tunews_manager.py index b921b0da98..f17996f5c2 100644 --- a/tests/news_managers/test_tunews_manager.py +++ b/tests/news_managers/test_tunews_manager.py @@ -1,11 +1,12 @@ from __future__ import annotations from datetime import datetime, UTC -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest from django.core.cache import cache +from integreat_cms.cms.models import Language from integreat_cms.news_managers.abstract_news_manager import clean_html from integreat_cms.news_managers.amalnews_manager import AmalnewsManager from integreat_cms.news_managers.tunews_manager import TunewsManager @@ -116,24 +117,54 @@ @pytest.mark.django_db -def test_import_news_item(load_test_data: None, clean_news_cache: None) -> None: +def test_import_tunews_item(clean_news_cache: None) -> None: + response_1 = Mock() + response_1.status_code = 200 + response_1.json.return_value = dummy_news_items + + response_2 = Mock() + response_2.status_code = 400 + + first_language_slug = Language.objects.first().slug + with patch( "integreat_cms.news_managers.tunews_manager.requests.get" ) as fake_tunews_server: - fake_tunews_server.return_value.status_code = 200 - fake_tunews_server.return_value.json.return_value = dummy_news_items + fake_tunews_server.side_effect = [response_1, response_2] * ( + Language.objects.all().count() + ) - assert not cache.get(f"{tu_short_name}:de") + assert not cache.get(f"{tu_short_name}:{first_language_slug}") TunewsManager().import_news_items() - assert cache.get(f"{tu_short_name}:de") == expected_result_tunews + assert ( + cache.get(f"{tu_short_name}:{first_language_slug}") + == expected_result_tunews + ) + + +@pytest.mark.django_db +def test_import_amalnews_item(clean_news_cache: None) -> None: + response_1 = Mock() + response_1.status_code = 200 + response_1.json.return_value = dummy_news_items + + first_language_slug = Language.objects.first().slug + + with patch( + "integreat_cms.news_managers.amalnews_manager.requests.get" + ) as fake_tunews_server: + fake_tunews_server.side_effect = [response_1] * (Language.objects.all().count()) - assert not cache.get(f"{amal_short_name}:de") + assert not cache.get(f"{amal_short_name}:{first_language_slug}") AmalnewsManager().import_news_items() - assert cache.get(f"{amal_short_name}:de") == expected_result_amalnews + assert ( + cache.get(f"{amal_short_name}:{first_language_slug}") + == expected_result_amalnews + ) def test_clean_html_keeps_plain_text() -> None: diff --git a/tools/test.sh b/tools/test.sh index 484c1b8c54..b67d458965 100755 --- a/tools/test.sh +++ b/tools/test.sh @@ -31,6 +31,10 @@ export INTEGREAT_CMS_LINKCHECK_DISABLE_LISTENERS=1 # Disable background tasks during testing export INTEGREAT_CMS_BACKGROUND_TASKS_ENABLED=0 +# Disable re-importing of external news posts on demand +export INTEGREAT_CMS_EXTERNALNEWS_DISABLE_AUTO_REIMPORT=1 + + TESTS=() # Parse given command line arguments