Fetch all available news from Tü News

integreat_cms/api/v3/news.py

@@ -75,11 +75,14 @@ def news(
             result for result in sorted_result if result["source"] in sources
         ]
 
-    page_num = request.GET.get("page", 1)
     try:
         page_size = min(int(request.GET.get("size", 20)), 500)
     except (TypeError, ValueError):
         page_size = 20
+
+    if "page" not in request.GET and "size" not in request.GET:
+        return JsonResponse(list(sorted_result), safe=False)
+    page_num = request.GET.get("page", 1)
     paginator = Paginator(sorted_result, page_size)
     page = get_safe_page(paginator, page_num)
 

integreat_cms/api/v3/social_media_headers.py

@@ -112,7 +112,7 @@ def render_error_headers(request: HttpRequest, error: str) -> HttpResponse:
 @partial_html_response
 def root_social_media_headers(
     request: HttpRequest,
-    language_slug: str = settings.LANGUAGE_CODE,
+    language_slug: str | None = None,
 ) -> HttpResponse:
     """
     Renders the social media headers for a root page
@@ -122,7 +122,10 @@ def root_social_media_headers(
 
     :return: HTML social meta headers required by social media platforms
     """
-    language = get_object_or_404(Language, slug=language_slug)
+    language = get_object_or_404(
+        Language,
+        slug=language_slug or settings.LANGUAGE_CODE,
+    )
     title = language.social_media_webapp_title or settings.BRANDING_TITLE
     url = site_url(request)
 

integreat_cms/core/circleci_settings.py

@@ -40,6 +40,8 @@
 LINKCHECK_DISABLE_LISTENERS = True
 # Disable background tasks during testing
 BACKGROUND_TASKS_ENABLED = False
+# Disable auto re-import of external news on demand
+EXTERNALNEWS_DISABLE_AUTO_REIMPORT = True
 #: Enable logging of all entries from the messages framework
 MESSAGE_LOGGING_ENABLED = True
 #: Use debug logging on CircleCI

integreat_cms/core/settings.py

@@ -281,6 +281,24 @@
     24,
 )
 
+#################
+# External News #
+#################
+
+#: How many days back news posts should be fetched from Tü News
+#: Default to FCM_HISTORY_DAYS
+TUNEWS_HISTORY_DAYS: Final[int] = env_int(
+    "INTEGREAT_CMS_TUNEWS_HISTORY_DAYS",
+    FCM_HISTORY_DAYS,
+)
+
+#: Disable re-importing of external news posts on demand e.g. in test context
+EXTERNALNEWS_DISABLE_AUTO_REIMPORT: bool = bool(
+    strtobool(
+        os.environ.get("INTEGREAT_CMS_EXTERNALNEWS_DISABLE_AUTO_REIMPORT", "False")
+    ),
+)
+
 ###########
 # GVZ API #
 ###########

integreat_cms/news_managers/abstract_news_manager.py

@@ -5,6 +5,7 @@
 from io import StringIO
 from typing import TYPE_CHECKING, TypedDict
 
+from django.conf import settings
 from django.core.cache import cache
 from django.http import Http404, JsonResponse
 from lxml import etree
@@ -75,9 +76,14 @@ def get_cached_news_items(self, language_slug: str) -> list[NewsItem]:
         cache_key = f"{self.short_name}:{language_slug}"
         posts = cache.get(cache_key, _CACHE_MISS)
         if posts is _CACHE_MISS:
-            logger.info("Cache miss for %s; importing news items on demand.", cache_key)
-            self.import_news_items()
-            posts = cache.get(cache_key, [])
+            if not settings.EXTERNALNEWS_DISABLE_AUTO_REIMPORT:
+                logger.info(
+                    "Cache miss for %s; importing news items on demand.", cache_key
+                )
+                self.import_news_items()
+                posts = cache.get(cache_key, [])
+            else:
+                return []
         return posts
 
     def collect_news_items(

integreat_cms/news_managers/tunews_manager.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
 
 import logging
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import TypedDict
 
 import requests
+from django.conf import settings
 from django.core.cache import cache
 
 from ..cms.models import Language
@@ -42,54 +43,76 @@ def import_news_items(self) -> None:
         """
         Imports Tü News posts and save them as cache
         """
-        for language in Language.objects.all():
-            try:
-                response = requests.get(
-                    f"https://tuenews.de/wp-json/wp/v2/posts/?lang={language.slug}",
-                    timeout=10,
-                )
+        per_page = 100
+        oldest_date = (
+            datetime.now() - timedelta(days=settings.TUNEWS_HISTORY_DAYS)
+        ).isoformat()
 
-                if response.status_code != 200:
-                    logger.error(
-                        "Could not find posts in %s.",
-                        language,
+        for language in Language.objects.all():
+            news = []
+            page_count = 1
+            import_failed = False
+            while True:
+                try:
+                    response = requests.get(
+                        f"https://tuenews.de/wp-json/wp/v2/posts/?lang={language.slug}&per_page={per_page}&page={page_count}&after={oldest_date}",
+                        timeout=10,
+                    )
+                except requests.exceptions.RequestException:
+                    logger.exception(
+                        "Failed to fetch posts from TuNews in %s.", language
                     )
-                    continue
+                    import_failed = True
+                    break
 
-                posts = response.json()
+                result = response.json()
+                if response.status_code != 200:
+                    code = result.get("code")
+                    if code == "rest_invalid_param" and "lang" in result.get(
+                        "data", {}
+                    ).get("params", {}):
+                        # Language genuinely not served by Tü News, not a failure:
+                        # cache the (empty) result to avoid retrying import at every request.
+                        logger.debug(
+                            "Tü News does not serve %s; caching empty result.", language
+                        )
+                    elif code == "rest_post_invalid_page_number":
+                        # No more pages, not a failure: keep the posts gathered so far.
+                        logger.debug("Reached end of pagination for %s.", language)
+                    else:
+                        logger.error(
+                            "Could not fetch page %s in %s.", page_count, language
+                        )
+                        import_failed = True
+                    break
 
                 logger.info(
-                    "Got %s posts in %s.",
-                    len(posts),
+                    "Got %s result in %s.",
+                    len(result),
                     language,
                 )
-
-                news = []
-
-                for post in posts:
+                for post in result:
                     try:
                         if not post["acf"]["integreat"]:
                             continue
                         news.append(self.transform_post(post))
                     except (KeyError, TypeError, ValueError):
                         logger.exception(
-                            "Malformed %s post (id=%s); skipped.",
-                            self.name,
+                            "Malformed Tü News post (id=%s); skipped.",
                             post.get("id"),
                         )
 
+                if len(result) < per_page:
+                    break
+                page_count += 1
+
+            if not import_failed:
                 cache.set(f"{self.short_name}:{language.slug}", news, timeout=None)
-                logger.info("Saving %s news in %s", len(news), language)
                 logger.info(
                     "Saved %s news in %s",
                     len(cache.get(f"{self.short_name}:{language.slug}")),
                     language,
                 )
-                for post in cache.get(f"{self.short_name}:{language.slug}"):
-                    logger.info(post.get("title"))
-
-            except requests.exceptions.RequestException:
-                logger.exception("Failed to fetch posts in %s.", language)
 
     def transform_post(self, post: _TunewsPost) -> NewsItem:
         """

tests/api/test_news_endpoint.py

@@ -44,7 +44,9 @@ def _create_push_notification(
 
 
 @pytest.mark.django_db
-def test_news_endpoint(load_test_data: None, clean_news_cache: None) -> None:
+def test_news_endpoint(
+    load_test_data: None, clean_news_cache: None, disable_auto_news_reimport: None
+) -> None:
     """
     The combined endpoint merges items from every news source and sorts them
     by ``display_date``, newest first.
@@ -121,7 +123,9 @@ def test_news_endpoint(load_test_data: None, clean_news_cache: None) -> None:
 
 
 @pytest.mark.django_db
-def test_news_endpoint_pagination(load_test_data: None, clean_news_cache: None) -> None:
+def test_news_endpoint_pagination(
+    load_test_data: None, clean_news_cache: None, disable_auto_news_reimport: None
+) -> None:
     """
     The combined endpoint returns only one page of results at a time.
     Page 1 contains the most recent items; page 2 contains the next batch.
@@ -170,7 +174,7 @@ def test_news_endpoint_pagination(load_test_data: None, clean_news_cache: None)
 
 @pytest.mark.django_db
 def test_news_endpoint_source_filter(
-    load_test_data: None, clean_news_cache: None
+    load_test_data: None, clean_news_cache: None, disable_auto_news_reimport: None
 ) -> None:
     """
     The combined endpoint supports filtering by source.
@@ -260,7 +264,9 @@ def test_news_endpoint_source_filter(
 
 
 @pytest.mark.django_db
-def test_single_news_endpoint(load_test_data: None, clean_news_cache: None) -> None:
+def test_single_news_endpoint(
+    load_test_data: None, clean_news_cache: None, disable_auto_news_reimport: None
+) -> None:
     """
     The single news endpoint returns one news that matches the given id.
 

tests/conftest.py

@@ -176,3 +176,11 @@ def clean_news_cache(load_test_data: None) -> Generator[None]:
     yield
     for key in keys:
         cache.delete(key)
+
+
+@pytest.fixture()
+def disable_auto_news_reimport(settings: SettingsWrapper) -> None:
+    """
+    Disable re-import of external news on demand to avoid hitting the real APIs and getting real news posts
+    """
+    settings.EXTERNALNEWS_DISABLE_AUTO_REIMPORT = True

tests/news_managers/test_tunews_manager.py

@@ -1,11 +1,12 @@
 from __future__ import annotations
 
 from datetime import datetime, UTC
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 import pytest
 from django.core.cache import cache
 
+from integreat_cms.cms.models import Language
 from integreat_cms.news_managers.abstract_news_manager import clean_html
 from integreat_cms.news_managers.amalnews_manager import AmalnewsManager
 from integreat_cms.news_managers.tunews_manager import TunewsManager
@@ -116,24 +117,54 @@
 
 
 @pytest.mark.django_db
-def test_import_news_item(load_test_data: None, clean_news_cache: None) -> None:
+def test_import_tunews_item(clean_news_cache: None) -> None:
+    response_1 = Mock()
+    response_1.status_code = 200
+    response_1.json.return_value = dummy_news_items
+
+    response_2 = Mock()
+    response_2.status_code = 400
+
+    first_language_slug = Language.objects.first().slug
+
     with patch(
         "integreat_cms.news_managers.tunews_manager.requests.get"
     ) as fake_tunews_server:
-        fake_tunews_server.return_value.status_code = 200
-        fake_tunews_server.return_value.json.return_value = dummy_news_items
+        fake_tunews_server.side_effect = [response_1, response_2] * (
+            Language.objects.all().count()
+        )
 
-        assert not cache.get(f"{tu_short_name}:de")
+        assert not cache.get(f"{tu_short_name}:{first_language_slug}")
 
         TunewsManager().import_news_items()
 
-        assert cache.get(f"{tu_short_name}:de") == expected_result_tunews
+        assert (
+            cache.get(f"{tu_short_name}:{first_language_slug}")
+            == expected_result_tunews
+        )
+
+
+@pytest.mark.django_db
+def test_import_amalnews_item(clean_news_cache: None) -> None:
+    response_1 = Mock()
+    response_1.status_code = 200
+    response_1.json.return_value = dummy_news_items
+
+    first_language_slug = Language.objects.first().slug
+
+    with patch(
+        "integreat_cms.news_managers.amalnews_manager.requests.get"
+    ) as fake_tunews_server:
+        fake_tunews_server.side_effect = [response_1] * (Language.objects.all().count())
 
-        assert not cache.get(f"{amal_short_name}:de")
+        assert not cache.get(f"{amal_short_name}:{first_language_slug}")
 
         AmalnewsManager().import_news_items()
 
-        assert cache.get(f"{amal_short_name}:de") == expected_result_amalnews
+        assert (
+            cache.get(f"{amal_short_name}:{first_language_slug}")
+            == expected_result_amalnews
+        )
 
 
 def test_clean_html_keeps_plain_text() -> None:

tools/test.sh

@@ -31,6 +31,10 @@ export INTEGREAT_CMS_LINKCHECK_DISABLE_LISTENERS=1
 # Disable background tasks during testing
 export INTEGREAT_CMS_BACKGROUND_TASKS_ENABLED=0
 
+# Disable re-importing of external news posts on demand
+export INTEGREAT_CMS_EXTERNALNEWS_DISABLE_AUTO_REIMPORT=1
+
+
 TESTS=()
 
 # Parse given command line arguments