From 94513885548ac7ef1c586756682ffc889ece02e7 Mon Sep 17 00:00:00 2001 From: nickwolf Date: Wed, 15 Jul 2026 16:25:29 -0600 Subject: [PATCH] fix(cache): stream library refresh instead of loading every episode at once refresh_library_cache() iterated over PlexServer.episodes(), which materializes every episode of every non-ignored show library into a single list before the loop body runs. Peak memory therefore scales with library size. On a 65,906-episode library this exceeds 1GB and the process is OOM-killed part way through the refresh, before it can complete. Because refresh_library_on_scan defaults to true, every Plex library scan retriggers it, so the container ends up in a restart loop (observed: 40 OOM kills in one day, 228 restarts). Raising the memory limit only moves the ceiling, as the allocation grows with the library. Add UnprivilegedPlexServer.iter_episodes(), which pages through each show section and yields each page before requesting the next, so pages are released as they are consumed. episodes() is kept as a thin list(iter_episodes()) wrapper for compatibility. Notes on the implementation: - maxresults is required, not redundant. Without it fetchItems() paginates internally up to the section's total size and returns it in one list, which would silently reinstate the behaviour being fixed. - The cursor advances by len(page) and stops only on an empty page. Advancing by a fixed page_size would leave a permanent gap if a short page were ever returned mid-section, and a skipped episode is not benign: it is evicted from episode_parts, so the next refresh treats it as newly added and re-applies the show's track selection over every user's manual choice. - Results are sorted by addedAt so episodes imported while the scan is running sort after the cursor, rather than shifting the offsets of unread pages. The refresh runs precisely because content was just added, so this window is real. - The unprivileged branch previously issued a library-wide query; episodes only exist in show sections, so iterating those covers the same items without asking the server for everything at once. refresh_library_cache() additionally skips collecting added/updated when the cache is cold. There is nothing to diff against, so every episode would be reported as "added" and retained to build a list that the only cold-cache caller discards. Measured against a 65,906-episode library (plexapi 4.18.1): before: OOM-killed, >1024MB, refresh never completed after: 92MB peak RSS, 65,906/65,906 episodes, no duplicates, completes in 129s The remaining growth is the episode_parts dict of part-key strings (~0.3KB per episode), not the Episode objects. --- plex_auto_languages/plex_server.py | 69 +++++++++++++++++++----- plex_auto_languages/plex_server_cache.py | 18 ++++++- 2 files changed, 73 insertions(+), 14 deletions(-) diff --git a/plex_auto_languages/plex_server.py b/plex_auto_languages/plex_server.py index f2dde76..df09ca7 100644 --- a/plex_auto_languages/plex_server.py +++ b/plex_auto_languages/plex_server.py @@ -5,7 +5,7 @@ import re import concurrent.futures from urllib.parse import urlparse -from typing import Union, Callable, List, Tuple, Optional +from typing import Union, Callable, Iterator, List, Tuple, Optional from datetime import datetime, timedelta from requests import ConnectionError as RequestsConnectionError from urllib3.exceptions import InsecureRequestWarning @@ -155,24 +155,69 @@ def fetch_item(self, item_id: Union[str, int]) -> Optional[object]: except NotFound: return None + def iter_episodes(self, page_size: int = 1000) -> Iterator[Episode]: + """ + Iterate over all episodes from non-ignored libraries, one page at a time. + + Each page is released once consumed, so peak memory stays flat regardless + of library size. Prefer this over :meth:`episodes` when episodes are + consumed one at a time, as that method holds the whole library at once. + + Args: + page_size (int): Number of episodes to request per page. + + Yields: + Episode: Episodes from non-ignored Plex libraries. + """ + if hasattr(self, 'should_ignore_library'): + # For PlexServer instances that have the should_ignore_library method + sections = self.get_show_sections() + else: + # For UnprivilegedPlexServer instances or fallback. Episodes only ever + # live in show sections, so this covers the same items as the previous + # library-wide query without requesting the whole library at once. + sections = [s for s in self._plex.library.sections() if isinstance(s, ShowSection)] + + for section in sections: + container_start = 0 + while True: + # `maxresults` is required, not redundant: without it fetchItems() + # paginates internally up to the section's total size and returns + # the whole thing in one list, which is the behaviour this method + # exists to avoid. Capping it to one page keeps this loop in charge. + # Sorting by addedAt keeps episodes imported mid-scan after the + # cursor, so they are picked up at the tail rather than shifting + # the offsets of pages that have not been read yet. + page = section.search( + libtype="episode", + sort="addedAt:asc", + container_start=container_start, + container_size=page_size, + maxresults=page_size + ) + if not page: + break + yield from page + # Advance by what was actually returned rather than by page_size: + # a short page mid-section would otherwise leave a permanent gap, + # and a skipped episode is not harmless here. It gets evicted from + # episode_parts, so the next refresh sees it as newly added and + # re-applies the show's track selection over every user's manual + # choice. + container_start += len(page) + def episodes(self) -> List[Episode]: """ Get all episodes from non-ignored libraries in the Plex server. + Note: + This holds every episode in memory at once. Prefer :meth:`iter_episodes` + when the episodes can be consumed one at a time. + Returns: List[Episode]: A list of all episodes in non-ignored Plex libraries. """ - if hasattr(self, 'should_ignore_library'): - # For PlexServer instances that have the should_ignore_library method - non_ignored_sections = self.get_show_sections() - episodes = [] - for section in non_ignored_sections: - section_episodes = section.all(libtype="episode", container_size=1000) - episodes.extend(section_episodes) - return episodes - else: - # For UnprivilegedPlexServer instances or fallback - return self._plex.library.all(libtype="episode", container_size=1000) + return list(self.iter_episodes()) def get_recently_added_episodes(self, minutes: int) -> List[Episode]: """ diff --git a/plex_auto_languages/plex_server_cache.py b/plex_auto_languages/plex_server_cache.py index dd2e3d2..b309445 100644 --- a/plex_auto_languages/plex_server_cache.py +++ b/plex_auto_languages/plex_server_cache.py @@ -418,6 +418,11 @@ def refresh_library_cache(self) -> tuple[list, list]: Updates the episode_parts dictionary with current data from the Plex server. Identifies episodes that have been added or updated since the last refresh. + On a cold cache there is nothing to diff against, so no changes are collected + and both returned lists are empty. The only caller in that situation discards + the return value anyway, and collecting would mean retaining every Episode in + the library to report the whole thing as "added". + Returns: tuple[list, list]: A tuple containing two lists: - List of newly added episodes @@ -434,12 +439,21 @@ def refresh_library_cache(self) -> tuple[list, list]: added = [] updated = [] new_episode_parts = {} - - for episode in self._plex.episodes(): + # A cold cache diffs against nothing, so every episode would land in + # `added` and be retained for a result the caller throws away. + collect_changes = bool(self.episode_parts) + + # Iterate lazily: holding the whole library at once costs >1GB on + # large libraries, which is enough to get the process OOM-killed + # before the refresh completes. + for episode in self._plex.iter_episodes(): part_list = new_episode_parts.setdefault(episode.key, []) for part in episode.iterParts(): part_list.append(part.key) + if not collect_changes: + continue + if episode.key in self.episode_parts and set(self.episode_parts[episode.key]) != set(part_list): updated.append(episode) elif episode.key not in self.episode_parts: