diff --git a/src/anthias_server/celery_tasks.py b/src/anthias_server/celery_tasks.py index 9a77fd395..076b7b9f3 100755 --- a/src/anthias_server/celery_tasks.py +++ b/src/anthias_server/celery_tasks.py @@ -686,15 +686,21 @@ def download_youtube_asset(asset_id: str, uri: str) -> None: from yt_dlp import YoutubeDL from yt_dlp.utils import DownloadError + from anthias_server.processing import preferred_download_vcodec + ydl_opts = { - # ``format_sort`` mirrors the previous CLI's `-S - # vcodec:h264,fps,res:1080,acodec:m4a` — bias toward h264 - # video and m4a audio, keep resolution at 1080p, prefer - # higher fps. yt-dlp still picks the *best matching* format, - # falling back to whatever is available if no exact match - # exists. Strict `format=` filters would reject videos that - # happen to have only vp9, which we don't want. - 'format_sort': ['vcodec:h264', 'fps', 'res:1080', 'acodec:m4a'], + # ``format_sort`` biases toward the board's preferred codec, + # m4a audio, 1080p resolution, and higher fps. yt-dlp still + # picks the *best matching* format, falling back to whatever + # is available if no exact match exists. Strict ``format=`` + # filters would reject videos that only have other codecs, + # which we don't want — normalize_video_asset handles that. + 'format_sort': [ + f'vcodec:{preferred_download_vcodec()}', + 'fps', + 'res:1080', + 'acodec:m4a', + ], # Final filename — yt-dlp writes .part during the # download and renames on success. cleanup() recognises # .part / .info.json sidecars and skips them inside the 1h diff --git a/src/anthias_server/processing.py b/src/anthias_server/processing.py index 8b3ed0e80..87b074fc8 100644 --- a/src/anthias_server/processing.py +++ b/src/anthias_server/processing.py @@ -885,7 +885,11 @@ def _ffprobe_summary(input_path: str) -> dict[str, Any]: # V4L2 bcm2835-codec), not the Qt5 GStreamer fbdev path. 'pi3-64': frozenset({'h264'}), 'pi4-64': frozenset({'h264', 'hevc'}), - 'pi5': frozenset({'hevc'}), + # hevc: hardware-decoded via v4l2-request (BCM2712 VideoCore VII). + # h264: software-decoded — Cortex-A76 handles 1080p H.264 without + # frame drops, and YouTube rarely serves HEVC so excluding h264 + # would block all YouTube downloads on Pi 5. + 'pi5': frozenset({'hevc', 'h264'}), 'rockpi4': frozenset({'h264', 'hevc'}), 'x86': frozenset({'h264', 'hevc'}), } @@ -933,6 +937,28 @@ def _hw_decoded_codecs() -> frozenset[str]: return _HW_DECODE_VIDEO_CODECS.get(resolve_device_key(), frozenset()) +# Preferred yt-dlp ``vcodec`` sort key per board. Distinct from the +# accepted-codec gate above: Pi 5 accepts H.264 via software decode +# (Cortex-A76 handles 1080p without frame drops) but HEVC is still the +# hardware path, so downloads should bias toward it when available. +# Boards not listed here default to ``h264`` — it is more widely +# available on YouTube and is their primary hardware decode path. +_PREFERRED_DOWNLOAD_VCODEC: dict[str, str] = { + 'pi5': 'hevc', +} + + +def preferred_download_vcodec() -> str: + """yt-dlp ``vcodec`` sort preference for the current board. + + Returns the codec string to place first in ``format_sort`` so + yt-dlp biases downloads toward the board's best playback path. + Falls back to ``'h264'`` for unknown boards and all boards where + H.264 is the primary hardware decode path. + """ + return _PREFERRED_DOWNLOAD_VCODEC.get(resolve_device_key(), 'h264') + + def _ffmpeg_reencode_recipe( supported: frozenset[str], source_filename: str = '', @@ -945,10 +971,10 @@ def _ffmpeg_reencode_recipe( Prefers libx264 when H.264 is in the board's supported set — libx264 is roughly 5-10× faster than libx265 at comparable quality, which matters when the operator is doing the encode by - hand. Falls back to libx265 + ``-tag:v hvc1`` for Pi 5 (HEVC- - only board). Returns an empty string when the board has no HW - decode set at all — there's nothing the operator can transcode - to that would land in a supported pipe. + hand. Falls back to libx265 + ``-tag:v hvc1`` for HEVC-only boards. + Returns an empty string when the board has no HW decode set at all — + there's nothing the operator can transcode to that would land in a + supported pipe. ``source_filename``, when supplied, substitutes the bare upload filename (no path) for the ``INPUT`` placeholder and reuses its @@ -1025,10 +1051,11 @@ def _handbrake_steps(supported: frozenset[str]) -> list[str]: — there's no separate downscale step to spell out. The only board-specific tweak is the encoder: an HEVC-only board - (Pi 5) can't use the preset's default H.264, so the operator flips - ``Video Encoder`` to ``H.265 (x265)`` on the Video tab. HandBrake - ships no H.265-at-1080p MP4 preset, so the encoder swap is the - cleanest route to a 1080p HEVC MP4. + (Pi 5 before the H.264 software-decode fallback was added) can't + use the preset's default H.264, so the operator flips ``Video + Encoder`` to ``H.265 (x265)`` on the Video tab. HandBrake ships no + H.265-at-1080p MP4 preset, so the encoder swap is the cleanest + route to a 1080p HEVC MP4. Returns an empty list when the board has no HW decode set at all — there's nothing to transcode to, exactly as the recipe returns an @@ -1036,8 +1063,8 @@ def _handbrake_steps(supported: frozenset[str]) -> list[str]: list stands alone when surfaced as plain text via the v2 API. """ if 'h264' in supported: - # H.264 board (Pi 2/3/4, x86, ...): the stock preset already - # outputs the codec we want — no encoder change. + # H.264 board (Pi 2/3/4, Pi 5, x86, ...): the stock preset + # already outputs an accepted codec — no encoder change needed. prefers_h264 = True elif 'hevc' in supported: prefers_h264 = False diff --git a/tests/test_celery_tasks.py b/tests/test_celery_tasks.py index fc1656feb..45e9fefb2 100644 --- a/tests/test_celery_tasks.py +++ b/tests/test_celery_tasks.py @@ -986,6 +986,120 @@ def test_download_youtube_asset_on_failure_writes_error_metadata() -> None: mock_notify.assert_called_once_with('yt-1') +# --------------------------------------------------------------------------- +# download_youtube_asset — per-board format_sort selection (GH #3092) +# --------------------------------------------------------------------------- + + +@pytest.mark.django_db +def test_download_youtube_asset_pi5_prefers_hevc_format( + fake_youtube_dl: mock.MagicMock, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Pi 5 biases yt-dlp toward HEVC — it is the hardware decode path + (VideoCore VII). H.264 is accepted as a software fallback (GH #3092) + but HEVC is preferred when YouTube serves it.""" + monkeypatch.setenv('DEVICE_TYPE', 'pi5') + _make_youtube_asset() + fake_youtube_dl.extract_info.return_value = {'title': 't', 'duration': 10} + with ( + mock.patch('anthias_server.app.consumers.notify_asset_update'), + mock.patch('anthias_server.processing.dispatch_normalize_video'), + ): + download_youtube_asset('yt-1', 'https://www.youtube.com/watch?v=abc') + + ydl_opts = fake_youtube_dl._cls.call_args.args[0] + assert ydl_opts['format_sort'] == [ + 'vcodec:hevc', + 'fps', + 'res:1080', + 'acodec:m4a', + ], f'Pi 5 must bias yt-dlp toward HEVC; got: {ydl_opts["format_sort"]!r}' + + +@pytest.mark.django_db +def test_download_youtube_asset_pi4_prefers_h264_format( + fake_youtube_dl: mock.MagicMock, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Pi 4 hardware-decodes both H.264 and HEVC, so it keeps the H.264 + preference — H.264 is more widely available on YouTube and Pi 4 plays + it just as efficiently as HEVC.""" + monkeypatch.setenv('DEVICE_TYPE', 'pi4-64') + _make_youtube_asset() + fake_youtube_dl.extract_info.return_value = {'title': 't', 'duration': 10} + with ( + mock.patch('anthias_server.app.consumers.notify_asset_update'), + mock.patch('anthias_server.processing.dispatch_normalize_video'), + ): + download_youtube_asset('yt-1', 'https://www.youtube.com/watch?v=abc') + + ydl_opts = fake_youtube_dl._cls.call_args.args[0] + assert ydl_opts['format_sort'][0] == 'vcodec:h264' + + +@pytest.mark.django_db +def test_download_youtube_asset_x86_prefers_h264_format( + fake_youtube_dl: mock.MagicMock, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """x86 keeps the H.264 preference — HEVC via VAAPI is known to produce + a black screen on x86, so biasing toward HEVC downloads would cause + silent playback failures when YouTube serves HEVC streams.""" + monkeypatch.setenv('DEVICE_TYPE', 'x86') + _make_youtube_asset() + fake_youtube_dl.extract_info.return_value = {'title': 't', 'duration': 10} + with ( + mock.patch('anthias_server.app.consumers.notify_asset_update'), + mock.patch('anthias_server.processing.dispatch_normalize_video'), + ): + download_youtube_asset('yt-1', 'https://www.youtube.com/watch?v=abc') + + ydl_opts = fake_youtube_dl._cls.call_args.args[0] + assert ydl_opts['format_sort'][0] == 'vcodec:h264' + + +@pytest.mark.django_db +def test_download_youtube_asset_pi3_prefers_h264_format( + fake_youtube_dl: mock.MagicMock, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Pi 3 supports H.264 only; format_sort must request h264.""" + monkeypatch.setenv('DEVICE_TYPE', 'pi3') + _make_youtube_asset() + fake_youtube_dl.extract_info.return_value = {'title': 't', 'duration': 10} + with ( + mock.patch('anthias_server.app.consumers.notify_asset_update'), + mock.patch('anthias_server.processing.dispatch_normalize_video'), + ): + download_youtube_asset('yt-1', 'https://www.youtube.com/watch?v=abc') + + ydl_opts = fake_youtube_dl._cls.call_args.args[0] + assert ydl_opts['format_sort'][0] == 'vcodec:h264' + + +@pytest.mark.django_db +def test_download_youtube_asset_unknown_board_falls_back_to_h264( + fake_youtube_dl: mock.MagicMock, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An unrecognised DEVICE_TYPE yields an empty HW codec set; the + task must not crash and must fall back to the h264 preference so + the download still proceeds. normalize_video_asset will gate on + the actual codec if the board turns out not to support it.""" + monkeypatch.setenv('DEVICE_TYPE', 'unrecognised-board-xyz') + _make_youtube_asset() + fake_youtube_dl.extract_info.return_value = {'title': 't', 'duration': 10} + with ( + mock.patch('anthias_server.app.consumers.notify_asset_update'), + mock.patch('anthias_server.processing.dispatch_normalize_video'), + ): + download_youtube_asset('yt-1', 'https://www.youtube.com/watch?v=abc') + + ydl_opts = fake_youtube_dl._cls.call_args.args[0] + assert ydl_opts['format_sort'][0] == 'vcodec:h264' + + # --------------------------------------------------------------------------- # download_remote_video_asset — generic http(s) single-file video URLs # --------------------------------------------------------------------------- diff --git a/tests/test_processing.py b/tests/test_processing.py index db27b0520..03c8d1ab6 100644 --- a/tests/test_processing.py +++ b/tests/test_processing.py @@ -632,83 +632,106 @@ def test_video_supported_codec_writes_metadata_and_clears_processing( assert path.exists(src) -@pytest_ffmpeg @pytest.mark.django_db def test_video_unsupported_codec_raises_with_ffmpeg_recipe( asset_dir: str, monkeypatch: pytest.MonkeyPatch ) -> None: - """An H.264 upload on a Pi 5 (HEVC only — Pi 5's mpv has no v4l2- - request H.264 hwdec) is rejected. The exception's message names - the rejected codec and supported set; its ``recipe`` attribute - carries an ffmpeg command pre-filled with the upload's filename - (taken from ``metadata.upload_name``, which the upload view - stashes at create time) so the operator can copy-paste it - verbatim.""" - monkeypatch.setenv('DEVICE_TYPE', 'pi5') + """A codec outside the board's HW decode set is rejected. The + exception's message names the rejected codec and supported set; + its ``recipe`` attribute carries an ffmpeg command pre-filled with + the upload's filename so the operator can copy-paste it verbatim. + + Pi 3 (H.264 only) is used here — an HEVC upload is the clearest + unsupported-codec case for that board.""" + monkeypatch.setenv('DEVICE_TYPE', 'pi3') src = path.join(asset_dir, 'sample.mp4') - _make_video(src, codec='libx264', container='mp4', audio='aac') + # Create a minimal placeholder — ffprobe is mocked below so the + # file content doesn't matter, only path.isfile() needs to pass. + with open(src, 'wb') as f: + f.write(b'\x00' * 16) asset = _make_processing_asset( - 'vid-h264-pi5', + 'vid-hevc-pi3', src, mimetype='video', metadata={'upload_name': 'beach-clip.mp4'}, ) + fake_summary = { + 'container': 'mp4', + 'video_codec': 'hevc', + 'video_width': 1920, + 'video_height': 1080, + 'video_fps': 30.0, + 'audio_codec': 'aac', + 'duration_seconds': 60, + } - with mock.patch.object(processing, '_notify'): + with ( + mock.patch.object(processing, '_notify'), + mock.patch.object( + processing, '_ffprobe_summary', return_value=fake_summary + ), + ): with pytest.raises(processing.UnsupportedVideoCodecError) as excinfo: processing._run_video_normalisation(asset) import shlex as _shlex msg = str(excinfo.value) - assert "'h264'" in msg - assert 'hevc' in msg - # Recipe is on the exception, not in the message body. + assert "'hevc'" in msg + assert 'h264' in msg recipe = excinfo.value.recipe - assert 'libx265' in recipe # Pi 5 supports HEVC only. - assert '-tag:v hvc1' in recipe - # The upload's filename appears in the recipe's input slot — - # operator can copy and paste it without hand-editing INPUT. + assert ( + 'libx264' in recipe + ) # Pi 3 supports H.264 — recipe encodes to H.264. tokens = _shlex.split(recipe) assert tokens[1] == '-i' assert tokens[2] == 'beach-clip.mp4' - # Output filename carries a ``.hevc.`` suffix so the recipe - # doesn't ask the operator to overwrite their source file. - assert tokens[-1] == 'beach-clip.hevc.mp4' + assert tokens[-1] == 'beach-clip.h264.mp4' - # The terminal-free alternative: HandBrake steps targeting the - # same codec (H.265 on a Pi 5), with a link to the download page. handbrake = excinfo.value.handbrake assert handbrake joined = ' '.join(handbrake) assert 'HandBrake' in joined assert processing.HANDBRAKE_URL in joined - assert 'H.265 (x265)' in joined - # Codec-only rejection: no resolution-limit step. + # H.264 board: the stock Fast 1080p30 preset is already H.264, + # no encoder-switch step needed. + assert 'H.265 (x265)' not in joined assert not any('Resolution Limit' in step for step in handbrake) - # Metadata was still written so the operator can see *what* they - # uploaded next to the error message in the asset list. asset.refresh_from_db() - assert asset.metadata.get('video_codec') == 'h264' - assert asset.metadata.get('video_width') == 32 + assert asset.metadata.get('video_codec') == 'hevc' + assert asset.metadata.get('video_width') == 1920 -@pytest_ffmpeg @pytest.mark.django_db def test_video_unsupported_codec_recipe_falls_back_to_upload_placeholder( asset_dir: str, monkeypatch: pytest.MonkeyPatch ) -> None: - """When the row has no ``metadata.upload_name`` (YouTube - downloads, pre-rebrand rows), the recipe uses a stable - ``upload`` placeholder so the operator still sees the - correct input extension to substitute.""" - monkeypatch.setenv('DEVICE_TYPE', 'pi5') + """When the row has no ``metadata.upload_name`` (YouTube downloads, + pre-rebrand rows), the recipe uses a stable ``upload`` + placeholder so the operator still sees the correct input extension + to substitute.""" + monkeypatch.setenv('DEVICE_TYPE', 'pi3') src = path.join(asset_dir, 'noname.mp4') - _make_video(src, codec='libx264', container='mp4', audio='aac') + with open(src, 'wb') as f: + f.write(b'\x00' * 16) asset = _make_processing_asset('vid-noname', src, mimetype='video') + fake_summary = { + 'container': 'mp4', + 'video_codec': 'hevc', + 'video_width': 1920, + 'video_height': 1080, + 'video_fps': 30.0, + 'audio_codec': 'aac', + 'duration_seconds': 60, + } - with mock.patch.object(processing, '_notify'): + with ( + mock.patch.object(processing, '_notify'), + mock.patch.object( + processing, '_ffprobe_summary', return_value=fake_summary + ), + ): with pytest.raises(processing.UnsupportedVideoCodecError) as excinfo: processing._run_video_normalisation(asset) @@ -718,7 +741,85 @@ def test_video_unsupported_codec_recipe_falls_back_to_upload_placeholder( tokens = _shlex.split(recipe) assert tokens[1] == '-i' assert tokens[2] == 'upload.mp4' - assert tokens[-1] == 'upload.hevc.mp4' + assert tokens[-1] == 'upload.h264.mp4' + + +@pytest.mark.django_db +def test_video_h264_accepted_on_pi5_software_decode( + asset_dir: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """H.264 is accepted on Pi 5 via software decode — the + Cortex-A76 handles 1080p H.264 without frame drops, and YouTube + rarely serves HEVC so blocking H.264 on Pi 5 would prevent all + YouTube downloads. (GH #3092)""" + monkeypatch.setenv('DEVICE_TYPE', 'pi5') + src = path.join(asset_dir, 'sample.mp4') + with open(src, 'wb') as f: + f.write(b'\x00' * 16) + asset = _make_processing_asset('vid-h264-pi5-sw', src, mimetype='video') + fake_summary = { + 'container': 'mp4', + 'video_codec': 'h264', + 'video_width': 1920, + 'video_height': 1080, + 'video_fps': 30.0, + 'audio_codec': 'aac', + 'duration_seconds': 60, + } + + with ( + mock.patch.object(processing, '_notify') as mock_notify, + mock.patch.object( + processing, '_ffprobe_summary', return_value=fake_summary + ), + ): + processing._run_video_normalisation(asset) + + asset.refresh_from_db() + assert asset.is_processing is False + assert asset.metadata.get('video_codec') == 'h264' + mock_notify.assert_called_once_with('vid-h264-pi5-sw') + + +@pytest.mark.django_db +def test_video_unsupported_codec_still_rejected_on_pi5( + asset_dir: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """VP9 / AV1 uploads on Pi 5 are still rejected even though H.264 + software-decode was added to the Pi 5 codec set. The rejection recipe + tells the operator to re-encode to H.264 (accepted via software decode — + faster to encode than HEVC and still plays fine on Cortex-A76).""" + monkeypatch.setenv('DEVICE_TYPE', 'pi5') + src = path.join(asset_dir, 'sample.mp4') + with open(src, 'wb') as f: + f.write(b'\x00' * 16) + asset = _make_processing_asset('vid-vp9-pi5', src, mimetype='video') + fake_summary = { + 'container': 'mp4', + 'video_codec': 'vp9', + 'video_width': 1920, + 'video_height': 1080, + 'video_fps': 30.0, + 'audio_codec': 'aac', + 'duration_seconds': 60, + } + + with ( + mock.patch.object(processing, '_notify'), + mock.patch.object( + processing, '_ffprobe_summary', return_value=fake_summary + ), + ): + with pytest.raises(processing.UnsupportedVideoCodecError) as excinfo: + processing._run_video_normalisation(asset) + + import shlex as _shlex + + recipe = excinfo.value.recipe + tokens = _shlex.split(recipe) + assert tokens[-1] == 'upload.h264.mp4' + assert 'libx264' in recipe + assert 'libx265' not in recipe @pytest_ffmpeg @@ -752,6 +853,36 @@ def test_pi3_64_hw_decode_set_is_h264_only( assert processing._hw_decoded_codecs() == frozenset({'h264'}) +# --------------------------------------------------------------------------- +# preferred_download_vcodec — per-board yt-dlp format_sort preference +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + 'device_type, expected', + [ + # Pi 5: HEVC is the hardware decode path; H.264 is software-only. + ('pi5', 'hevc'), + # All other boards: H.264 is the primary HW path and more available + # on YouTube. x86 HEVC via VAAPI is known-broken (black screen). + ('pi4-64', 'h264'), + ('pi3', 'h264'), + ('pi3-64', 'h264'), + ('x86', 'h264'), + ('rockpi4', 'h264'), + # Unknown board must not crash and must default to h264. + ('unrecognised-board-xyz', 'h264'), + ], +) +def test_preferred_download_vcodec( + device_type: str, + expected: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv('DEVICE_TYPE', device_type) + assert processing.preferred_download_vcodec() == expected + + @pytest.mark.parametrize( 'filename', [