diff --git a/.gitignore b/.gitignore index 3bac592..ae1f727 100644 --- a/.gitignore +++ b/.gitignore @@ -189,3 +189,6 @@ staticfiles/ # Prevent accidental publishing of private settings script_settings.plist config.json + +# Lock files +*.lock diff --git a/README.md b/README.md index a570606..fdc960f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # AutoPkg Runner [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg?style=for-the-badge)](LICENSE)
-![Version 3.0.1](https://img.shields.io/badge/version-3.0.1-green?style=for-the-badge) +![Version 3.0.2](https://img.shields.io/badge/version-3.0.2-green?style=for-the-badge) A web-based management interface for [AutoPkg](https://github.com/autopkg/autopkg) - the macOS software packaging automation tool. AutoPkg Runner wraps your AutoPkg workflows in a Django web application with real-time run monitoring, a REST API, a mobile PWA, and scheduled execution. @@ -10,7 +10,7 @@ A web-based management interface for [AutoPkg](https://github.com/autopkg/autopk ### Web UI - **Dashboard** - last run summary, 30-day success rate, next scheduled run, and a one-click manual trigger -- **Run detail** - GitHub Actions-style stage timeline with live log streaming; stage status icons and the log panel update in real time without a page refresh +- **Run detail** - GitHub Actions-style stage timeline with live log streaming over SSE; stage status icons and the log panel update in real time without a page refresh; multiple concurrent viewers are supported with no additional database load per viewer - **Run history** - paginated list of all pipeline executions with status badges and duration - **Run cancellation** - cancel an in-progress run from the run detail page - **Run sharing** - generate a shareable, unauthenticated link to any completed run report; optional expiry window configurable in notification settings @@ -166,7 +166,7 @@ Open `http://127.0.0.1:8000` and log in with the credentials shown. All configur | Command | Description | |-|-| | `autopkg-runner setup` | One-shot initialisation: migrate, create defaults, generate admin account | -| `autopkg-runner serve` | Start the server (`--bind`, `--port`, `--workers`, `--threads`) | +| `autopkg-runner serve` | Start the production server (`--bind`, `--port`, `--workers`) | | `autopkg-runner resetpassword ` | Generate and set a new random password for a user account | | `autopkg-runner generate_vapid_keys` | Generate VAPID keys for WebPush notifications and store them in the database | | `autopkg-runner install_sftp_deps` | Install macFUSE and sshfs via Homebrew (required for SFTP repository connections) | @@ -268,13 +268,21 @@ This installs macFUSE and sshfs via Homebrew. macFUSE requires a system reboot a ## Production deployment -The Django development server is single-threaded and will block on SSE connections. For production, use a multi-threaded WSGI server: +The Django development server (`manage.py serve`) is suitable for local testing but not for production. For production, AutoPkg Runner runs under **gunicorn with uvicorn workers** (ASGI), which provides: + +- Async SSE streaming — each client watching a live run is a lightweight coroutine rather than a blocked thread, so the server handles many concurrent viewers without exhausting resources +- A single DB-polling thread per active run regardless of how many clients are connected (fan-out broadcaster) ```bash -pip3 install gunicorn -gunicorn autopkgrunner.wsgi:application --workers 1 --threads 8 --bind 0.0.0.0:8000 +pip3 install gunicorn "uvicorn[standard]" +gunicorn autopkgrunner.asgi:application \ + --worker-class uvicorn.workers.UvicornWorker \ + --config python:autopkgrunner.gunicorn_conf \ + --workers 1 --bind 0.0.0.0:8000 ``` +> **Workers and the scheduler** — APScheduler runs inside each gunicorn worker. With more than one worker, only the worker that holds `scheduler.lock` fires scheduled jobs; the rest stand by and take over automatically if that worker exits. Use `--workers 1` unless you specifically need process-level redundancy. + Set `DJANGO_DEBUG=false` and `DJANGO_ALLOWED_HOSTS` to your server's hostname. Restrict permissions on the database file since it contains API tokens and repository credentials: ```bash @@ -316,7 +324,6 @@ A macOS authentication dialog will appear to request administrator credentials f | `--bind
` | `127.0.0.1` | Address the server binds to | | `--port ` | `8000` | Port the server listens on | | `--workers ` | `1` | Number of worker processes | -| `--threads ` | `8` | Number of threads per worker | ### Useful commands diff --git a/__info__.py b/__info__.py index 57b0b41..a65af5b 100644 --- a/__info__.py +++ b/__info__.py @@ -1,7 +1,7 @@ # Application Info APP_NAME = "autopkg-runner" FRIENDLY_APP_NAME = "AutoPkg Runner" -APP_VERSION_STR = "3.0.1" +APP_VERSION_STR = "3.0.2" APP_CHANNEL = "Production" BUNDLE_ID = "com.bytefloater.autopkg-runner" diff --git a/autopkg_runner.spec b/autopkg_runner.spec index f3e3c1d..e8efc05 100644 --- a/autopkg_runner.spec +++ b/autopkg_runner.spec @@ -63,6 +63,10 @@ hidden_imports = ( + collect_submodules('cryptography') + collect_submodules('zeroconf') + collect_submodules('gunicorn') + + collect_submodules('uvicorn') + + collect_submodules('h11') + + collect_submodules('httptools') + + collect_submodules('websockets') + collect_submodules('argon2') + collect_submodules('dns') + collect_submodules('webapp') diff --git a/autopkgrunner/asgi.py b/autopkgrunner/asgi.py index a8799c5..1951649 100644 --- a/autopkgrunner/asgi.py +++ b/autopkgrunner/asgi.py @@ -1,9 +1,70 @@ +import atexit +import fcntl +import logging import os +import shutil +import sys +import tempfile +import warnings from dotenv import load_dotenv load_dotenv() # Load .env before Django reads settings -from django.core.asgi import get_asgi_application +# Django 4.2 bug: StreamingHttpResponse.__aiter__ issues this warning whenever +# a sync iterator (e.g. WhiteNoise FileResponse, GZipMiddleware output) is +# async-iterated by the ASGI handler. The fallback path works correctly; the +# warning is spurious. Fixed in Django 5.x. +warnings.filterwarnings( + 'ignore', + message='StreamingHttpResponse must consume synchronous iterators', + category=Warning, +) os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'autopkgrunner.settings') + +# -- Static-file collection ---------------------------------------------------- +# Must happen before get_asgi_application() so WhiteNoise finds the files when +# it scans STATIC_ROOT during middleware initialisation. +# +# A file lock ensures only the first worker to boot runs collectstatic; the +# rest wait for the lock, see the files already exist, and skip it. +# This avoids the ObjC fork-safety crash that occurs when django.setup() is +# called in the gunicorn master (on_starting) before workers are forked. + +_via_manage = ( + sys.argv and os.path.basename(sys.argv[0]) in ('manage.py', 'manage') + or os.environ.get('AUTOPKG_MODE') == 'manage' +) + +if not _via_manage: + import django + django.setup() + + from django.conf import settings + from django.core.management import call_command + + logger = logging.getLogger('autopkg_runner') + _static_root = settings.STATIC_ROOT + + _lock_path = os.path.join(tempfile.gettempdir(), 'autopkg-runner-collectstatic.lock') + with open(_lock_path, 'w') as _lf: + fcntl.flock(_lf.fileno(), fcntl.LOCK_EX) + try: + if not _static_root.exists() or not any(_static_root.iterdir()): + logger.info('Collecting static files…') + call_command('collectstatic', '--noinput', verbosity=0) + _file_count = sum(1 for _ in _static_root.rglob('*') if _.is_file()) + logger.info('Static files ready (%d files in %s).', _file_count, _static_root) + + def _cleanup_static(): + if _static_root.exists(): + shutil.rmtree(_static_root, ignore_errors=True) + + atexit.register(_cleanup_static) + finally: + fcntl.flock(_lf.fileno(), fcntl.LOCK_UN) + +# -- ASGI application ---------------------------------------------------------- + +from django.core.asgi import get_asgi_application application = get_asgi_application() diff --git a/autopkgrunner/gunicorn_conf.py b/autopkgrunner/gunicorn_conf.py index c2e0412..07c84fc 100644 --- a/autopkgrunner/gunicorn_conf.py +++ b/autopkgrunner/gunicorn_conf.py @@ -3,6 +3,9 @@ Passed to gunicorn via --config python:autopkgrunner.gunicorn_conf. +on_starting runs once in the master process (pre-fork) and handles tasks that +must only happen once regardless of worker count: static file collection. + The post_fork hook starts background services (scheduler, stale-run cleanup, recipe cache warm-up) inside each worker after forking, keeping the master process completely thread-free. This prevents the macOS ObjC fork-safety @@ -14,6 +17,7 @@ """ + def post_fork(server, worker): """Called in each worker process immediately after it is forked.""" from django.apps import apps diff --git a/autopkgrunner/wsgi.py b/autopkgrunner/wsgi.py index 5d84e09..4eb2e94 100644 --- a/autopkgrunner/wsgi.py +++ b/autopkgrunner/wsgi.py @@ -1,55 +1,13 @@ -import atexit -import logging import os -import shutil -import sys from dotenv import load_dotenv load_dotenv() # Load .env before Django reads settings os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'autopkgrunner.settings') -# -- Static-file collection ---------------------------------------------------- -# Must happen *before* get_wsgi_application() so that WhiteNoise finds the -# files when it scans STATIC_ROOT during middleware initialisation. -# This also means files are ready before the first request is served. - -import django -django.setup() - -from django.conf import settings -from django.core.management import call_command - -logger = logging.getLogger('autopkg_runner') - -# Detect whether we were loaded by manage.py (dev server) vs a WSGI server -# (gunicorn). Must be set before the setup guard below. -_via_manage = ( - sys.argv and os.path.basename(sys.argv[0]) in ('manage.py', 'manage') - or os.environ.get('AUTOPKG_MODE') == 'manage' -) - -# Only collect static files when loaded directly by a WSGI server (gunicorn). -# The dev server serves static files itself, so collectstatic is not needed. -if not _via_manage: - _static_root = settings.STATIC_ROOT - - logger.info('Collecting static files…') - call_command('collectstatic', '--noinput', verbosity=0) - - _file_count = sum(1 for _ in _static_root.rglob('*') if _.is_file()) if _static_root.exists() else 0 - logger.info('Static files ready (%d files in %s).', _file_count, _static_root) - - def _cleanup_static(): - """Remove collected static files on process exit so the next startup - always starts from a clean slate (no stale renamed/deleted assets).""" - if _static_root.exists(): - shutil.rmtree(_static_root, ignore_errors=True) - - atexit.register(_cleanup_static) - -# -- WSGI application ---------------------------------------------------------- -# WhiteNoise middleware initialises here and scans the now-populated STATIC_ROOT. +# Static file collection is handled by gunicorn_conf.py on_starting (runs once +# in the master before workers fork). WhiteNoise finds the already-populated +# STATIC_ROOT when the WSGI application initialises here in each worker. from django.core.wsgi import get_wsgi_application application = get_wsgi_application() diff --git a/libs/orchestrator.py b/libs/orchestrator.py index ef91ab0..ed412a1 100644 --- a/libs/orchestrator.py +++ b/libs/orchestrator.py @@ -49,8 +49,12 @@ def configure_stages(self, override_stage_name): enabled_stage_names = [cls.__class__.__name__ for cls in self.stages] self.logger.info(f"Configured stages: {enabled_stage_names}") - def execute(self) -> bool: - """Run all configured stages. Returns True on full success, False otherwise.""" + def execute(self, cancel_flag=None) -> bool: + """Run all configured stages. Returns True on full success, False otherwise. + + cancel_flag is an optional threading.Event; when set the pipeline aborts + before the next stage and cleanup runs for all completed stages. + """ if not hasattr(self, 'stages'): self.logger.error('Stages not configured') return False @@ -68,6 +72,10 @@ def execute(self) -> bool: try: for stage in pipeline_stages: + if cancel_flag and cancel_flag.is_set(): + self.logger.info('Run cancelled — stopping pipeline before next stage.') + success = False + break current_stage = stage self._notify(stage.name, 'running') stage() diff --git a/libs/run_command.py b/libs/run_command.py index 2ce5d4b..6f5beb6 100644 --- a/libs/run_command.py +++ b/libs/run_command.py @@ -11,7 +11,12 @@ def info(self, msg: str, /) -> None: ... def error(self, msg: str, /) -> None: ... -def run_cmd(command: list[str], logger: _SupportsLogging): +def run_cmd(command: list[str], logger: _SupportsLogging, on_proc=None): + """Run *command*, streaming stdout/stderr to *logger*. + + on_proc: optional callback invoked with the Popen object immediately after + the process starts — used to register the process for external cancellation. + """ # For Python children, ensure unbuffered output; harmless for others. env = os.environ.copy() env.setdefault("PYTHONUNBUFFERED", "1") @@ -24,6 +29,8 @@ def run_cmd(command: list[str], logger: _SupportsLogging): bufsize=1, # line-buffered env=env, ) + if on_proc is not None: + on_proc(proc) # Read both pipes in the *calling* thread rather than spawning reader # threads. This is essential so that Logbook's thread-local handler diff --git a/requirements.txt b/requirements.txt index b79f693..5089271 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ psutil==7.0.0 zeroconf>=0.148.0 pywebpush>=2.0 gunicorn>=23.0.0 +uvicorn[standard]>=0.29 whitenoise>=6.0 pip-system-certs certifi diff --git a/run.py b/run.py index 19bf1ea..798e2a8 100644 --- a/run.py +++ b/run.py @@ -23,11 +23,10 @@ _COL = 30 # left column width for alignment _HELP_COMMANDS = [ - ('serve', 'Start the server (gunicorn)', [ + ('serve', 'Start the server (gunicorn + uvicorn)', [ ('--bind ADDRESS', 'Bind address (default: 0.0.0.0)'), ('--port PORT', 'Port (default: 8000)'), ('--workers N', 'Worker processes (default: 2)'), - ('--threads N', 'Threads per worker (default: 4)'), ]), ('setup', 'First-time initialisation', [ ('--username USER', 'Admin username (default: admin)'), @@ -44,7 +43,6 @@ ('--port PORT', 'Port (default: 8000)'), ('--bind ADDRESS', 'Bind address (default: 127.0.0.1)'), ('--workers N', 'Gunicorn worker processes (default: 1)'), - ('--threads N', 'Threads per worker (default: 8)'), ]), ('purge', 'Remove all app artefacts (plist, DB, logs)', [ ('--keep-data', 'Preserve the database and Application Support directories'), @@ -201,7 +199,6 @@ def main() -> None: p.add_argument('--bind', default='0.0.0.0', metavar='ADDRESS') p.add_argument('--port', default='8000', metavar='PORT') p.add_argument('--workers', default='2', metavar='N') - p.add_argument('--threads', default='4', metavar='N') opts = p.parse_args(sys.argv[2:]) class _NoWinch(logging.Filter): @@ -212,20 +209,20 @@ def filter(self, record): # runs. setup() replaces handlers but leaves logger-level filters intact, # so this survives into the master process and all forked workers. logging.getLogger('gunicorn.error').addFilter(_NoWinch()) - # Defence-in-depth for subprocess.run() calls from gthread worker threads: + # Defence-in-depth for subprocess.run() calls from worker threads: # if a worker thread has initialised ObjC and another thread calls # subprocess.run() (which does fork+exec), the subprocess child would # crash without this flag. The master→worker fork is safe without it # (the post_fork hook keeps the master thread-free), but subprocess calls - # from workers are not, so we keep it here. + # from within workers are not, so we keep it here. os.environ['OBJC_DISABLE_INITIALIZE_FORK_SAFETY'] = 'YES' from gunicorn.app.wsgiapp import run as gunicorn_run sys.argv = [ - sys.argv[0], 'autopkgrunner.wsgi:application', + sys.argv[0], 'autopkgrunner.asgi:application', '--config', 'python:autopkgrunner.gunicorn_conf', + '--worker-class', 'uvicorn.workers.UvicornWorker', '--bind', f'{opts.bind}:{opts.port}', '--workers', opts.workers, - '--threads', opts.threads, ] gunicorn_run() else: diff --git a/scripts/build.sh b/scripts/build.sh index 4fffedd..c9268dd 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -101,10 +101,24 @@ else codesign --sign - --deep --force "$APP" fi +read APP_NAME VERSION_STR < <(python3 -c " +import importlib.util, pathlib +p = pathlib.Path('$REPO_ROOT/__info__.py') +s = importlib.util.spec_from_file_location('__info__', p) +m = importlib.util.module_from_spec(s); s.loader.exec_module(m) +print(m.APP_NAME, m.APP_VERSION_STR) +") +ZIP_NAME="${APP_NAME}_${VERSION_STR}.zip" +ZIP_PATH="dist/$ZIP_NAME" +echo "==> Creating $ZIP_PATH..." +(cd dist && zip -r --symlinks "$ZIP_NAME" "AutoPkg Runner.app") +echo " Created $ZIP_PATH ($(du -sh "$ZIP_PATH" | cut -f1))" + echo "" echo "Build complete." echo " App bundle : $APP" echo " Binary : $BINARY" +echo " Archive : $ZIP_PATH" echo "" echo "First-run setup:" echo " 1. Grant Full Disk Access to the app in:" diff --git a/stages/notify.py b/stages/notify.py index 32dc1a3..4af8c9c 100644 --- a/stages/notify.py +++ b/stages/notify.py @@ -13,8 +13,8 @@ Available template variables ----------------------------- - {status} "succeeded" | "failed" - {status_emoji} "✅" | "❌" + {status} "succeeded" | "failed" | "cancelled" + {status_emoji} "✅" | "❌" | "🚫" {imports} count of Munki imports {failures} count of recipe failures {downloads} count of URL downloads + pkg copies @@ -198,9 +198,9 @@ def _build_summary(self, run_id) -> dict: def _build_template_context(self, summary: dict, run_id) -> dict: """Build the ``{variable}`` substitution dict for message templates.""" - # Infer pipeline success from stage executions - at this point in the - # pipeline all preceding stages have completed and their status is in DB. - succeeded = True + # Infer pipeline outcome from run status and stage executions. + # Default to "succeeded" when no run_id is available (no failures known). + run_status = "succeeded" duration_str = "" triggered_by = "" date_str = "" @@ -218,16 +218,19 @@ def _build_template_context(self, summary: dict, run_id) -> dict: mins, secs = divmod(total, 60) duration_str = f"{mins}m {secs}s" if mins else f"{secs}s" - # If any stage failed, the overall run is a failure. - succeeded = not StageExecution.objects.filter( - run_id=run_id, status="failed" - ).exists() + if run.status == "cancelled": + run_status = "cancelled" + elif StageExecution.objects.filter(run_id=run_id, status="failed").exists(): + run_status = "failed" + else: + run_status = "succeeded" except Exception: pass + _emoji = {"succeeded": "✅", "failed": "❌", "cancelled": "🚫"} return { - "status": "succeeded" if succeeded else "failed", - "status_emoji": "✅" if succeeded else "❌", + "status": run_status, + "status_emoji": _emoji.get(run_status, "❓"), "imports": summary["imports"], "failures": summary["failures"], "downloads": summary["downloads"], diff --git a/stages/run_autopkg.py b/stages/run_autopkg.py index dec4681..09b3634 100644 --- a/stages/run_autopkg.py +++ b/stages/run_autopkg.py @@ -39,7 +39,14 @@ def run(self) -> Optional[Any]: ) self._tmp_plist.close() + run_id = self.ctx.get('run_id') try: + from webapp.runner import register_active_proc, unregister_active_proc + + def _on_proc(proc): + if run_id: + register_active_proc(str(run_id), proc) + run_cmd([ str(self.autopkg_fpath), "run", @@ -49,9 +56,12 @@ def run(self) -> Optional[Any]: "-q", "-k", f"MUNKI_REPO={self.local_mnt}" - ], self.logger) + ], self.logger, on_proc=_on_proc) except subprocess.CalledProcessError as err: self.logger.error("Some recipes failed to execute: " + str(err)) + finally: + if run_id: + unregister_active_proc(str(run_id)) self._write_recipe_results() diff --git a/tests/test_runner.py b/tests/test_runner.py index cd12e3e..00e5c33 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -230,7 +230,7 @@ def test_mid_flight_cancel_preserved_in_finally(self): run = Run.objects.create(status='pending', config_snapshot={}) task = Task.objects.create(task_type='pipeline_run', status='pending', run=run) - def cancel_run_during_execution(): + def cancel_run_during_execution(cancel_flag=None): # Simulate user clicking Cancel while pipeline is running Run.objects.filter(id=run.id).update(status='cancelled') Task.objects.filter(id=task.id).update(status='cancelled') @@ -284,7 +284,7 @@ def fake_orchestrator_cls(**kwargs): # Capture the stage_callback so we can call it in execute() cb = kwargs.get('stage_callback') - def execute_with_callback(): + def execute_with_callback(cancel_flag=None): if cb: cb('UpdateRepos', 'running', datetime.now(timezone.utc)) cb('UpdateRepos', 'success', datetime.now(timezone.utc)) diff --git a/tests/views/test_views_runs.py b/tests/views/test_views_runs.py index 8cc7bc4..8b6cff1 100644 --- a/tests/views/test_views_runs.py +++ b/tests/views/test_views_runs.py @@ -209,6 +209,31 @@ def test_does_not_delete_pending_runs(self, run_manager_client, pending_run): assert Run.objects.filter(id=pending_run.id).exists() +def _collect_stream(resp): + """Consume a streaming response body — handles both sync and async generators.""" + from asgiref.sync import async_to_sync + sc = resp.streaming_content + if hasattr(sc, '__aiter__'): + async def _gather(): + return b''.join([c async for c in sc]) + return async_to_sync(_gather)() + return b''.join(sc) + + +class FakeBroadcaster: + """Minimal RunBroadcaster substitute that returns pre-built SSE frames.""" + + def __init__(self, frames=(), done=True): + self._frames = list(frames) + self._done = done + + def events_since(self, cursor): + raw = self._frames[cursor + 1:] + start = cursor + 1 + out = [f'id: {start + i}\n'.encode() + f for i, f in enumerate(raw)] + return out, self._done + + @pytest.mark.django_db class TestRunStream: def _url(self, run_id): @@ -219,109 +244,97 @@ def test_requires_login(self, anon_client, run): assert resp.status_code == 302 def test_returns_event_stream_content_type(self, run_manager_client, run): - resp = run_manager_client.get(self._url(run.id)) + fake = FakeBroadcaster() + with patch('webapp.run_broadcaster.broadcaster_manager') as m: + m.get.return_value = fake + resp = run_manager_client.get(self._url(run.id)) assert resp.status_code == 200 assert 'text/event-stream' in resp.get('Content-Type', '') def test_stream_emits_complete_event_for_finished_run(self, run_manager_client, run): - """Consuming the generator for a completed run yields a 'complete' event.""" - resp = run_manager_client.get(self._url(run.id)) - chunks = b''.join(resp.streaming_content) + """Broadcaster frames containing a complete event are passed through.""" + complete = f'data: {json.dumps({"type": "complete", "status": run.status})}\n\n'.encode() + done_frame = b'event: done\ndata: {}\n\n' + fake = FakeBroadcaster(frames=[complete, done_frame]) + with patch('webapp.run_broadcaster.broadcaster_manager') as m: + m.get.return_value = fake + resp = run_manager_client.get(self._url(run.id)) + chunks = _collect_stream(resp) assert b'complete' in chunks assert run.status.encode() in chunks def test_stream_emits_log_entries(self, run_manager_client, run): - """Log entries are included in the SSE stream.""" - from datetime import datetime, timezone - from webapp.models import LogEntry - LogEntry.objects.create( - run=run, - level='INFO', - message='Test log message', - stage_name='', - timestamp=datetime.now(timezone.utc), - ) - resp = run_manager_client.get(self._url(run.id)) - chunks = b''.join(resp.streaming_content) + """Log frames from the broadcaster are forwarded to the client.""" + log_frame = f'data: {json.dumps({"type": "log", "level": "INFO", "stage": "", "message": "Test log message", "timestamp": "2024-01-01T00:00:00+00:00"})}\n\n'.encode() + complete = b'data: {"type": "complete", "status": "success"}\n\n' + done_frame = b'event: done\ndata: {}\n\n' + fake = FakeBroadcaster(frames=[log_frame, complete, done_frame]) + with patch('webapp.run_broadcaster.broadcaster_manager') as m: + m.get.return_value = fake + resp = run_manager_client.get(self._url(run.id)) + chunks = _collect_stream(resp) assert b'Test log message' in chunks def test_stream_emits_stage_updates(self, run_manager_client, run): - """Stage execution records are included in the SSE stream.""" - from webapp.models import StageExecution - from datetime import datetime, timezone - StageExecution.objects.create( - run=run, - name='UpdateRepos', - status='success', - order=0, - started_at=datetime.now(timezone.utc), - completed_at=datetime.now(timezone.utc), - ) - resp = run_manager_client.get(self._url(run.id)) - chunks = b''.join(resp.streaming_content) + """Stage frames from the broadcaster are forwarded to the client.""" + stage_frame = f'data: {json.dumps({"type": "stage", "name": "UpdateRepos", "status": "success", "order": 0, "started_at": None, "completed_at": None})}\n\n'.encode() + complete = b'data: {"type": "complete", "status": "success"}\n\n' + done_frame = b'event: done\ndata: {}\n\n' + fake = FakeBroadcaster(frames=[stage_frame, complete, done_frame]) + with patch('webapp.run_broadcaster.broadcaster_manager') as m: + m.get.return_value = fake + resp = run_manager_client.get(self._url(run.id)) + chunks = _collect_stream(resp) assert b'UpdateRepos' in chunks - def test_stream_sends_error_for_nonexistent_run(self, run_manager_client): - """A nonexistent run_id causes an error event to be emitted.""" - missing_id = uuid.uuid4() - resp = run_manager_client.get(self._url(missing_id)) - chunks = b''.join(resp.streaming_content) - assert b'error' in chunks or b'not found' in chunks + def test_stream_returns_404_for_nonexistent_run(self, run_manager_client): + """A nonexistent run_id returns 404 before the stream is opened.""" + resp = run_manager_client.get(self._url(uuid.uuid4())) + assert resp.status_code == 404 - def test_stream_respects_from_param(self, run_manager_client, run): - """?from= skips log entries with id <= that value.""" - from datetime import datetime, timezone - from webapp.models import LogEntry - entry = LogEntry.objects.create( - run=run, - level='DEBUG', - message='older message', - stage_name='', - timestamp=datetime.now(timezone.utc), - ) - # Request with from=entry.id → should skip the entry above - resp = run_manager_client.get(self._url(run.id) + f'?from={entry.id}') - chunks = b''.join(resp.streaming_content) - assert b'older message' not in chunks + def test_stream_respects_last_event_id(self, run_manager_client, run): + """Last-Event-ID header sets the cursor so already-seen frames are skipped.""" + older = b'data: {"type": "log", "message": "older"}\n\n' + newer = b'data: {"type": "log", "message": "newer"}\n\n' + complete = b'data: {"type": "complete", "status": "success"}\n\n' + done_frame = b'event: done\ndata: {}\n\n' + # older is at index 0; sending Last-Event-ID: 0 means cursor starts at 0 + # so events_since(0) returns only [newer, complete, done_frame] + fake = FakeBroadcaster(frames=[older, newer, complete, done_frame]) + with patch('webapp.run_broadcaster.broadcaster_manager') as m: + m.get.return_value = fake + resp = run_manager_client.get(self._url(run.id), HTTP_LAST_EVENT_ID='0') + chunks = _collect_stream(resp) + assert b'older' not in chunks + assert b'newer' in chunks def test_stream_no_cache_control_header(self, run_manager_client, run): - resp = run_manager_client.get(self._url(run.id)) + fake = FakeBroadcaster() + with patch('webapp.run_broadcaster.broadcaster_manager') as m: + m.get.return_value = fake + resp = run_manager_client.get(self._url(run.id)) assert resp.get('Cache-Control') == 'no-cache' - def test_stream_polls_until_run_completes(self, run_manager_client, db): - """Stream loop sleeps once for a running run, then emits complete after status changes.""" - from webapp.models import Run, LogEntry, StageExecution - now = datetime.now(timezone.utc) + def test_stream_polls_until_run_completes(self, run_manager_client, run): + """Generator keeps polling the broadcaster until done=True.""" + complete = b'data: {"type": "complete", "status": "success"}\n\n' + done_frame = b'event: done\ndata: {}\n\n' - # Build two fake run objects: first 'running', then 'success' - def make_fake_run(status): - r = MagicMock() - r.status = status - r.id = uuid.uuid4() - return r - - running_run = make_fake_run('running') - success_run = make_fake_run('success') call_count = {'n': 0} - - def fake_run_filter(**kwargs): - qs = MagicMock() - call_count['n'] += 1 - qs.first.return_value = running_run if call_count['n'] == 1 else success_run - return qs - - empty_qs = MagicMock() - empty_qs.__iter__ = MagicMock(return_value=iter([])) - empty_qs.filter.return_value = empty_qs - empty_qs.order_by.return_value = iter([]) - - with patch('webapp.models.Run.objects') as mock_run_mgr, \ - patch('webapp.models.LogEntry.objects') as mock_log_mgr, \ - patch('webapp.models.StageExecution.objects') as mock_stage_mgr, \ - patch('time.sleep'): - mock_run_mgr.filter.side_effect = fake_run_filter - mock_log_mgr.filter.return_value.order_by.return_value = iter([]) - mock_stage_mgr.filter.return_value.__iter__ = MagicMock(return_value=iter([])) - resp = run_manager_client.get(self._url(running_run.id)) - chunks = b''.join(resp.streaming_content) + frames = [complete, done_frame] + + class SlowBroadcaster: + def events_since(self, cursor): + call_count['n'] += 1 + done = call_count['n'] >= 3 + raw = frames[cursor + 1:] if done else [] + start = cursor + 1 + out = [f'id: {start + i}\n'.encode() + f for i, f in enumerate(raw)] + return out, done + + with patch('webapp.run_broadcaster.broadcaster_manager') as m: + m.get.return_value = SlowBroadcaster() + resp = run_manager_client.get(self._url(run.id)) + chunks = _collect_stream(resp) assert b'complete' in chunks + assert call_count['n'] >= 3 diff --git a/webapp/apps.py b/webapp/apps.py index 314a984..ea56013 100644 --- a/webapp/apps.py +++ b/webapp/apps.py @@ -100,12 +100,34 @@ def _start_services_in_worker(self): the scheduler lock starts APScheduler, preventing duplicate scheduled runs when workers > 1. """ + import fcntl + import time as _time + import tempfile + import os as _os from webapp.scheduler import acquire_scheduler_lock, start_scheduler from webapp.views.recipes import _start_cache_build - from webapp.recipe_index import ensure_fresh as index_ensure_fresh + from webapp.recipe_index import _fetch as _index_fetch self._mark_interrupted_runs() _start_cache_build() - index_ensure_fresh() + # Only the first worker to boot fetches the recipe index on startup. + # ensure_fresh() returns immediately (fires a daemon thread), so we call + # _fetch() directly while holding the lock so the sentinel is only written + # after the fetch actually completes. + _lock_path = _os.path.join(tempfile.gettempdir(), 'autopkg-runner-index-warm.lock') + _sentinel_path = _os.path.join(tempfile.gettempdir(), 'autopkg-runner-index-warm.done') + with open(_lock_path, 'w') as _lf: + fcntl.flock(_lf.fileno(), fcntl.LOCK_EX) + try: + try: + age = _time.time() - _os.path.getmtime(_sentinel_path) + except OSError: + age = float('inf') + if age > 300: + _index_fetch() + with open(_sentinel_path, 'w') as _sf: + pass + finally: + fcntl.flock(_lf.fileno(), fcntl.LOCK_UN) if acquire_scheduler_lock(): start_scheduler() diff --git a/webapp/management/commands/service_daemon.py b/webapp/management/commands/service_daemon.py index b81a06f..6cfa8fd 100644 --- a/webapp/management/commands/service_daemon.py +++ b/webapp/management/commands/service_daemon.py @@ -80,12 +80,6 @@ def add_arguments(self, parser): 'Increasing this will cause duplicate scheduled runs.' ), ) - parser.add_argument( - '--threads', - default='8', - metavar='N', - help='Number of threads per gunicorn worker (default: 8).', - ) # ------------------------------------------------------------------ @@ -106,12 +100,10 @@ def _do_install(self, options): bind = options['bind'] port = options['port'] workers = options['workers'] - threads = options['threads'] self._validate_user(user) self._validate_port(port) self._validate_workers(workers) - self._validate_threads(threads) self._validate_project_location(user) self._validate_project_ownership(user) @@ -121,16 +113,16 @@ def _do_install(self, options): # --config python:autopkgrunner.gunicorn_conf) transparently. prog_args = [sys.executable, 'serve', '--bind', bind, '--port', port, - '--workers', workers, '--threads', threads] + '--workers', workers] else: gunicorn_prefix = self._find_gunicorn() prog_args = gunicorn_prefix + [ '--chdir', str(PROJECT_ROOT), '--config', 'python:autopkgrunner.gunicorn_conf', - 'autopkgrunner.wsgi:application', + '--worker-class', 'uvicorn.workers.UvicornWorker', + 'autopkgrunner.asgi:application', '--bind', f'{bind}:{port}', '--workers', str(workers), - '--threads', str(threads), '--timeout', '120', ] @@ -143,7 +135,6 @@ def _do_install(self, options): self.stdout.write(f' ├─ binary : {prog_args[0]}') self.stdout.write(f' ├─ bind : {bind}:{port}') self.stdout.write(f' ├─ workers : {workers}') - self.stdout.write(f' ├─ threads : {threads}') self.stdout.write(f' └─ plist : {PLIST_DEST}') self.stdout.write('') @@ -314,7 +305,7 @@ def _remove_escalated(self): f"done", '', "# Force-kill any orphaned gunicorn workers that bootout didn't reach.", - "pkill -KILL -f 'autopkgrunner.wsgi:application' 2>/dev/null || true", + "pkill -KILL -f 'autopkgrunner.asgi:application' 2>/dev/null || true", '', f"rm -f '{PLIST_DEST}'", ]) @@ -371,11 +362,11 @@ def _kill_stale_workers(self) -> None: launchctl bootout only kills processes it is currently tracking. Legacy-API installs (launchctl load) left processes untracked, so bootout removes the registration without sending any signal. We - target processes by the wsgi app argument, which is unique to the + target processes by the asgi app argument, which is unique to the server and absent from management command processes. """ result = subprocess.run( - ['pkill', '-KILL', '-f', 'autopkgrunner.wsgi:application'], + ['pkill', '-KILL', '-f', 'autopkgrunner.asgi:application'], capture_output=True, ) if result.returncode == 0: @@ -443,16 +434,6 @@ def _validate_workers(self, workers: str): ' disabled the built-in scheduler.\n' )) - def _validate_threads(self, threads: str): - try: - n = int(threads) - if n < 1: - raise ValueError - except ValueError: - raise CommandError( - f"Invalid --threads value '{threads}'. Must be a positive integer." - ) - def _validate_project_location(self, user: str): """Verify the project root won't be blocked by macOS TCC at runtime. diff --git a/webapp/middleware.py b/webapp/middleware.py index 9348646..3fcde02 100644 --- a/webapp/middleware.py +++ b/webapp/middleware.py @@ -1,6 +1,7 @@ import logging import re +from asgiref.sync import iscoroutinefunction, markcoroutinefunction from django.db.utils import OperationalError from django.http import HttpResponse @@ -98,10 +99,17 @@ class DatabaseWriteGuardMiddleware: caught here rather than surfacing as a Django debug traceback. """ + async_capable = True + sync_capable = True + def __init__(self, get_response): self.get_response = get_response + if iscoroutinefunction(get_response): + markcoroutinefunction(self) def __call__(self, request): + if iscoroutinefunction(self.get_response): + return self._async_call(request) try: return self.get_response(request) except OperationalError as exc: @@ -114,6 +122,19 @@ def __call__(self, request): content_type='text/html; charset=utf-8') raise + async def _async_call(self, request): + try: + return await self.get_response(request) + except OperationalError as exc: + if _is_readonly_db_error(exc): + logger.error( + 'Database is read-only — returning 503. path=%s exc=%s', + request.path, exc, + ) + return HttpResponse(_DB_READONLY_HTML, status=503, + content_type='text/html; charset=utf-8') + raise + # --------------------------------------------------------------------------- @@ -124,43 +145,68 @@ def __call__(self, request): class RemoveServerHeaderMiddleware: + async_capable = True + sync_capable = True + def __init__(self, get_response): self.get_response = get_response + if iscoroutinefunction(get_response): + markcoroutinefunction(self) def __call__(self, request): + if iscoroutinefunction(self.get_response): + return self._async_call(request) response = self.get_response(request) response.headers.pop('Server', None) return response + async def _async_call(self, request): + response = await self.get_response(request) + response.headers.pop('Server', None) + return response + class MobileDetectionMiddleware: + async_capable = True + sync_capable = True + def __init__(self, get_response): self.get_response = get_response + if iscoroutinefunction(get_response): + markcoroutinefunction(self) - def __call__(self, request): + def _set_request_flag(self, request): ua = request.META.get('HTTP_USER_AGENT', '') force_desktop = ( request.COOKIES.get('desktop_mode') == '1' or request.GET.get('desktop') == '1' ) - # iPadOS 13+ sends a macOS-style UA so the regex above misses it. # base_desktop.html detects the touch fingerprint in JS and sets this # cookie, which we pick up on the next (reloaded) request. ipad_detected = request.COOKIES.get('ipad_detected') == '1' - request.is_mobile = ( bool(_MOBILE_UA_RE.search(ua)) or ipad_detected ) and not force_desktop - response = self.get_response(request) - + def _set_response_cookies(self, request, response): if request.GET.get('desktop') == '1': response.set_cookie('desktop_mode', '1', max_age=86400 * 365) response.delete_cookie('ipad_detected', path='/') - if request.GET.get('mobile') == '1': response.set_cookie('ipad_detected', '1', max_age=86400 * 365) response.delete_cookie('desktop_mode', path='/') + def __call__(self, request): + if iscoroutinefunction(self.get_response): + return self._async_call(request) + self._set_request_flag(request) + response = self.get_response(request) + self._set_response_cookies(request, response) + return response + + async def _async_call(self, request): + self._set_request_flag(request) + response = await self.get_response(request) + self._set_response_cookies(request, response) return response diff --git a/webapp/run_broadcaster.py b/webapp/run_broadcaster.py new file mode 100644 index 0000000..4b45442 --- /dev/null +++ b/webapp/run_broadcaster.py @@ -0,0 +1,243 @@ +"""In-process SSE fan-out broadcaster. + +One RunBroadcaster is created per active run. A single daemon thread polls +the database and appends serialised SSE event frames to an in-memory list. +Async SSE generators in run_stream read from that list using a cursor +(the list index of the last event they have seen), so the database is +queried exactly once per second per run regardless of how many clients +are watching. + +The manager expires finished broadcasters after a TTL so late-connecting +clients can still receive the full event history for a recently completed run. +""" +from __future__ import annotations + +import json +import logging +import threading +import time + +from django.db import close_old_connections + +logger = logging.getLogger('autopkg_runner') + +_DONE_TTL = 300 # seconds to keep a finished broadcaster alive + + +class RunBroadcaster: + """Polls the database for one run and caches events for all subscribers.""" + + def __init__(self, run_id: str): + self.run_id = run_id + self._events: list[bytes] = [] # pre-rendered SSE frames without id: line + self._lock = threading.Lock() + self._done = False + self._done_at: float | None = None + self._thread = threading.Thread( + target=self._poll_loop, + daemon=True, + name=f'run-broadcaster-{run_id}', + ) + self._thread.start() + + # ------------------------------------------------------------------ + # Public API (called from async SSE generators) + # ------------------------------------------------------------------ + + def events_since(self, cursor: int) -> tuple[list[bytes], bool]: + """Return (new_frames, is_done). + + cursor is the index of the last event the caller has consumed. + Pass -1 to receive all events from the beginning. + Frames are returned with their SSE ``id:`` line prepended. + """ + with self._lock: + raw = self._events[cursor + 1:] + start = cursor + 1 + frames = [ + f'id: {start + i}\n'.encode() + frame + for i, frame in enumerate(raw) + ] + return frames, self._done + + @property + def is_expired(self) -> bool: + if not self._done or self._done_at is None: + return False + return (time.monotonic() - self._done_at) > _DONE_TTL + + # ------------------------------------------------------------------ + # Internal poll loop (runs in daemon thread) + # ------------------------------------------------------------------ + + def _poll_loop(self): + try: + self._run() + except Exception: + logger.exception('RunBroadcaster error for run %s', self.run_id) + finally: + close_old_connections() + with self._lock: + self._done = True + self._done_at = time.monotonic() + + def _run(self): + from webapp.models import LogEntry, Run, StageExecution + + last_log_id = 0 + last_stage_data: dict[str, str] = {} + + while True: + close_old_connections() + + run = Run.objects.filter(id=self.run_id).first() + if not run: + break + + new_frames: list[bytes] = [] + + entries = LogEntry.objects.filter( + run_id=self.run_id, id__gt=last_log_id + ).order_by('id') + for entry in entries: + payload = json.dumps({ + 'type': 'log', + 'id': entry.id, + 'level': entry.level, + 'stage': entry.stage_name, + 'message': entry.message, + 'timestamp': entry.timestamp.isoformat(), + }) + new_frames.append(f'data: {payload}\n\n'.encode()) + last_log_id = entry.id + + for stage in StageExecution.objects.filter(run_id=self.run_id): + key = f'{stage.name}:{stage.status}' + if last_stage_data.get(stage.name) != key: + last_stage_data[stage.name] = key + payload = json.dumps({ + 'type': 'stage', + 'name': stage.name, + 'status': stage.status, + 'order': stage.order, + 'started_at': stage.started_at.isoformat() if stage.started_at else None, + 'completed_at': stage.completed_at.isoformat() if stage.completed_at else None, + }) + new_frames.append(f'data: {payload}\n\n'.encode()) + + if new_frames: + with self._lock: + self._events.extend(new_frames) + + if run.status in ('success', 'failed', 'cancelled'): + # Wait briefly then do one final pass to collect any log entries + # or stage updates that were written to the DB slightly after the + # run status was set (common on fast-failing runs). + time.sleep(0.3) + close_old_connections() + for entry in LogEntry.objects.filter( + run_id=self.run_id, id__gt=last_log_id + ).order_by('id'): + payload = json.dumps({ + 'type': 'log', + 'level': entry.level, + 'stage': entry.stage_name, + 'message': entry.message, + 'timestamp': entry.timestamp.isoformat(), + }) + new_frames.append(f'data: {payload}\n\n'.encode()) + last_log_id = entry.id + + for stage in StageExecution.objects.filter(run_id=self.run_id): + key = f'{stage.name}:{stage.status}' + if last_stage_data.get(stage.name) != key: + last_stage_data[stage.name] = key + payload = json.dumps({ + 'type': 'stage', + 'name': stage.name, + 'status': stage.status, + 'order': stage.order, + 'started_at': stage.started_at.isoformat() if stage.started_at else None, + 'completed_at': stage.completed_at.isoformat() if stage.completed_at else None, + }) + new_frames.append(f'data: {payload}\n\n'.encode()) + + payload = json.dumps({'type': 'complete', 'status': run.status}) + terminal_frames = [ + f'data: {payload}\n\n'.encode(), + b'event: done\ndata: {}\n\n', + ] + with self._lock: + self._events.extend(new_frames) + self._events.extend(terminal_frames) + run_list_broadcaster.notify() + break + + time.sleep(1) + + +# --------------------------------------------------------------------------- +# Manager singleton +# --------------------------------------------------------------------------- + +class _BroadcasterManager: + def __init__(self): + self._broadcasters: dict[str, RunBroadcaster] = {} + self._lock = threading.Lock() + + def get(self, run_id: str) -> RunBroadcaster: + """Return an existing broadcaster or create one for *run_id*.""" + run_id = str(run_id) + with self._lock: + self._expire() + if run_id not in self._broadcasters: + self._broadcasters[run_id] = RunBroadcaster(run_id) + return self._broadcasters[run_id] + + def _expire(self): + """Remove broadcasters whose TTL has elapsed (called under lock).""" + stale = [rid for rid, b in self._broadcasters.items() if b.is_expired] + for rid in stale: + del self._broadcasters[rid] + + +broadcaster_manager = _BroadcasterManager() + + +# --------------------------------------------------------------------------- +# Global run-list broadcaster +# --------------------------------------------------------------------------- +# Signals the list page whenever any run reaches a terminal state. +# Subscribers long-poll via run_list_stream; on receiving an event they +# trigger an HTMX refresh of the run list. + +class RunListBroadcaster: + """Simple monotonic-counter broadcaster for run-list change notifications.""" + + def __init__(self): + self._seq = 0 + self._lock = threading.Lock() + self._event = threading.Event() + + def notify(self) -> None: + """Called when any run changes to a terminal state.""" + with self._lock: + self._seq += 1 + self._event.set() + + def wait_for_change(self, last_seq: int, timeout: float = 30.0) -> int: + """Block until seq > last_seq (or timeout). Returns current seq.""" + deadline = time.monotonic() + timeout + while True: + with self._lock: + if self._seq > last_seq: + return self._seq + remaining = deadline - time.monotonic() + if remaining <= 0: + with self._lock: + return self._seq + self._event.wait(timeout=min(remaining, 1.0)) + self._event.clear() + + +run_list_broadcaster = RunListBroadcaster() diff --git a/webapp/runner.py b/webapp/runner.py index 8ccf3a6..3a3ec76 100644 --- a/webapp/runner.py +++ b/webapp/runner.py @@ -1,4 +1,5 @@ """Background pipeline execution. Implemented fully in Phase 4.""" +import subprocess as _subprocess import threading import uuid as _uuid from datetime import datetime, timezone @@ -8,6 +9,53 @@ class RunAlreadyRunningError(Exception): """Raised when a run is already in progress and a new trigger is attempted.""" +# --------------------------------------------------------------------------- +# Cancellation registry +# --------------------------------------------------------------------------- +# Maps run_id (str) → (cancel_event, active_subprocess | None). +# Used by cancel_run() to signal the pipeline thread to abort cleanly and +# to SIGTERM any running child process (e.g. autopkg) immediately. + +_cancel_lock = threading.Lock() +_cancel_events: dict[str, threading.Event] = {} +_active_procs: dict[str, _subprocess.Popen] = {} + + +def _register_run(run_id: str, event: threading.Event) -> None: + with _cancel_lock: + _cancel_events[str(run_id)] = event + + +def _unregister_run(run_id: str) -> None: + with _cancel_lock: + _cancel_events.pop(str(run_id), None) + _active_procs.pop(str(run_id), None) + + +def register_active_proc(run_id: str, proc: _subprocess.Popen) -> None: + """Called by RunAutoPkg when a child process starts.""" + with _cancel_lock: + _active_procs[str(run_id)] = proc + + +def unregister_active_proc(run_id: str) -> None: + """Called by RunAutoPkg when a child process finishes.""" + with _cancel_lock: + _active_procs.pop(str(run_id), None) + + +def cancel_run(run_id: str) -> None: + """Signal the pipeline thread for *run_id* to abort and kill any running subprocess.""" + run_id = str(run_id) + with _cancel_lock: + event = _cancel_events.get(run_id) + proc = _active_procs.get(run_id) + if event: + event.set() + if proc and proc.poll() is None: + proc.terminate() + + def trigger_manual_run(triggered_by: str = 'manual') -> _uuid.UUID: from webapp.models import Run, Task from libs.config import config_from_settings, pipeline_config_to_dict @@ -65,6 +113,9 @@ def _execute_run(run_id: _uuid.UUID, task_id: _uuid.UUID): # it stuck in 'pending' forever. from webapp.models import Run, Task, StageExecution + cancel_event = threading.Event() + _register_run(str(run_id), cancel_event) + final_status = 'failed' db_handler = None @@ -118,7 +169,7 @@ def stage_callback(stage_name: str, status: str, timestamp: datetime): ) orchestrator.ctx = ctx orchestrator.configure_stages(override_stage_name=None) - success = orchestrator.execute() + success = orchestrator.execute(cancel_flag=cancel_event) final_status = 'success' if success else 'failed' except Exception: # Log if we got far enough to have a logger/handler; otherwise the @@ -130,6 +181,7 @@ def stage_callback(stage_name: str, status: str, timestamp: datetime): pass final_status = 'failed' finally: + _unregister_run(str(run_id)) completed_at = datetime.now(timezone.utc) # Only update if the run hasn't been cancelled from outside # (e.g. the user hit "Cancel" in the UI while the pipeline was running). diff --git a/webapp/templates/webapp/mobile/runs/detail.html b/webapp/templates/webapp/mobile/runs/detail.html index e285e9f..55afa6d 100644 --- a/webapp/templates/webapp/mobile/runs/detail.html +++ b/webapp/templates/webapp/mobile/runs/detail.html @@ -324,6 +324,7 @@ runComplete: ['success', 'failed', 'cancelled'].includes(initialStatus), liveEntries: [], eventSource: null, + _sseComplete: false, init() { if (!this.runComplete) { @@ -337,7 +338,8 @@ }, startSSE(runId) { - this.eventSource = new EventSource(`/runs/${runId}/stream/?from=${lastLogId}`); + this._sseComplete = false; + this.eventSource = new EventSource(`/runs/${runId}/stream/`); this.eventSource.onmessage = (e) => { const data = JSON.parse(e.data); @@ -345,6 +347,7 @@ if (data.type === 'log') { // Guard: ignore entries delivered by a reconnect after completion if (this.runComplete) return; + if (data.id && data.id <= lastLogId) return; const t = new Date(data.timestamp).toLocaleTimeString('en-GB', { timeZone: '{{ local_tz }}', hour12: false }); this.liveEntries.push({ time: t, level: data.level, stage: data.stage, message: data.message }); this.$nextTick(() => { @@ -356,6 +359,7 @@ this.handleStageUpdate(data); } else if (data.type === 'complete') { + this._sseComplete = true; this.currentStatus = data.status; this.runComplete = true; this.updateStatusBadge(data.status); @@ -366,8 +370,15 @@ } }; - this.eventSource.addEventListener('done', () => { this._closeSSE(); }); - this.eventSource.onerror = () => { this._closeSSE(); }; + this.eventSource.addEventListener('done', () => { + this._sseComplete = true; + this._closeSSE(); + }); + this.eventSource.onerror = () => { + if (this._sseComplete) return; + this._closeSSE(); + this._fetchRunStatus(runId); + }; }, // Null out all handlers then close - prevents the browser auto-reconnect @@ -381,6 +392,23 @@ } }, + _fetchRunStatus(runId) { + fetch(`/runs/${runId}/status/`) + .then(r => r.ok ? r.json() : null) + .then(data => { + if (!data) return; + const terminal = ['success', 'failed', 'cancelled']; + if (terminal.includes(data.status)) { + this.currentStatus = data.status; + this.runComplete = true; + this.updateStatusBadge(data.status); + } else { + this.startSSE(runId); + } + }) + .catch(() => {}); + }, + // Handle a stage SSE event - creates the row if it doesn't exist yet, // then updates the icon and (if completed) the duration. handleStageUpdate(data) { diff --git a/webapp/templates/webapp/mobile/runs/list.html b/webapp/templates/webapp/mobile/runs/list.html index be16f15..37c31de 100644 --- a/webapp/templates/webapp/mobile/runs/list.html +++ b/webapp/templates/webapp/mobile/runs/list.html @@ -64,7 +64,7 @@
@@ -101,11 +101,12 @@ hx-post="{% url 'run-cancel' run.id %}" hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}' hx-swap="none" + hx-on::before-request="this.querySelector('button[type=submit]').disabled=true" hx-on::after-request="if(event.detail.successful){ if(window._pwaNavigate) _pwaNavigate('{% url 'run-list' %}', true); else location.reload(); }" @click.stop> {% csrf_token %} @@ -181,3 +182,24 @@
{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/webapp/templates/webapp/recipes/override_editor.html b/webapp/templates/webapp/recipes/override_editor.html index 4534628..ead28a9 100644 --- a/webapp/templates/webapp/recipes/override_editor.html +++ b/webapp/templates/webapp/recipes/override_editor.html @@ -94,7 +94,7 @@

{{ t.RECIPES_VIE +{% endblock %} diff --git a/webapp/translations/en-US.json b/webapp/translations/en-US.json index 2a5e6fe..608b4c6 100644 --- a/webapp/translations/en-US.json +++ b/webapp/translations/en-US.json @@ -9,6 +9,7 @@ "STATE_LIVE": "Live", "STATE_PENDING": "Pending", "STATE_CANCELLED": "Cancelled", + "STATE_CANCELLING": "Cancelling…", "STATE_IN_PROGRESS": "In Progress", "STATE_SKIPPED": "Skipped", "STATE_DONE": "Done", @@ -477,6 +478,8 @@ "FIND_LOADING": "Fetching recipe index…", "FIND_EMPTY": "No recipes match your search.", "FIND_EMPTY_QUERY": "Enter a search term above to find recipes.", + "FIND_INDEX_BUILDING": "Building recipe index…", + "FIND_INDEX_BUILDING_SUB": "This only happens once after the server starts.", "FIND_ERROR": "Could not fetch the recipe index.", "FIND_REFRESH_SUCCESS": "Recipe index refreshed successfully.", "FIND_TOTAL": "results", diff --git a/webapp/urls.py b/webapp/urls.py index 39cdced..2264db9 100644 --- a/webapp/urls.py +++ b/webapp/urls.py @@ -27,8 +27,12 @@ runs.RunCancelView.as_view(), name='run-cancel'), path('runs//', runs.RunDetailView.as_view(), name='run-detail'), + path('runs//status/', + runs.run_status, name='run-status'), path('runs//stream/', runs.run_stream, name='run-stream'), + path('runs/stream/', + runs.run_list_stream, name='run-list-stream'), path('config/schedule/', schedule.ScheduleView.as_view(), name='schedule'), path('config/about/', diff --git a/webapp/views/runs.py b/webapp/views/runs.py index 0eb43b2..1175f1e 100644 --- a/webapp/views/runs.py +++ b/webapp/views/runs.py @@ -1,6 +1,7 @@ import os import plistlib import ssl +import threading import time import urllib.request @@ -9,13 +10,28 @@ from django.contrib.auth.mixins import LoginRequiredMixin from django.core.paginator import Paginator from django.db import close_old_connections -from django.http import StreamingHttpResponse, JsonResponse +from django.http import StreamingHttpResponse, JsonResponse, HttpResponse + + +class _AsyncStreamingHttpResponse(StreamingHttpResponse): + """Work around a Django 4.2 bug where StreamingHttpResponse.streaming_content + synchronously consumes async generators via async_to_sync even when is_async=True, + causing a spurious warning and defeating the async streaming. Overriding + __aiter__ to iterate _iterator directly bypasses the broken getter.""" + + async def __aiter__(self): + async for chunk in self._iterator: + yield self.make_bytes(chunk) from django.shortcuts import get_object_or_404, redirect from django.views import View from django.views.generic import TemplateView _MUNKI_CATALOG_CACHE: dict = {'data': None, 'url': '', 'auth': '', 'catalog': '', 'ts': 0.0} +# Limit concurrent icon proxy requests to prevent exhausting the sync thread pool +# that Django uses for non-async views under ASGI. +_ICON_PROXY_SEMAPHORE = threading.Semaphore(3) + def _make_auth_header(username: str, password: str) -> str: if not username: @@ -212,72 +228,145 @@ def post(self, request, run_id): run.status = 'cancelled' run.completed_at = now run.save(update_fields=['status', 'completed_at']) + # Signal the pipeline thread to stop and kill any running subprocess. + from webapp.runner import cancel_run + cancel_run(run_id) if request.headers.get('HX-Request'): from django.http import HttpResponse - return HttpResponse(status=204) + resp = HttpResponse(status=204) + resp['HX-Trigger'] = 'runListRefresh' + return resp return redirect('run-list') @login_required -@perm_required(PERM_VIEW_RUNS, PERM_TRIGGER_RUNS) -def run_stream(request, run_id): - """SSE endpoint: streams LogEntry rows for a running pipeline.""" - from webapp.models import LogEntry, Run, StageExecution - import json +def run_status(request, run_id): + """Lightweight JSON endpoint — returns run status and latest stage statuses. - def event_stream(): - last_log_id = int(request.GET.get('from', 0)) - last_stage_data = {} + Used as an SSE fallback: when the SSE stream closes before the ``complete`` + event is received, the JS polls this once to sync final state. + """ + from webapp.models import Run, StageExecution + from webapp.perms import user_has_perm, PERM_VIEW_RUNS, PERM_TRIGGER_RUNS + if not (user_has_perm(request.user, PERM_VIEW_RUNS) or + user_has_perm(request.user, PERM_TRIGGER_RUNS)): + return JsonResponse({}, status=403) + run = Run.objects.filter(id=run_id).first() + if not run: + return JsonResponse({}, status=404) + stages = { + s.name: s.status + for s in StageExecution.objects.filter(run_id=run_id) + } + return JsonResponse({'status': run.status, 'stages': stages}) + + +async def run_stream(request, run_id): + """SSE endpoint — async, fan-out via RunBroadcaster. + + Uses an async generator so each viewer is a cheap coroutine rather than + a blocked thread. All DB work is done by the broadcaster's single daemon + thread, so the database is polled once per second per run regardless of + how many clients are connected. + """ + import asyncio + from asgiref.sync import sync_to_async + from webapp.run_broadcaster import broadcaster_manager + from webapp.perms import user_has_perm, PERM_VIEW_RUNS, PERM_TRIGGER_RUNS + + # Inline auth — sync decorators don't compose cleanly with async views. + # auser() is only available on ASGIRequest; fall back under WSGI dev server. + # request.user is a SimpleLazyObject that hits the DB — must be evaluated + # in a thread via sync_to_async to satisfy Django's async safety guard. + if hasattr(request, 'auser'): + user = await request.auser() + else: + from django.contrib.auth import get_user as _get_user + user = await sync_to_async(_get_user)(request) + if not user.is_authenticated: + from django.contrib.auth.views import redirect_to_login + return redirect_to_login(request.get_full_path()) + has_perm = await sync_to_async( + lambda: user_has_perm(user, PERM_VIEW_RUNS) or user_has_perm(user, PERM_TRIGGER_RUNS) + )() + if not has_perm: + return HttpResponse(status=403) + + from webapp.models import Run + run_exists = await sync_to_async(Run.objects.filter(id=run_id).exists)() + if not run_exists: + return HttpResponse(status=404) - while True: - close_old_connections() + broadcaster = broadcaster_manager.get(run_id) - run = Run.objects.filter(id=run_id).first() - if not run: - yield b'event: error\ndata: {"error": "run not found"}\n\n' + # The Last-Event-ID header is sent automatically by EventSource on reconnect. + try: + cursor = int(request.META.get('HTTP_LAST_EVENT_ID', -1)) + except (ValueError, TypeError): + cursor = -1 + + async def event_stream(): + nonlocal cursor + while True: + frames, done = broadcaster.events_since(cursor) + for frame in frames: + yield frame + cursor += 1 + if done and not frames: break + await asyncio.sleep(0.5) - entries = LogEntry.objects.filter( - run_id=run_id, id__gt=last_log_id - ).order_by('id') - for entry in entries: - payload = json.dumps({ - 'type': 'log', - 'level': entry.level, - 'stage': entry.stage_name, - 'message': entry.message, - 'timestamp': entry.timestamp.isoformat(), - }) - yield f'data: {payload}\n\n'.encode() - last_log_id = entry.id + response = _AsyncStreamingHttpResponse(event_stream(), content_type='text/event-stream') + response['Cache-Control'] = 'no-cache' + response['X-Accel-Buffering'] = 'no' + # Prevent GZipMiddleware from buffering the stream before compressing it — + # SSE requires each event to be flushed immediately. + response['Content-Encoding'] = 'identity' + return response - for stage in StageExecution.objects.filter(run_id=run_id): - key = f'{stage.name}:{stage.status}' - if last_stage_data.get(stage.name) != key: - last_stage_data[stage.name] = key - payload = json.dumps({ - 'type': 'stage', - 'name': stage.name, - 'status': stage.status, - 'order': stage.order, - 'started_at': stage.started_at.isoformat() if stage.started_at else None, - 'completed_at': stage.completed_at.isoformat() if stage.completed_at else None, - }) - yield f'data: {payload}\n\n'.encode() - - if run.status in ('success', 'failed', 'cancelled'): - yield b'retry: 86400000\n\n' - payload = json.dumps({'type': 'complete', 'status': run.status}) - yield f'data: {payload}\n\n'.encode() - yield b'event: done\ndata: {}\n\n' - break - time.sleep(1) +async def run_list_stream(request): + """SSE endpoint that fires a single event whenever any run reaches a terminal state. - response = StreamingHttpResponse(event_stream(), content_type='text/event-stream') + The list page subscribes to this and uses the event to trigger an immediate + HTMX refresh of the run table, without waiting for the 10-second poll. + """ + import asyncio + from asgiref.sync import sync_to_async + from webapp.run_broadcaster import run_list_broadcaster + from webapp.perms import user_has_perm, PERM_VIEW_RUNS, PERM_TRIGGER_RUNS + + if hasattr(request, 'auser'): + user = await request.auser() + else: + from django.contrib.auth import get_user as _get_user + user = await sync_to_async(_get_user)(request) + if not user.is_authenticated: + from django.contrib.auth.views import redirect_to_login + return redirect_to_login(request.get_full_path()) + has_perm = await sync_to_async( + lambda: user_has_perm(user, PERM_VIEW_RUNS) or user_has_perm(user, PERM_TRIGGER_RUNS) + )() + if not has_perm: + return HttpResponse(status=403) + + async def event_stream(): + last_seq = run_list_broadcaster._seq + while True: + new_seq = await asyncio.get_event_loop().run_in_executor( + None, run_list_broadcaster.wait_for_change, last_seq, 30.0 + ) + if new_seq > last_seq: + last_seq = new_seq + yield b'data: {}\n\n' + else: + yield b': keepalive\n\n' + + response = _AsyncStreamingHttpResponse(event_stream(), content_type='text/event-stream') response['Cache-Control'] = 'no-cache' response['X-Accel-Buffering'] = 'no' + response['Content-Encoding'] = 'identity' return response @@ -300,11 +389,16 @@ def munki_icon_proxy(request): Setting.get('repository.public_url_username', ''), Setting.get('repository.public_url_password', ''), ) + if not _ICON_PROXY_SEMAPHORE.acquire(blocking=False): + return HttpResponse(status=503) + ctx = ssl.create_default_context(cafile=certifi.where()) headers = {'Authorization': auth_header} if auth_header else {} - req = urllib.request.Request(f'{public_url}/{path}', headers=headers) + from urllib.parse import quote + encoded_path = quote(path, safe='/') + req = urllib.request.Request(f'{public_url}/{encoded_path}', headers=headers) try: - with urllib.request.urlopen(req, timeout=10, context=ctx) as r: + with urllib.request.urlopen(req, timeout=4, context=ctx) as r: content_type = r.headers.get('Content-Type', 'image/png') data = r.read() response = HttpResponse(data, content_type=content_type) @@ -312,3 +406,5 @@ def munki_icon_proxy(request): return response except Exception: return HttpResponse(status=404) + finally: + _ICON_PROXY_SEMAPHORE.release()