diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..5bcf387 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,36 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Checklist** + +- [ ] I have searched for, and not found, an issue for this bug + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..7054484 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,24 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + +**Checklist** + +- [ ] I have searched for, and not found, an issue for this feature. + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml index c9918f2..1412e04 100644 --- a/.github/workflows/ci-checks.yml +++ b/.github/workflows/ci-checks.yml @@ -20,7 +20,7 @@ jobs: - name: "Set up Python 3" uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.12' - name: "Install dependencies" run: | pip install --upgrade pip @@ -38,7 +38,7 @@ jobs: - name: "Set up Python 3" uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.12' - name: "Install dependencies" run: "pip install flake8" - name: "Run flake8!" @@ -53,10 +53,10 @@ jobs: username: ${{ secrets.TEST_DOCKER_USERNAME }} password: ${{ secrets.TEST_DOCKER_PASSWORD }} - uses: actions/checkout@v2 - - name: "Set up Python 3.8" + - name: "Set up Python 3.12" uses: actions/setup-python@v2 - with: - python-version: "3.8" + with: + python-version: "3.12" - name: "Install dependencies" run: | pip install --upgrade pip @@ -79,7 +79,7 @@ jobs: needs: [black, pylint, flake8, pytest-manager] strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: ["3.10", "3.11", "3.12"] steps: - name: Login to Docker Hub uses: docker/login-action@v1 @@ -118,12 +118,13 @@ jobs: - name: "Set up Python 3" uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.12' - name: "Install dependencies" run: | pip install --upgrade pip - pip install -r requirements_manager.txt + pip install -r requirements_manager.txt pip install -r requirements_sdk.txt pip install -r requirements_dev.txt + python -m ipykernel install --user --name python3 - name: "Running E2E tests with pytest" run: "python -m pytest --verbose tests/e2e_test/" \ No newline at end of file diff --git a/.github/workflows/pypi-upload.yml b/.github/workflows/pypi-upload.yml index ba0a726..a38096e 100644 --- a/.github/workflows/pypi-upload.yml +++ b/.github/workflows/pypi-upload.yml @@ -18,7 +18,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel twine + pip install setuptools wheel twine packaging - name: Build and publish to pypi.org env: TWINE_USERNAME: ${{ secrets.GLOBAL_PYPI_USERNAME }} @@ -26,4 +26,4 @@ jobs: DAEPLOY_RELEASE_VERSION: ${{ github.event.release.tag_name }} run: | python setup.py bdist_wheel - twine upload dist/* \ No newline at end of file + twine upload dist/* diff --git a/.pylintrc b/.pylintrc index 7a6b09e..01fd8d1 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,18 +1,25 @@ [pylint] -disable = +disable = R0801, - C0330, - C0326, no-self-argument, no-name-in-module, too-few-public-methods, too-many-arguments, + too-many-positional-arguments, logging-fstring-interpolation, fixme, missing-module-docstring, missing-function-docstring, missing-class-docstring, raise-missing-from, - unsubscriptable-object # TODO: Only required in python 3.9 + unsubscriptable-object, + consider-using-with, + use-dict-literal, + missing-timeout, + unspecified-encoding, + useless-option-value, + invalid-name, + import-error -max-line-length = 88 \ No newline at end of file +max-line-length = 88 +ignored-modules = IPython \ No newline at end of file diff --git a/CHANGES.md b/CHANGES.md index 7e2967f..1bf36e1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,24 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). +## 1.4.0 + +### New Features + +- Redesigned web UI: a modern dark login, dashboard, and a streaming logs view with a Follow / auto-scroll toggle (for both service and manager logs). Fully self-contained — no external CDNs or hot-linked assets. + +### Changed + +- Modernized the manager and SDK dependency stack and moved to Python 3.12. + +### Bugfixes + +- Docker Engine 29 compatibility: `daeploy ls`, the dashboard, and service inspection no longer return HTTP 500 on hosts using the newer Docker storage drivers. + +### Breaking + +- Dropped Python 3.9. Building services requires the Python 3.12 `daeploy/s2i-python` builder image (release 0.1.3). + ## 1.3.0 Daeploy goes Open Source and free to use for any purpose! diff --git a/Dockerfile b/Dockerfile index 24644d0..86ff11b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ## Stage 1: Build image -FROM python:3.8 AS build-image +FROM python:3.12 AS build-image # Install S2i RUN wget -c https://github.com/openshift/source-to-image/releases/download/v1.3.0/source-to-image-v1.3.0-eed2850f-linux-amd64.tar.gz \ @@ -20,7 +20,7 @@ COPY ./requirements_manager.txt . RUN pip install -r requirements_manager.txt ## Stage 2: Production image -FROM python:3.8-slim AS production-image +FROM python:3.12-slim AS production-image # Install Git RUN apt-get update && apt-get install -y git diff --git a/README.md b/README.md index 702df3d..e399aac 100644 --- a/README.md +++ b/README.md @@ -101,8 +101,8 @@ docker run -v /var/run/docker.sock:/var/run/docker.sock -p 80:80 -p 443:443 -e D The application will be available according to the following: -- Docker container listens on ports 80 (http) and 433 (https) -- Proxy listens on ports 5080 (http) (and 5443 (https), NOT fully implemented yet) +- Docker container listens on ports 80 (http) and 443 (https) +- Proxy listens on ports 5080 (http) and 5443 (https) - Manager is available at ```/``` (or locally on port 8000) - Any started services are available at ```/services/{service_name}/``` (or locally on ports 8001 and upwards) - Proxy built-in dashboard is available at ```/proxy/dashboard/``` @@ -133,4 +133,4 @@ Can be found here: [Scope Statement](https://vikinganalytics.sharepoint.com/site ## Architecture Overview -![Architecture Overview](docs/source/content/img/daeploy_architecture.jpg) \ No newline at end of file +![Architecture Overview](docs/source/content/img/daeploy_architecture.jpg) diff --git a/daeploy/_service/db.py b/daeploy/_service/db.py index 5fd852c..fb84e63 100644 --- a/daeploy/_service/db.py +++ b/daeploy/_service/db.py @@ -8,9 +8,9 @@ from contextlib import contextmanager import json -from sqlalchemy import create_engine, and_ +from sqlalchemy import create_engine, and_, MetaData from sqlalchemy.ext.automap import automap_base -from sqlalchemy.orm import sessionmaker, mapper, clear_mappers +from sqlalchemy.orm import sessionmaker, declarative_base from sqlalchemy import Column, DateTime, Float, Text from daeploy.utilities import get_db_table_limit @@ -20,7 +20,7 @@ SERVICE_DB_PATH = Path("service_db.db") ENGINE = create_engine(f"sqlite:///{str(SERVICE_DB_PATH)}") -Base = automap_base() +Base = declarative_base() Session = sessionmaker(bind=ENGINE) QUEUE = queue.Queue() @@ -69,9 +69,6 @@ def create_new_ts_table(name: str, dtype: Type) -> Type: # Create the actual table MapperClass.__table__.create(ENGINE, checkfirst=True) - # Map everything - mapper(MapperClass, MapperClass.__table__) - LOGGER.info(f"Created new table for variable {name}") return MapperClass @@ -221,16 +218,17 @@ def initialize_db(): global QUEUE QUEUE = queue.Queue() global TABLES - Base.prepare(ENGINE, reflect=True) # Automap any existing tables - TABLES = dict(Base.classes) # Make sure we keep track of the auto-mapped tables + # Reflect any existing tables using automap + AutoBase = automap_base(metadata=MetaData()) + AutoBase.prepare(autoload_with=ENGINE) + TABLES = dict(AutoBase.classes) WRITER_THREAD.start() LOGGER.info("DB started!") def remove_db(): """Remove db""" - global WRITER_THREAD - global QUEUE + global WRITER_THREAD, Base # Stop and join writer thread if alive if WRITER_THREAD.is_alive(): @@ -240,10 +238,16 @@ def remove_db(): # Reset it WRITER_THREAD = threading.Thread(target=_writer, daemon=True) + # Reset tables tracking + TABLES.clear() + # Remove db - SERVICE_DB_PATH.unlink() + ENGINE.dispose() + try: + SERVICE_DB_PATH.unlink() + except FileNotFoundError: + pass - # Reset mappers and metadata object - clear_mappers() - Base.metadata.clear() + # Reset base so new tables get fresh mappers + Base = declarative_base() LOGGER.info("DB has been shut down!") diff --git a/daeploy/_service/service.py b/daeploy/_service/service.py index 754f940..aca252e 100644 --- a/daeploy/_service/service.py +++ b/daeploy/_service/service.py @@ -1,3 +1,4 @@ +import os import asyncio import datetime import functools @@ -13,7 +14,8 @@ from fastapi import Body, Request, FastAPI from fastapi.encoders import jsonable_encoder from fastapi.concurrency import run_in_threadpool -from pydantic import create_model, validate_arguments +from fastapi.middleware.cors import CORSMiddleware +from pydantic import create_model, validate_call from daeploy._service.logger import setup_logging from daeploy._service.db import clean_database, initialize_db, remove_db, write_to_ts @@ -31,7 +33,6 @@ ) from daeploy.communication import notify, Severity - setup_logging() logger = logging.getLogger(__name__) @@ -45,8 +46,29 @@ def _disable_http_logs(path: str): ) +def cors_allowed_origins(): + """assumes allowed origin are passed as a single string separated by ; + Example 'https://origin1.com;https://orogin2.com' + + Returns: + list: url of allowed origins + """ + return os.environ.get("DAEPLOY_ALLOW_ORIGIN", "").split(";") + + +def get_cors_config(): + cors_config = {} + cors_config["allow_credentials"] = False + cors_config["allow_origins"] = cors_allowed_origins() + cors_config["allow_methods"] = ["GET", "POST", "PUT", "DELETE"] + cors_config["allow_headers"] = ["Authorization"] + return cors_config + + class _Service: def __init__(self): + cors_config = get_cors_config() + self.app = FastAPI( root_path=get_service_root_path(), title=f"{get_service_name()} {get_service_version()}", @@ -58,6 +80,10 @@ def __init__(self): ], ) + # Custom Middleware + if bool(os.environ.get("DAEPLOY_ENABLE_CORS")): + self.app.add_middleware(CORSMiddleware, **cors_config) + self.parameters = {} # daeploy-specific setup @@ -192,7 +218,7 @@ async def wrapper(_request: Request, *args, **kwargs): _disable_http_logs(path) # Wrap the original func in a pydantic validation wrapper and return that - return validate_arguments(deco_func) + return validate_call(deco_func) # This ensures that we can use the decorator with or without arguments if not (callable(func) or func is None): @@ -343,7 +369,7 @@ def add_parameter( if isinstance(value, Number): value = float(value) - @validate_arguments() + @validate_call() def update_parameter(value: value.__class__) -> Any: logger.info(f"Parameter {parameter} changed to {value}") self.parameters[parameter]["value"] = value diff --git a/daeploy/cli/cli.py b/daeploy/cli/cli.py index f8de691..55bc5cc 100644 --- a/daeploy/cli/cli.py +++ b/daeploy/cli/cli.py @@ -5,8 +5,7 @@ import os import json -import click -import pkg_resources +from importlib.metadata import version as get_version, PackageNotFoundError import pytest import requests import typer @@ -74,9 +73,9 @@ def version_callback(value: bool): # Get SDK Version try: - sdk_version = pkg_resources.get_distribution("daeploy").version + sdk_version = get_version("daeploy") typer.echo(f"SDK version: {sdk_version}") - except pkg_resources.DistributionNotFound: + except PackageNotFoundError: pass # Get Manager Version @@ -123,7 +122,7 @@ def _callback( """ state = config.CliState() # Skip host and token checks if --help flag is included. - if "--help" in click.get_os_args(): + if "--help" in sys.argv[1:]: return # Skip host and token checks if running login, init or test function. @@ -270,12 +269,12 @@ def deploy( name: str = typer.Argument( ..., help="Name of the new service.", - autocompletion=_autocomplete_service_name, + shell_complete=_autocomplete_service_name, ), version: str = typer.Argument( ..., help="Version of the new service.", - autocompletion=_autocomplete_service_version, + shell_complete=_autocomplete_service_version, ), source: str = typer.Argument( ..., @@ -388,12 +387,12 @@ def ls( name: Optional[str] = typer.Argument( None, help="List services with this name.", - autocompletion=_autocomplete_service_name, + shell_complete=_autocomplete_service_name, ), version: Optional[str] = typer.Argument( None, help="List service with this version.", - autocompletion=_autocomplete_service_version, + shell_complete=_autocomplete_service_version, ), ): """List running services, filtered by name and version. @@ -432,13 +431,13 @@ def logs( name: str = typer.Argument( ..., help="Name of the service to read logs from", - autocompletion=_autocomplete_service_name, + shell_complete=_autocomplete_service_name, ), version: Optional[str] = typer.Argument( None, help="Version of the service to read logs from." " Defaults to the main version of the service", - autocompletion=_autocomplete_service_version, + shell_complete=_autocomplete_service_version, ), tail: Optional[str] = typer.Option( DEFAULT_NUMBER_OF_LOGS, @@ -517,12 +516,12 @@ def kill( name: Optional[str] = typer.Argument( None, help="Name of the service(s) to kill.", - autocompletion=_autocomplete_service_name, + shell_complete=_autocomplete_service_name, ), version: Optional[str] = typer.Argument( None, help="Version of the service to kill.", - autocompletion=_autocomplete_service_version, + shell_complete=_autocomplete_service_version, ), all_: Optional[bool] = typer.Option(False, "--all", "-a", help="Kill all services"), validation: Optional[bool] = typer.Option( @@ -589,12 +588,12 @@ def assign( name: str = typer.Argument( ..., help="Name of version to change main", - autocompletion=_autocomplete_service_name, + shell_complete=_autocomplete_service_name, ), version: str = typer.Argument( ..., help="Version of service to set as main", - autocompletion=_autocomplete_service_version, + shell_complete=_autocomplete_service_version, ), validation: Optional[bool] = typer.Option( False, @@ -655,13 +654,13 @@ def init( raise typer.Exit(1) # Find out which daeploy version that should be used by the service try: - dist = pkg_resources.get_distribution("daeploy") + daeploy_version = get_version("daeploy") daeploy_specifier = ( - str(dist.as_requirement()) - if dist.version != "0.0.0.dev0" - else dist.project_name + f"daeploy=={daeploy_version}" + if daeploy_version != "0.0.0.dev0" + else "daeploy" ) # Use full specificer unless in dev environment, then just go for the latest - except pkg_resources.DistributionNotFound: + except PackageNotFoundError: typer.echo( "`daeploy` package not found, assuming latest version " "should be used for the generated project." diff --git a/daeploy/cli/user.py b/daeploy/cli/user.py index f7083f8..75f1bb4 100644 --- a/daeploy/cli/user.py +++ b/daeploy/cli/user.py @@ -5,7 +5,6 @@ from daeploy.cli import cliutils - app = typer.Typer(help="Collection of user management commands") typer.Option(None, "-p", "--password", expose_value=False) diff --git a/daeploy/data_types.py b/daeploy/data_types.py index b2ec2a3..d55f266 100644 --- a/daeploy/data_types.py +++ b/daeploy/data_types.py @@ -1,20 +1,24 @@ -# pylint: disable=too-many-ancestors +# pylint: disable=too-many-ancestors, unused-argument from typing import Any, List, Dict import numpy as np import pandas as pd +from pydantic import GetCoreSchemaHandler +from pydantic_core import core_schema class ArrayInput(np.ndarray): """Pydantic compatible data type for numpy ndarray input.""" @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + return core_schema.no_info_plain_validator_function(cls.validate) @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="array", items={}) + def __get_pydantic_json_schema__(cls, schema, handler): + return {"type": "array", "items": {}} @classmethod def validate(cls, value: List) -> np.ndarray: @@ -26,12 +30,14 @@ class ArrayOutput(np.ndarray): """Pydantic compatible data type for numpy ndarray output.""" @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + return core_schema.no_info_plain_validator_function(cls.validate) @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="array", items={}) + def __get_pydantic_json_schema__(cls, schema, handler): + return {"type": "array", "items": {}} @classmethod def validate(cls, value: np.ndarray) -> List: @@ -43,16 +49,18 @@ class DataFrameInput(pd.DataFrame): """Pydantic compatible data type for pandas DataFrame input.""" @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + return core_schema.no_info_plain_validator_function(cls.validate) @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="object") + def __get_pydantic_json_schema__(cls, schema, handler): + return {"type": "object"} @classmethod def validate(cls, value: Dict[str, Any]) -> pd.DataFrame: - # Transform input to ndarray + # Transform input to DataFrame return pd.DataFrame.from_dict(value) @@ -60,14 +68,16 @@ class DataFrameOutput(pd.DataFrame): """Pydantic compatible data type for pandas DataFrame output.""" @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + return core_schema.no_info_plain_validator_function(cls.validate) @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="object") + def __get_pydantic_json_schema__(cls, schema, handler): + return {"type": "object"} @classmethod def validate(cls, value: pd.DataFrame) -> Dict[str, Any]: - # Transform input to ndarray + # Transform DataFrame to dict return value.to_dict() diff --git a/docs/render_redirect_index_html_file.py b/docs/render_redirect_index_html_file.py index ebd8fc2..b197742 100644 --- a/docs/render_redirect_index_html_file.py +++ b/docs/render_redirect_index_html_file.py @@ -18,13 +18,27 @@ THIS_DIR = Path(__file__).parent BUILD_DIR = (THIS_DIR / "build" / "html").resolve() -versions = filter(lambda x: x.is_dir(), BUILD_DIR.iterdir()) -versions = map(lambda x: version.parse(x.name), versions) +def _as_release(name): + """Parse a build-dir name as a release version. -versions = filter(lambda v: not (v.is_prerelease or v.is_devrelease), versions) + Returns None for non-version dirs (e.g. ``develop``) and for + pre-/dev-releases, so they are excluded from the redirect target. + Newer ``packaging`` raises ``InvalidVersion`` instead of returning a + legacy version, so the parse must be guarded. + """ + try: + parsed = version.parse(name) + except version.InvalidVersion: + return None + if parsed.is_prerelease or parsed.is_devrelease: + return None + return parsed -versions = sorted(versions, reverse=True) + +dir_names = [p.name for p in BUILD_DIR.iterdir() if p.is_dir()] + +versions = sorted(filter(None, (_as_release(name) for name in dir_names)), reverse=True) the_version = str(versions[0]) diff --git a/docs/source/content/advanced_tutorials/adv_deploy.rst b/docs/source/content/advanced_tutorials/adv_deploy.rst index 64021e9..279bc53 100644 --- a/docs/source/content/advanced_tutorials/adv_deploy.rst +++ b/docs/source/content/advanced_tutorials/adv_deploy.rst @@ -165,7 +165,7 @@ Daeploy (`github link `_ for a full list of available tags. -By default Daeploy uses a builder image based on ubuntu with python 3.8. There +By default Daeploy uses a builder image based on python 3.12. There can be situations, however, where that image might not be suitable. For example if you want to use some library that is not supported for that version of python, or if you need a certain OS. diff --git a/docs/source/content/advanced_tutorials/manager_configuration.rst b/docs/source/content/advanced_tutorials/manager_configuration.rst index 3aff44a..14d0bcc 100644 --- a/docs/source/content/advanced_tutorials/manager_configuration.rst +++ b/docs/source/content/advanced_tutorials/manager_configuration.rst @@ -37,6 +37,10 @@ The Manager is highly configurable using environment variables. +--------------------------------------------+-----------------------+-------------------------------------------------------------+ | DAEPLOY_ADMIN_PASSWORD | admin | Password for the admin user. Defualt to admin. | +--------------------------------------------+-----------------------+-------------------------------------------------------------+ +| DAEPLOY_ENBALE_CORS | False | Enables CORS functionality. | ++--------------------------------------------+-----------------------+-------------------------------------------------------------+ +| DAEPLOY_ALLOW_ORIGIN | null | Allowed origins as string separated by ; | ++--------------------------------------------+-----------------------+-------------------------------------------------------------+ User Management --------------- diff --git a/docs/source/content/getting_started/getting_started.rst b/docs/source/content/getting_started/getting_started.rst index c49a743..5b46e5f 100644 --- a/docs/source/content/getting_started/getting_started.rst +++ b/docs/source/content/getting_started/getting_started.rst @@ -11,7 +11,7 @@ applications. Prerequisites ------------- -To use Daeploy, you need to have **python >= 3.6** installed in your development environment +To use Daeploy, you need to have **python >= 3.10** installed in your development environment and **docker** in your deployment environment. Installation diff --git a/docs/superpowers/plans/2026-06-21-ui-redesign.md b/docs/superpowers/plans/2026-06-21-ui-redesign.md new file mode 100644 index 0000000..bad7e13 --- /dev/null +++ b/docs/superpowers/plans/2026-06-21-ui-redesign.md @@ -0,0 +1,571 @@ +# Daeploy UI Redesign Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Reskin Daeploy's login page and Dash dashboard into a modern-dark "control plane" UI, and add a streaming logs view with a Follow/auto-scroll toggle — fully self-contained (no CDNs). + +**Architecture:** Keep the existing stack. Login and the new logs view are FastAPI-served Jinja2 templates; the dashboard stays a Plotly Dash app whose layout is built in Python (`manager/routers/dashboard_api.py`) and styled by CSS in `manager/assets/` (Dash auto-loads `assets/*.css`). A shared `manager/assets/tokens.css` (color/type tokens + bundled `@font-face`) is loaded by all three surfaces. A new `/assets` static mount serves tokens, fonts, and the logo SVG to the Jinja pages. The logs view is a thin HTML shell that `fetch`-streams the existing `GET /services/~logs?...&follow=true` `text/plain` endpoint and renders lines client-side. + +**Tech Stack:** FastAPI, Jinja2, Plotly Dash 4.1, Starlette `StaticFiles`, vanilla CSS + JS (no build step, no external libraries). + +**Canonical visual reference:** `docs/superpowers/specs/2026-06-21-ui-redesign.mockup.html` (the approved mockup, committed). All exact CSS rules, markup structure, the sonar-canvas script, and the logs streaming/Follow JS are taken **verbatim** from it; this plan says which slice of the mockup goes into which file and supplies all the FastAPI/Dash integration code that the mockup (a single static file) doesn't contain. When a step says "copy from the mockup", open that file and copy the named block exactly. + +**Design spec:** `docs/superpowers/specs/2026-06-21-ui-redesign-design.md`. + +## Global Constraints + +- **No external network dependencies in the UI.** No CDN ``/` + + +``` +Keep the `{{ ACTION }}` form exactly. The mark is now `` (the mockup used inline SVG; either is fine — use the img so it shares the asset). + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `./venv/bin/python -m pytest tests/manager_test/test_ui_redesign.py -k "login or assets" -v` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add manager/app.py manager/templates/login.html tests/manager_test/test_ui_redesign.py +git commit -m "Reskin login page; serve bundled assets via /assets static mount" +``` + +--- + +## Task 3: Reskin the dashboard (CSS + Dash layout) + +**Files:** +- Rewrite: `manager/assets/dashboard_styles.css` +- Modify: `manager/routers/dashboard_api.py` (layout-building functions only — NOT callbacks) +- Test: `tests/manager_test/test_ui_redesign.py` + +**Interfaces:** +- Consumes: tokens.css (auto-loaded by Dash from `assets_folder="../assets"`). +- Produces: restyled dashboard. `generate_table_services`, `generate_table_notifications`, `build_banner`, `build_user_section` keep their **names and call signatures**; only the returned component tree + classNames change. `update_content` and the clear-notifications callback are unchanged. + +- [ ] **Step 1: Write the failing tests** + +```python +# append to tests/manager_test/test_ui_redesign.py +def test_dashboard_css_uses_tokens(): + css = (ASSETS / "dashboard_styles.css").read_text() + assert "var(--ground)" in css and "var(--accent)" in css + assert "http://" not in css and "https://" not in css + +def test_dashboard_layout_builds(): + # importing must not raise and layout must be present + from manager.routers import dashboard_api + assert dashboard_api.app.layout is not None + # helper functions still exist with the same names + for fn in ["generate_table_services", "generate_table_notifications", + "build_banner", "build_user_section", "update_content"]: + assert hasattr(dashboard_api, fn) +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `./venv/bin/python -m pytest tests/manager_test/test_ui_redesign.py -k dashboard -v` +Expected: `test_dashboard_css_uses_tokens` FAILS (old CSS has hard-coded hex, no `var(--…)`); `test_dashboard_layout_builds` may already pass (acceptable — it guards the refactor in Step 4). + +- [ ] **Step 3: Rewrite `manager/assets/dashboard_styles.css`** + +Replace the whole file. Port the mockup's **DASHBOARD** CSS section (`.top`, `.vchip`, `.actions`, `.act`, `.page`, `.grid`, `.panel*`, `.svc*`, `.pin*`, `.state*`, `.sdot*`, `.badge*`, `.lnk`, `.note*`, `.sev*`, `.empty`, `@keyframes pulse`, the responsive `@media` block) verbatim, plus a `body`/links base that uses the tokens: +```css +body{background:var(--ground);color:var(--text);font-family:var(--sans);margin:0;overflow-x:hidden;} +a{color:inherit;text-decoration:none;} +/* …then the ported dashboard rules from the mockup, unchanged… */ +``` +Delete the old `.banner`, `#big-app-container`, `#app-container`, `.daeploy_custom-tab*`, `tr:nth-child`, `.logout`, `.severity-*`, `.user-actions` rules (their roles are replaced below). + +- [ ] **Step 4: Rewrite the layout helpers in `manager/routers/dashboard_api.py`** + +Keep all imports, `app = dash.Dash(...)`, callbacks, and `read_services`/`inspect_service` usage. Replace the component-building functions so the tree matches the mockup's dashboard markup. Concretely: + +`build_banner()` → returns the top bar (mockup `.top`): logo `html.Img(src=app.get_asset_url("daeploy_mark.svg"))` + wordmark + `html.Span("manager v: ", className="vchip")`. + +`build_user_section()` → returns the `.actions` nav: `html.A("Logs", …, className="act")`, `html.A("API Docs", …, className="act")`, `html.Button("Clear notifications", id="clear-notifications-button", n_clicks=0, className="act")`, `html.A("Log out", …, className="act danger")`. **Keep `id="clear-notifications-button"`** (the callback depends on it). + +`generate_table_services()` → for each service, build a `.svc` row Div with: status `.sdot` (`run`/`run live` if main running, `shadow`, or `stop`), name + `.ver` version (mono), `.pin main` ★ for main else `.pin` ○/↗, state label + `.since` (reuse `get_service_state`), and a `.svc-actions` Div with the Logs link (Task 4 view URL) + Docs link. + +`generate_table_notifications()` → for each notification build a `.note` row: `.sev info|warn|crit` rule (map severity 0/1/2), `.msg` message, `.meta` with severity tag + timestamp. Reuse the existing severity mapping in `get_severity_colors` (0=Info,1=Warning,2=Critical). + +`app.layout` → wrap in the page structure: keep `dcc.Interval(id="interval1", interval=5*1000, n_intervals=0)`; render `build_banner()`, `build_user_section()`, then a `.page` > `.grid` containing a Services `.panel` (header "Services" + `html.Div(id="app-content")`) and a Notifications `.panel`. **Keep `id="app-content"`** (the `update_content` callback targets it). Wrap classNames per the mockup. + +Use exact classNames from the mockup so the new CSS applies. Do not change `update_content`, the `@app.callback` decorators, or the clear-notifications logic. + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `./venv/bin/python -m pytest tests/manager_test/test_ui_redesign.py -k dashboard -v` +Expected: PASS. Also run the existing dashboard guard: +Run: `./venv/bin/python -m pytest tests/manager_test -k "dashboard or endpoint" -v` +Expected: no new failures vs. baseline (pre-existing env failures from [[daeploy-local-test-env-caveats]] excepted). + +- [ ] **Step 6: Commit** + +```bash +git add manager/assets/dashboard_styles.css manager/routers/dashboard_api.py tests/manager_test/test_ui_redesign.py +git commit -m "Reskin dashboard: token-based styles and control-plane layout" +``` + +--- + +## Task 4: Streaming logs view with Follow toggle + +**Files:** +- Create: `manager/templates/logs.html` +- Modify: `manager/routers/service_api.py` (add HTML view route) +- Modify: `manager/routers/dashboard_api.py` (`get_service_log_link` → point at the view) +- Test: `tests/manager_test/test_ui_redesign.py` + +**Interfaces:** +- Consumes: existing `GET /services/~logs?name&version&tail&follow&since` (`StreamingResponse`, `text/plain`); tokens.css at `/assets/tokens.css`. +- Produces: `GET /services/~logs/view?name=&version=` → HTML page (status 200) that streams the above endpoint. The dashboard service "Logs" link now points here. + +- [ ] **Step 1: Write the failing tests** + +```python +# append to tests/manager_test/test_ui_redesign.py +def test_logs_view_route_returns_page(test_client_logged_in): + r = test_client_logged_in.get("/services/~logs/view?name=demo&version=0.1.0") + assert r.status_code == 200 + body = r.text + assert 'id="console"' in body + assert 'id="followBox"' in body # the Follow checkbox + assert "/services/~logs?" in body # streams the real endpoint + assert "name=demo" in body and "version=0.1.0" in body + +def test_logs_view_template_self_contained(): + html = TPL.joinpath("logs.html").read_text() + low = html.lower() + for bad in FORBIDDEN: + assert bad not in low + assert "/assets/tokens.css" in html +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `./venv/bin/python -m pytest tests/manager_test/test_ui_redesign.py -k logs_view -v` +Expected: FAIL (route + template do not exist). + +- [ ] **Step 3: Create `manager/templates/logs.html`** + +Self-contained shell. Port the mockup's **LOGS** CSS (`.logs-head`, `.live-tag*`, `.follow*`, `.track`, `.console*`, `.logline*`, `.jump*`) and the logs markup (`.panel` with `.logs-head` + `.console-wrap` > `#console` + `#jumpBtn`). Replace the mockup's fake `POOL`/`appendLine` generator with a **real fetch-stream reader** of the `~logs` endpoint, keeping the same Follow/auto-scroll/jump logic verbatim. Read the target from template context: + +```html + + + + + Daeploy — {{ name }} logs + + + + + +
+
+
+
+
+ {{ name }}v{{ version }}
+
+ Live + +
+
+
+
+ +
+
+
+ + + +``` + +- [ ] **Step 4: Add the HTML view route in `manager/routers/service_api.py`** + +Add near the top (after the existing imports): +```python +from fastapi import Request +from fastapi.templating import Jinja2Templates +TEMPLATES = Jinja2Templates(directory="manager/templates") +``` +Add the route (place it just above the existing `@ROUTER.get("/~logs", ...)`): +```python +@ROUTER.get("/~logs/view", response_class=HTMLResponse) +def service_logs_view(request: Request, name: str, version: str): + """HTML view that streams a service's logs with a follow/auto-scroll toggle.""" + return TEMPLATES.TemplateResponse( + "logs.html", {"request": request, "name": name, "version": version} + ) +``` +Ensure `HTMLResponse` is imported (`from fastapi.responses import HTMLResponse, StreamingResponse`). This route deliberately does **not** use the `@async_check_service_exists_query_parameters` decorator — it only renders the shell; the `~logs` stream it calls already enforces existence. + +- [ ] **Step 5: Point the dashboard "Logs" link at the view** + +In `manager/routers/dashboard_api.py`, change `get_service_log_link` to: +```python +def get_service_log_link(service): + proxy_url = get_external_proxy_url() + return html.A( + "Logs", + href=f"{proxy_url}/services/~logs/view" + f"?name={service['name']}&version={service['version']}", + className="lnk", + ) +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `./venv/bin/python -m pytest tests/manager_test/test_ui_redesign.py -k logs_view -v` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add manager/templates/logs.html manager/routers/service_api.py manager/routers/dashboard_api.py tests/manager_test/test_ui_redesign.py +git commit -m "Add streaming logs view with Follow/auto-scroll toggle" +``` + +--- + +## Task 5: Offline + visual verification in the running manager + +**Files:** none (verification task). + +- [ ] **Step 1: Run the full new test module** + +Run: `./venv/bin/python -m pytest tests/manager_test/test_ui_redesign.py -v` +Expected: all PASS. + +- [ ] **Step 2: Lint the changed files (CI gates)** + +Run: `/home/kaveh/miniconda3/bin/python -m black --check manager/ tests/manager_test/test_ui_redesign.py` +Run: `/home/kaveh/miniconda3/bin/python -m flake8 manager/routers/service_api.py manager/routers/dashboard_api.py` +Expected: black clean; no *new* flake8 findings. + +- [ ] **Step 3: Build and run the manager from this branch** + +```bash +docker build -t daeploy/manager:latest . +docker rm -f daeploy-manager 2>/dev/null +docker run -d --name daeploy-manager -v /var/run/docker.sock:/var/run/docker.sock \ + -p 80:80 -p 443:443 -e DAEPLOY_AUTH_ENABLED=True -e DAEPLOY_HOST_NAME=localhost \ + -e DAEPLOY_ADMIN_PASSWORD=admin123 daeploy/manager:latest +``` + +- [ ] **Step 4: Visually verify (browser at http://localhost)** + +- Login renders dark with the sonar backdrop, teal "Log in"; logging in with `admin`/`admin123` works. +- Dashboard shows the restyled top bar, service rows with status dots/badges, and the notifications panel. +- Deploy a sample service and open its **Logs** link → the logs view streams; toggling **Follow** off lets you scroll history and shows **Jump to latest**; toggling on resumes auto-scroll. + +- [ ] **Step 5: Confirm no external requests (offline guarantee)** + +In the browser DevTools Network tab, reload login and the dashboard and confirm **every** request is same-origin (`localhost`) — no fonts/CSS/JS/images from any CDN or `daeploy.com`. + +- [ ] **Step 6: Tear down** + +```bash +docker rm -f daeploy-manager +``` + +- [ ] **Step 7: Commit any verification-driven fixes** + +```bash +git add -A && git commit -m "UI redesign: verification fixes" # only if changes were needed +``` + +--- + +## Self-Review + +**Spec coverage:** self-contained/no-CDN → Task 1 (tokens/fonts), Task 2 (login + `/assets` mount), Task 5 Step 5 (offline check). Login reskin → Task 2. Dashboard reskin (top bar, service rows, notifications) → Task 3. Logs view + Follow/pause/jump → Task 4. Bundled fonts → Task 1. Logo de-hotlink → Task 1 + Tasks 2/4. Color tokens verbatim → Global Constraints + Task 1. Keep stack/behavior → enforced in Tasks 2–4 (form contract, callback ids, helper names). Verification → Task 5. + +**Placeholder scan:** the "copy from the mockup" instructions point to a committed, complete reference file with named blocks — not vague TODOs. All integration code (static mount, routes, fonts, streaming JS, link change) is given in full. No "add error handling"/"TBD" left. + +**Type/name consistency:** callback-critical ids preserved exactly — `clear-notifications-button`, `app-content`, `interval1`. Helper functions keep names (`generate_table_services`, `generate_table_notifications`, `build_banner`, `build_user_section`, `get_service_state`, `get_service_log_link`). New route `GET /services/~logs/view?name&version` matches the link built in Task 4 Step 5 and the test in Step 1. Stream URL params (`name`, `version`, `follow`, `tail`) match the existing `read_service_logs` signature. diff --git a/docs/superpowers/specs/2026-06-21-ui-redesign-design.md b/docs/superpowers/specs/2026-06-21-ui-redesign-design.md new file mode 100644 index 0000000..b621679 --- /dev/null +++ b/docs/superpowers/specs/2026-06-21-ui-redesign-design.md @@ -0,0 +1,81 @@ +# Daeploy UI Redesign — Design Spec + +**Date:** 2026-06-21 +**Branch:** `modernize-ui` (based on `updating-requirements`, so it builds on the upgraded Dash 4.1 / Pydantic v2 stack from PR #90) +**Status:** Approved design (mockup), pending spec review → implementation plan +**Mockup reference:** clickable mockup of all three screens (login, dashboard, logs) approved by the user. + +## 1. Goal & scope + +Modernize Daeploy's two web surfaces into one coherent, distinctive "control plane" UI. + +- **In scope:** visual reskin of (a) the login page, (b) the dashboard, plus (c) a redesigned **logs view** with a streaming follow/auto-scroll toggle. +- **Pure reskin + one UX win:** same features and same stack (FastAPI-served Jinja login + Plotly **Dash** dashboard). The only new behavior is the logs **Follow** toggle (client-side auto-scroll control). +- **Out of scope:** backend/API changes, auth changes, new pages, framework replacement, light theme / theme toggle. + +## 2. Constraints + +- **Fully self-contained / offline-safe.** No external CDNs and no hot-linked images. Today `login.html` pulls Bootstrap 4.5 + jQuery from CDNs and hot-links the logo from `daeploy.com`; all of that must be replaced with local assets in `manager/assets/`. Daeploy is a deployment tool that may run air-gapped. +- **Keep the existing stack.** Login stays a Jinja2 template; the dashboard stays a Dash app whose layout is built in Python (`manager/routers/dashboard_api.py`) and styled by `manager/assets/dashboard_styles.css` (Dash auto-serves everything in `assets/`). +- **Preserve all current features & routes:** login form → `{{ ACTION }}`; dashboard service list (main/shadow, version, state, logs link, docs link); notifications (info/warning/critical); header actions (Logs, API Docs, Clear notifications, Log out); `v: ` indicator. + +## 3. Design language + +Direction: **modern dark "control plane"** for an audience of data scientists / algorithm engineers. Identity is grounded in the subject — Daeploy's whale mark and Viking Analytics' signal-analytics roots — via a sonar/waveform motif and a monospace data face for everything the machine reports. + +### 3.1 Color tokens (CSS custom properties) + +``` +--ground: #0E1320 /* deep navy-ink page background */ +--surface: #161C2C /* panels/cards */ +--surface-2: #1C2438 /* hover / inset */ +--line: #28324A /* borders */ +--line-soft: #1E2638 /* subtle dividers */ +--text: #E7ECF5 /* primary text (cool off-white) */ +--muted: #8B95AC /* secondary text */ +--faint: #5C6680 /* tertiary / captions */ +--accent: #5EE6D0 /* teal — single vivid accent, brand-derived */ +--accent-dim:#2E5A56 /* accent borders */ +--accent-ink:#072019 /* text on accent fills */ +--ok: #3DDC97 /* running */ +--warn: #F4B740 /* warning */ +--crit: #F2585B /* critical / stopped-error */ +``` + +Teal is the **only** vivid accent. Status colors are functional signals, not decoration. Every color in CSS derives from these `:root` variables. + +### 3.2 Typography + +- **UI / display:** system sans stack — `"Segoe UI", system-ui, -apple-system, Roboto, Helvetica, Arial, sans-serif`. Personality comes from weight contrast, tight tracking on the wordmark, and uppercase letterspaced micro-labels. +- **Data / utility:** monospace — used for all machine-reported data (versions, timestamps, state, severity tags, log lines, micro-labels). +- **Self-contained fonts:** the production build **bundles** woff2 files in `manager/assets/fonts/` and references them via `@font-face` (no CDN). Recommended: **Inter** (UI) + **JetBrains Mono** (data). The mockup approximated these with system stacks because the preview sandbox blocks external fonts; the real build ships the woff2s. Final font choice can be confirmed at implementation time, but it MUST be bundled, not linked. + +### 3.3 Layout & components + +- **Wordmark/logo:** small inline SVG "sonar wave" glyph in teal + `dae**ploy**` wordmark (teal second syllable). Replaces the hot-linked PNG. Ship as a local SVG/PNG asset. +- **Login:** centered glass card on the dark ground over a subtle, looping **sonar/waveform canvas** backdrop (the one deliberate motion moment; `prefers-reduced-motion` respected). Card holds wordmark, heading, Username + Password fields (teal focus ring), teal primary "Log in" button, and a footer line ("manager online" status + "by Viking Analytics"). +- **Dashboard:** top bar (wordmark, `manager v: latest` chip, actions). The **service list is the hero** — one row per service with: a status dot (running = pulsing green, shadow = teal, stopped = grey), name + monospace version, a teal ★ for the main version / `shadow` badge for shadow deployments, "Running since " in mono, and inline Logs/Docs links. A **notifications panel** on the right with a severity-coded left rule (info/warn/crit) and mono meta line. Responsive: collapses to a single column under ~760px. +- **Logs view:** top bar + a single console panel. + - Streaming, monospace console with severity styling: `INFO` muted, `WARN` amber left-rule, `ERROR` red left-rule + tint. Each line: timestamp · level · source tag · message. + - **Follow toggle** (switch styled checkbox, top-right): when ON, new lines append and the console auto-scrolls to the newest; a pulsing green **● Live** indicator shows. + - **Smart pause:** scrolling up while following auto-disables Follow, flips the indicator to **Paused**, and reveals a **"Jump to latest ↓"** pill. Clicking it (or re-checking Follow) snaps to the bottom and resumes. + - Line buffer capped (~400 lines) to keep the DOM light. + +### 3.4 Copy + +Written from the end user's side, sentence case, active voice. Examples: "Sign in to your control plane", "Mirroring traffic" for shadow services, actionable notification messages ("anomaly-detector stopped after 3 failed restarts"). Actions keep their names through the flow. + +## 4. Implementation outline (for the plan; not the plan itself) + +1. **Assets (`manager/assets/`):** add bundled font woff2s + `@font-face`; add a local logo SVG; (optionally) a shared `tokens.css` with the `:root` variables imported by both surfaces. +2. **Login (`manager/templates/login.html`):** drop Bootstrap/jQuery CDNs and the hot-linked logo; rebuild markup + inline (local) CSS per the design; keep the `{{ ACTION }}` POST form and field `name`s (`username`, `password`) intact; add the sonar canvas + reduced-motion guard. +3. **Dashboard (`manager/routers/dashboard_api.py` + `manager/assets/dashboard_styles.css`):** restyle and lightly restructure the Python-built layout (banner → top bar; services `html.Table` → row layout with status dots/badges; notifications panel). Preserve callbacks, tab/refresh interval, links, and the clear-notifications action. +4. **Logs view:** identify how logs are currently surfaced (the `/logs` and `/services/~logs` routes / Dash links) and render the streaming output in the new console with the client-side Follow/auto-scroll + jump-to-latest behavior. Backend already streams with `follow=true`; the toggle is presentation-only. +5. **Verification:** run the manager locally in Docker (build image, deploy a sample service) and visually confirm login, dashboard, and live logs; confirm no external network requests are made by the UI (offline check). + +## 5. Risks & notes + +- Dash builds HTML in Python, so dashboard restyling spans both the `.css` and the component tree in `dashboard_api.py`; keep changes scoped to layout/className, not callback logic. +- Bundled fonts add a few hundred KB to `manager/assets/` — acceptable for offline support; subset if size matters. +- The mockup's bottom screen-switcher is a mockup-only affordance and is NOT part of the product. +- Base branch builds on `updating-requirements`; if PR #90 merges first, rebase onto `develop`. diff --git a/docs/superpowers/specs/2026-06-21-ui-redesign.mockup.html b/docs/superpowers/specs/2026-06-21-ui-redesign.mockup.html new file mode 100644 index 0000000..24c6951 --- /dev/null +++ b/docs/superpowers/specs/2026-06-21-ui-redesign.mockup.html @@ -0,0 +1,687 @@ +Daeploy — UI redesign mockup + + + +
+ +
+ + +
+
+
+
+ + daeploy +
+ manager v: latest +
+ +
+ +
+
+ + +
+
+

Services

+ 4 deployed · 3 running +
+ +
+
+ ★ +
+
status_code
+
v0.1.0
+
+
+
+ +
+
Running
+
since 2026-06-21 17:24:13
+
+
+
+ Logs + Docs +
+
+ +
+
+ ★ +
+
forecast-api
+
v1.2.0
+
+
+
+ +
+
Running
+
since 2026-06-20 09:11:02
+
+
+
+ Logs + Docs +
+
+ +
+
+ ↗ +
+
forecast-api shadow
+
v1.3.0
+
+
+
+ +
+
Mirroring traffic
+
since 2026-06-21 14:02:55
+
+
+
+ Logs + Docs +
+
+ +
+
+ ○ +
+
anomaly-detector
+
v0.4.1
+
+
+
+ +
+
Stopped
+
since 2026-06-19 22:48:10
+
+
+
+ Logs + Docs +
+
+
+ + +
+
+

Notifications

+ 3 +
+ +
+ +
+
anomaly-detector stopped after 3 failed restarts.
+
CriticalYesterday · 22:49
+
+
+
+ +
+
anomaly-detector exited with code 1 and was restarted.
+
WarningYesterday · 22:48
+
+
+
+ +
+
forecast-api v1.3.0 deployed as a shadow of v1.2.0.
+
InfoToday · 14:02
+
+
+
+ +
+
+
+ + +
+
+
+
+ + daeploy +
+ manager v: latest +
+ +
+ +
+
+
+
+ + status_code + v0.1.0 +
+
+ Live + +
+
+ +
+
+ +
+
+
+
+ + +
+ + + +
+ + diff --git a/manager/app.py b/manager/app.py index f7236d9..759d4b9 100644 --- a/manager/app.py +++ b/manager/app.py @@ -2,8 +2,10 @@ import logging from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware from starlette.responses import RedirectResponse from starlette.middleware.wsgi import WSGIMiddleware +from starlette.staticfiles import StaticFiles from manager.routers import ( admin_api, @@ -16,8 +18,7 @@ from manager import proxy from manager.database.database import initialize_db from manager.database import service_db -from manager.constants import get_manager_version - +from manager.constants import get_manager_version, cors_enabled, cors_config # Setup logger logging_api.setup_logging() @@ -30,6 +31,10 @@ version=get_manager_version(), ) +# CORS middleware +if cors_enabled(): + app.add_middleware(CORSMiddleware, **cors_config()) + # Services subapi app.include_router(service_api.ROUTER, prefix="/services", tags=["Service"]) @@ -38,6 +43,9 @@ notification_api.ROUTER, prefix="/notifications", tags=["Notification"] ) +# Static assets +app.mount("/assets", StaticFiles(directory="manager/assets"), name="assets") + # Dashboard subapi app.mount("/dashboard", WSGIMiddleware(dashboard_api.app.server)) diff --git a/manager/assets/daeploy_mark.svg b/manager/assets/daeploy_mark.svg new file mode 100644 index 0000000..285b8e4 --- /dev/null +++ b/manager/assets/daeploy_mark.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/manager/assets/dashboard_styles.css b/manager/assets/dashboard_styles.css index 928f3b6..09d8e0b 100644 --- a/manager/assets/dashboard_styles.css +++ b/manager/assets/dashboard_styles.css @@ -1,238 +1,136 @@ -/* Table of contents -–––––––––––––––––––––––––––––––––––––––––––––––––– -- Banner -- Modal -- Tabs -- Main Dashboard Tab -- Measurement Tab -- Tables/Dropdown -- Containers -- Media Queries -–––––––––––––––––––––––––––––––––––––––––––––––––– */ - - -body { - background-color: #1e2130; - color: #f3f5f4; - font-family: "Open Sans", sans-serif; - width: 100%; - height: 100vh; - max-width: 100% !important; - overflow-x: hidden; - margin: 0; -} - -.banner { - height: fit-content; - background-color: #1e2130; - display: flex; - flex-direction: row; - align-items: center; - justify-content: space-between; - border-bottom: 1px solid #4B5460; - padding: 1rem 10rem; - width: 100%; -} - -.banner h5 { - font-family: 'Open Sans Semi Bold', sans-serif; - font-weight: 500; - line-height: 1.2; - font-size: 2rem; - letter-spacing: 0.1rem; - text-transform: uppercase; -} - -.banner h6 { - font-size: 1.6rem; - line-height: 1; -} - -#banner-logo { - display: flex; - flex-direction: row; - align-items: center; - justify-content: flex-end; -} - -.banner Img { - height: 3rem; - margin: 0px 10px; -} - -/* container */ -#big-app-container { - max-width: 100%; - display: flex; - flex-direction: column; - align-items: center; - padding: 0 6rem; -} - -#app-container { - background: #161a28 !important; - margin: 1rem 2rem !important; - max-width: 100% !important; - width: 100% !important; - height: calc(100vh - 10rem - 1px) !important; -} - -#app-container * { - box-sizing: border-box; - -moz-box-sizing: border-box; -} -/* Tabs -–––––––––––––––––––––––––––––––––––––––––––––––––– */ - -.daeploy_custom-tabs { - background-color: #161a28 !important; - text-transform: uppercase !important; - font-weight: 500 !important; - font-size: 16px !important; - height: fit-content !important; - cursor: pointer !important; - color: #f8f9fc !important; - padding-top: 10px !important; - padding-bottom: 10px !important; - border-top: #161a28 solid !important; - border-left: #161a28 solid !important; - border-right: #161a28 solid !important; -} - -.daeploy_custom-tab{ - border: #161a28 solid !important; -} - -.daeploy_custom-tab--selected{ - border-bottom: #91dfd2 solid !important; - border-top: #161a28 solid !important; - border-left: #161a28 solid !important; - border-right: #161a28 solid !important; -} - -/* Table */ -table { - margin-right: 3px; - margin-left: 3px; - border-collapse: collapse; - width: 100%; -} - -td, th { - text-align: left; - padding: 8px; +/* Daeploy dashboard styles — uses design tokens from tokens.css */ + +body{background:var(--ground);color:var(--text);font-family:var(--sans);margin:0;overflow-x:hidden;} +a{color:inherit;text-decoration:none;} + +/* ---------- top bar ---------- */ +.top{ + display:flex;align-items:center;justify-content:space-between;gap:1rem; + padding:1.05rem 1.6rem;border-bottom:1px solid var(--line-soft); + background:linear-gradient(180deg,rgba(22,28,44,.6),transparent); +} +.top .left{display:flex;align-items:center;gap:1.1rem} +.vchip{ + font-family:var(--mono);font-size:11px;letter-spacing:.08em;color:var(--muted); + border:1px solid var(--line);border-radius:999px;padding:.2rem .6rem; +} +.vchip b{color:var(--accent);font-weight:600} + +/* ---------- actions nav ---------- */ +.actions{display:flex;align-items:center;gap:.5rem;flex-wrap:wrap} +.act{ + font-family:var(--mono);font-size:11px;letter-spacing:.1em;text-transform:uppercase; + color:var(--muted);background:transparent;border:1px solid var(--line); + border-radius:8px;padding:.42rem .7rem;cursor:pointer;text-decoration:none; + transition:color .15s,border-color .15s,background .15s;display:inline-block; +} +.act:hover{color:var(--text);border-color:var(--accent-dim)} +.act:focus-visible{outline:2px solid var(--accent);outline-offset:2px;} +.act.danger:hover{color:var(--crit);border-color:var(--crit)} + +/* ---------- page / grid ---------- */ +.page{max-width:1180px;margin:0 auto;padding:1.8rem 1.6rem 4rem} +.grid{display:grid;grid-template-columns:1.85fr 1fr;gap:1.4rem;align-items:start} + +/* ---------- panels ---------- */ +.panel{ + background:var(--surface);border:1px solid var(--line-soft);border-radius:16px; + overflow:hidden; +} +.panel-head{ + display:flex;align-items:center;justify-content:space-between; + padding:1rem 1.2rem .85rem;border-bottom:1px solid var(--line-soft); +} +.panel-head h2{font-size:.82rem;letter-spacing:.04em;font-weight:600;margin:0} +.count{font-family:var(--mono);font-size:11px;color:var(--faint)} + +/* ---------- service rows ---------- */ +.svc{ + display:grid; + grid-template-columns:1.3fr .7fr auto; + grid-template-areas:"id state actions"; + align-items:center;gap:.8rem; + padding:.95rem 1.2rem;border-bottom:1px solid var(--line-soft); + transition:background .15s; +} +.svc:last-child{border-bottom:none} +.svc:hover{background:var(--surface-2)} +.svc .id{grid-area:id;display:flex;align-items:center;gap:.7rem;min-width:0} +.svc .name{font-weight:560;font-size:.96rem;letter-spacing:-.01em} +.svc .ver{font-family:var(--mono);font-size:11px;color:var(--muted);margin-top:.12rem} + +/* ---------- pin badges ---------- */ +.pin{ + flex:none;width:26px;height:26px;border-radius:8px;display:grid;place-items:center; + font-size:11px;font-family:var(--mono); + border:1px solid var(--line);color:var(--faint); +} +.pin.main{color:var(--accent);border-color:var(--accent-dim);background:rgba(94,230,208,.06)} + +/* ---------- state column ---------- */ +.state{grid-area:state;display:flex;align-items:center;gap:.55rem;min-width:0} +.sdot{width:8px;height:8px;border-radius:50%;flex:none} +.sdot.run{background:var(--ok);box-shadow:0 0 0 3px rgba(61,220,151,.15)} +.sdot.run.live{animation:pulse 2.4s ease-in-out infinite} +.sdot.shadow{background:var(--accent);box-shadow:0 0 0 3px rgba(94,230,208,.15)} +.sdot.stop{background:var(--faint)} +.state .lbl{font-size:.82rem} +.lbl.stopped{color:var(--muted)} +.state .since{font-family:var(--mono);font-size:10.5px;color:var(--faint);margin-top:.1rem} + +/* ---------- badges ---------- */ +.badge{ + font-family:var(--mono);font-size:9.5px;letter-spacing:.14em;text-transform:uppercase; + padding:.16rem .42rem;border-radius:5px; +} +.badge.shadow{color:var(--accent);background:rgba(94,230,208,.1)} + +/* ---------- service actions ---------- */ +.svc-actions{grid-area:actions;display:flex;gap:.4rem} +.lnk{ + font-family:var(--mono);font-size:10.5px;letter-spacing:.08em;text-transform:uppercase; + color:var(--muted);text-decoration:none;border-bottom:1px solid transparent;padding:.1rem 0; + transition:color .15s,border-color .15s; +} +.lnk:hover{color:var(--accent);border-color:var(--accent)} + +/* ---------- notifications ---------- */ +.note{ + display:grid;grid-template-columns:auto 1fr;gap:.75rem; + padding:.85rem 1.2rem;border-bottom:1px solid var(--line-soft); +} +.note:last-child{border-bottom:none} +.sev{width:3px;border-radius:3px;align-self:stretch} +.sev.info{background:var(--accent)} +.sev.warn{background:var(--warn)} +.sev.crit{background:var(--crit)} +.note .msg{font-size:.86rem;line-height:1.45} +.note .meta{ + font-family:var(--mono);font-size:10px;letter-spacing:.1em;text-transform:uppercase; + color:var(--faint);margin-top:.3rem;display:flex;gap:.6rem; +} +.sev-tag{font-weight:600} +.sev-tag.info{color:var(--accent)} +.sev-tag.warn{color:var(--warn)} +.sev-tag.crit{color:var(--crit)} + +.empty{padding:2.2rem 1.2rem;text-align:center;color:var(--faint);font-size:.85rem} + +/* ---------- wordmark ---------- */ +.mark{display:inline-flex;align-items:center;gap:.6rem} +.wordmark{font-weight:600;font-size:1.4rem;letter-spacing:-.02em} +.wordmark b{color:var(--accent);font-weight:600} + +/* ---------- keyframes ---------- */ +@keyframes pulse{0%,100%{box-shadow:0 0 0 0 rgba(61,220,151,.45)}50%{box-shadow:0 0 0 6px rgba(61,220,151,0)}} + +/* ---------- responsive ---------- */ +@media (max-width:760px){ + .grid{grid-template-columns:1fr} + .svc{grid-template-columns:1fr;grid-template-areas:"id" "state" "actions";gap:.55rem} + .svc-actions{justify-content:flex-start} + .top{flex-direction:column;align-items:flex-start;gap:.9rem} +} +@media (prefers-reduced-motion:reduce){ + .sdot.run.live{animation:none} } - -tr:nth-child(even) { - background-color: #343744; -} - -/* Links */ -a:visited { text-decoration: none; color:white; } -a:hover { text-decoration: none; color:green; } -a:focus { text-decoration: none; color:yellow; } -a:hover, a:active { text-decoration: none; color:green } - -/* Severity */ -.severity-info { - background-color: #99cc33; - color:black; - font-weight: bold; - text-align: center; - width: 1px; -} -.severity-warning { - background-color: #ffcc00; - color:black; - font-weight: bold; - text-align: center; - width: 1px; -} -.severity-critical { - background-color: #cc3300; - color:black; - font-weight: bold; - text-align: center; - width: 1px; -} - -/* Green Text */ -.green-text { - color: lightgreen; -} - -/* User information */ -.user-actions { - position:absolute; - right:10px; - top:5px; - display: block; - text-align: center; -} - -.logout { - cursor: pointer; - color: white; - text-decoration: none; - border: 2px solid white; - border-radius: 30px; - transition-duration: .2s; - -webkit-transition-duration: .2s; - -moz-transition-duration: .2s; - background-color: #161a28; - padding: 4px 20px; - font-size: 0.7rem; -} - -.logout:hover { - background-color:red; - color:white; -} - -.version-identifier { - text-align: center; - font-size: 1.1rem; -} - -/* - ##Device = Most of the Smartphones Mobiles / ipad (Portrait) - */ -@media only screen and (max-width: 950px) { - - body { - font-size: 1.3rem; - } - - #big-app-container { - padding: 1rem; - } - - .banner { - flex-direction: column-reverse; - padding: 1rem 0.5rem; - } - - #banner-text { - text-align: center; - } - - .banner h5 { - font-size: 1.4rem; - } - - .banner h6 { - font-size: 1.3rem; - } - - #banner-logo button { - display: none; - } - - .banner Img { - height: 3rem; - margin: 1rem; - } - - #app-container { - height: auto; - } \ No newline at end of file diff --git a/manager/assets/fonts/inter-400.woff2 b/manager/assets/fonts/inter-400.woff2 new file mode 100644 index 0000000..f15b025 Binary files /dev/null and b/manager/assets/fonts/inter-400.woff2 differ diff --git a/manager/assets/fonts/inter-500.woff2 b/manager/assets/fonts/inter-500.woff2 new file mode 100644 index 0000000..54f0a59 Binary files /dev/null and b/manager/assets/fonts/inter-500.woff2 differ diff --git a/manager/assets/fonts/inter-600.woff2 b/manager/assets/fonts/inter-600.woff2 new file mode 100644 index 0000000..d189794 Binary files /dev/null and b/manager/assets/fonts/inter-600.woff2 differ diff --git a/manager/assets/fonts/jbmono-400.woff2 b/manager/assets/fonts/jbmono-400.woff2 new file mode 100644 index 0000000..5858873 Binary files /dev/null and b/manager/assets/fonts/jbmono-400.woff2 differ diff --git a/manager/assets/fonts/jbmono-500.woff2 b/manager/assets/fonts/jbmono-500.woff2 new file mode 100644 index 0000000..be878e6 Binary files /dev/null and b/manager/assets/fonts/jbmono-500.woff2 differ diff --git a/manager/assets/tokens.css b/manager/assets/tokens.css new file mode 100644 index 0000000..37e31ad --- /dev/null +++ b/manager/assets/tokens.css @@ -0,0 +1,17 @@ +/* Daeploy design tokens — single source of truth for color + type. */ +@font-face{font-family:"Inter";font-weight:400;font-display:swap;src:url(fonts/inter-400.woff2) format("woff2");} +@font-face{font-family:"Inter";font-weight:500;font-display:swap;src:url(fonts/inter-500.woff2) format("woff2");} +@font-face{font-family:"Inter";font-weight:600;font-display:swap;src:url(fonts/inter-600.woff2) format("woff2");} +@font-face{font-family:"JetBrains Mono";font-weight:400;font-display:swap;src:url(fonts/jbmono-400.woff2) format("woff2");} +@font-face{font-family:"JetBrains Mono";font-weight:500;font-display:swap;src:url(fonts/jbmono-500.woff2) format("woff2");} + +:root{ + --ground:#0E1320; --surface:#161C2C; --surface-2:#1C2438; + --line:#28324A; --line-soft:#1E2638; + --text:#E7ECF5; --muted:#8B95AC; --faint:#5C6680; + --accent:#5EE6D0; --accent-dim:#2E5A56; --accent-ink:#072019; + --ok:#3DDC97; --warn:#F4B740; --crit:#F2585B; + --shadow:0 24px 60px -24px rgba(0,0,0,.7); + --sans:"Inter","Segoe UI",system-ui,-apple-system,Roboto,Helvetica,Arial,sans-serif; + --mono:"JetBrains Mono",ui-monospace,"SF Mono",Menlo,Consolas,monospace; +} diff --git a/manager/constants.py b/manager/constants.py index 1619800..6428d14 100644 --- a/manager/constants.py +++ b/manager/constants.py @@ -1,6 +1,7 @@ """ Constants and config """ + import os from pathlib import Path @@ -107,6 +108,29 @@ def configuration_email(): return os.environ.get("DAEPLOY_CONFIG_EMAIL") +def cors_enabled(): + return os.environ.get("DAEPLOY_ENABLE_CORS") + + +def cors_allowed_origins(): + """assumes allowed origin are passed as a single string separated by ; + Example 'https://origin1.com;https://orogin2.com' + + Returns: + list: url of allowed origins + """ + return os.environ.get("DAEPLOY_ALLOW_ORIGIN", "").split(";") + + +def cors_config(): + config = {} + config["allow_credentials"] = False + config["allow_origins"] = cors_allowed_origins() + config["allow_methods"] = ["GET", "POST", "PUT", "DELETE"] + config["allow_headers"] = ["Authorization"] + return config + + def notification_email_config(): sender_email = configuration_email() sender_pass = os.environ.get("DAEPLOY_CONFIG_EMAIL_PASSWORD") diff --git a/manager/data_models/request_models.py b/manager/data_models/request_models.py index 10e3f13..6b38a91 100644 --- a/manager/data_models/request_models.py +++ b/manager/data_models/request_models.py @@ -3,8 +3,8 @@ import semver from pydantic.types import SecretStr -from pydantic import BaseModel, validator, HttpUrl -from fastapi import Path, UploadFile +from pydantic import BaseModel, field_validator, HttpUrl, ConfigDict +from fastapi import UploadFile, Query from manager.constants import ( DAEPLOY_DEFAULT_INTERNAL_PORT, @@ -16,8 +16,8 @@ class BaseService(BaseModel): name: str version: str - # pylint: disable=no-self-use - @validator("name") + @field_validator("name") + @classmethod def must_adhere_to_docker_requirements(cls, name): # Only allow a name to contain lower case letters, numbers and underscore # anywhere but in the beginning and end @@ -29,16 +29,16 @@ def must_adhere_to_docker_requirements(cls, name): ) return name - # pylint: disable=no-self-use - @validator("version") + @field_validator("version") + @classmethod def must_be_semver_string(cls, version): - if not semver.VersionInfo.isvalid(version): + if not semver.Version.is_valid(version): raise ValueError("Version must be a semantic version string.") return version class BaseNewServiceRequest(BaseService): - port: int = Path(default=DAEPLOY_DEFAULT_INTERNAL_PORT, gt=0) + port: int = Query(default=DAEPLOY_DEFAULT_INTERNAL_PORT, gt=0) run_args: Dict = {} @@ -49,8 +49,8 @@ class BaseNewS2IServiceRequest(BaseNewServiceRequest): class ServiceImageRequest(BaseNewServiceRequest): image: str - class Config: - schema_extra = { + model_config = ConfigDict( + json_schema_extra={ "example": { "name": "myservice", "version": "0.0.1", @@ -59,13 +59,14 @@ class Config: "run_args": {}, } } + ) class ServiceGitRequest(BaseNewS2IServiceRequest): git_url: HttpUrl - class Config: - schema_extra = { + model_config = ConfigDict( + json_schema_extra={ "example": { "name": "myservice", "version": "0.0.1", @@ -74,13 +75,14 @@ class Config: "run_args": {}, } } + ) class ServiceTarRequest(BaseNewS2IServiceRequest): file: UploadFile - class Config: - schema_extra = { + model_config = ConfigDict( + json_schema_extra={ "example": { "name": "myservice", "version": "0.0.1", @@ -89,13 +91,14 @@ class Config: "run_args": {}, } } + ) class ServicePickleRequest(ServiceTarRequest): requirements: List[str] - class Config: - schema_extra = { + model_config = ConfigDict( + json_schema_extra={ "example": { "name": "myservice", "version": "0.0.1", @@ -104,6 +107,7 @@ class Config: "requirements": [], } } + ) class NotificationRequest(BaseModel): @@ -112,7 +116,7 @@ class NotificationRequest(BaseModel): msg: str severity: int dashboard: bool - emails: Union[List[str], None] + emails: Union[List[str], None] = None timer: int timestamp: str diff --git a/manager/data_models/response_models.py b/manager/data_models/response_models.py index 58c710a..1e238f7 100644 --- a/manager/data_models/response_models.py +++ b/manager/data_models/response_models.py @@ -34,28 +34,32 @@ class StateResponse(BaseModel): Error: str StartedAt: str FinishedAt: str - Health: Optional[HealthResponse] + Health: Optional[HealthResponse] = None class NetworkSettingsResponse(BaseModel): - Bridge: str + # Docker Engine 29+ no longer populates the legacy top-level network + # fields (Bridge, IPAddress, MacAddress, etc.) for containers attached + # only to a custom network; that data now lives under `Networks`. Keep + # them optional so inspection doesn't fail response validation. SandboxID: str - HairpinMode: bool - LinkLocalIPv6Address: str - LinkLocalIPv6PrefixLen: int Ports: dict SandboxKey: str - SecondaryIPAddresses: Optional[str] - SecondaryIPv6Addresses: Optional[str] - EndpointID: str - Gateway: str - GlobalIPv6Address: str - GlobalIPv6PrefixLen: int - IPAddress: str - IPPrefixLen: int - IPv6Gateway: str - MacAddress: str Networks: Dict[str, Dict] + Bridge: Optional[str] = None + HairpinMode: Optional[bool] = None + LinkLocalIPv6Address: Optional[str] = None + LinkLocalIPv6PrefixLen: Optional[int] = None + SecondaryIPAddresses: Optional[str] = None + SecondaryIPv6Addresses: Optional[str] = None + EndpointID: Optional[str] = None + Gateway: Optional[str] = None + GlobalIPv6Address: Optional[str] = None + GlobalIPv6PrefixLen: Optional[int] = None + IPAddress: Optional[str] = None + IPPrefixLen: Optional[int] = None + IPv6Gateway: Optional[str] = None + MacAddress: Optional[str] = None class InspectResponse(BaseModel): @@ -76,9 +80,14 @@ class InspectResponse(BaseModel): MountLabel: str ProcessLabel: str AppArmorProfile: str - ExecIDs: Optional[List[str]] + ExecIDs: Optional[List[str]] = None HostConfig: dict - GraphDriver: dict + # Docker 29+ with the containerd/overlayfs image store returns `Storage` + # (and `ImageManifestDescriptor`) instead of the legacy `GraphDriver`, so + # none of these can be required for inspection to work across drivers. + GraphDriver: Optional[dict] = None + Storage: Optional[dict] = None + ImageManifestDescriptor: Optional[dict] = None Mounts: list Configuration: dict = Field(..., alias="Config") NetworkSettings: NetworkSettingsResponse diff --git a/manager/database/auth_db.py b/manager/database/auth_db.py index 2865b94..0d73595 100644 --- a/manager/database/auth_db.py +++ b/manager/database/auth_db.py @@ -19,7 +19,8 @@ def add_user_record(username: str, password: str): """ with session_scope() as session: new_user = User( - name=username, password=bcrypt.hashpw(password.encode(), bcrypt.gensalt()) + name=username, + password=bcrypt.hashpw(password.encode(), bcrypt.gensalt()).decode(), ) session.add(new_user) diff --git a/manager/database/database.py b/manager/database/database.py index be8a6d4..ae2bc73 100644 --- a/manager/database/database.py +++ b/manager/database/database.py @@ -4,8 +4,7 @@ import logging from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker, declarative_base from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey from manager.constants import DAEPLOY_DATA_DIR, get_admin_password @@ -128,9 +127,8 @@ def initialize_db(): def remove_db(): """Removes db""" + engine.dispose() try: MANAGER_DB_PATH.unlink() except FileNotFoundError: - # Path.unlink(missing_ok=True) gives same behavior but was - # not introduced until python 3.8 pass diff --git a/manager/proxy.py b/manager/proxy.py index d44bc89..a035354 100644 --- a/manager/proxy.py +++ b/manager/proxy.py @@ -274,6 +274,10 @@ def get_manager_configuration() -> dict: """ rule = f"""Host(`{get_proxy_domain_name()}`)""" rule_login = f"""Host(`{get_proxy_domain_name()}`) && PathPrefix(`/auth/login`)""" + # Static UI assets (CSS, fonts, logo) must load on the login page *before* + # the user authenticates, so they are served by a router without the auth + # middleware. These are non-sensitive presentation files only. + rule_assets = f"""Host(`{get_proxy_domain_name()}`) && PathPrefix(`/assets`)""" config = { "http": { @@ -290,6 +294,12 @@ def get_manager_configuration() -> dict: middlewares=None, tls=https_proxy(), ), + "static_assets": get_router_configuration( + rule=rule_assets, + service="manager_service", + middlewares=None, + tls=https_proxy(), + ), }, "services": { "manager_service": { diff --git a/manager/routers/auth_api.py b/manager/routers/auth_api.py index a528335..7b0e268 100644 --- a/manager/routers/auth_api.py +++ b/manager/routers/auth_api.py @@ -115,8 +115,9 @@ def show_login_page(request: Request, destination: Optional[str] = "/"): # noqa: DAR101,DAR201,DAR401 """ return TEMPLATES.TemplateResponse( - "login.html", - {"request": request, "ACTION": f"/auth/login?destination={destination}"}, + request=request, + name="login.html", + context={"ACTION": f"/auth/login?destination={destination}"}, status_code=401, ) @@ -143,7 +144,12 @@ def login_user( LOGGER.exception(f"User {username} failed to login!") return RedirectResponse(url=destination, status_code=303) - if not bcrypt.checkpw(password.get_secret_value().encode(), record.password): + stored_pw = ( + record.password.encode() + if isinstance(record.password, str) + else record.password + ) + if not bcrypt.checkpw(password.get_secret_value().encode(), stored_pw): return RedirectResponse(url=destination, status_code=303) # Construct token diff --git a/manager/routers/dashboard_api.py b/manager/routers/dashboard_api.py index 30eb2af..67ef13f 100644 --- a/manager/routers/dashboard_api.py +++ b/manager/routers/dashboard_api.py @@ -2,9 +2,7 @@ from datetime import datetime import dash -import dash_core_components as dcc -import dash_html_components as html -from dash.dependencies import Input, Output +from dash import dcc, html, Input, Output from manager.routers.service_api import read_services, inspect_service from manager.routers.notification_api import get_notifications, delete_notifications @@ -23,88 +21,68 @@ def build_user_section(): - return html.Div( - id="user-actions", - className="user-actions", + return html.Nav( + className="actions", children=[ - html.P(f"v:{get_manager_version()}", className="version-identifier"), - html.P(), html.A( - "LOGS", - id="manager-logs-buttom", - href=f"{get_external_proxy_url()}/logs", - className="logout", + "Logs", + href=f"{get_external_proxy_url()}/logs/view", + className="act", ), html.A( - "API DOCS", - id="documenation-button", + "API Docs", href=f"{get_external_proxy_url()}/docs", - className="logout", + className="act", ), - html.P(), html.Button( - "CLEAR NOTIFICATIONS", + "Clear notifications", id="clear-notifications-button", n_clicks=0, - className="logout", + className="act", ), html.A( - "LOGOUT", - id="logout-button", + "Log out", href=f"{get_external_proxy_url()}/auth/logout", - className="logout", + className="act danger", ), ], ) def build_banner(): - return html.Div( - id="banner", - className="banner", + return html.Header( + className="top", children=[ html.Div( - id="banner-text", + className="left", children=[ - html.Img(src=app.get_asset_url("daeploy_white_icon.png")), - dcc.Markdown( - """ - ### Daeploy Dashboard - by Viking Analytics AB - """ + html.Div( + className="mark", + children=[ + html.Img( + src=app.get_asset_url("daeploy_mark.svg"), + width=26, + height=26, + ), + html.Span( + children=[ + "dae", + html.B("ploy"), + ], + className="wordmark", + ), + ], ), - ], - ), - ], - ) - - -def build_tabs(): - return html.Div( - id="daeploy_tabs", - children=[ - dcc.Tabs( - id="app-tabs", - className="daeploy_custom-tabs", - # Set tab-1 to active from start. - value="tab1", - children=[ - dcc.Tab( - id="services", - label="Services", - value="tab1", - className="daeploy_custom-tabs", - selected_className="daeploy_custom-tab--selected", - ), - dcc.Tab( - id="notification", - label="Notifications", - value="tab2", - className="daeploy_custom-tabs", - selected_className="daeploy_custom-tab--selected", + html.Span( + children=[ + "manager ", + html.B(f"v: {get_manager_version()}"), + ], + className="vchip", ), ], - ) + ), + build_user_section(), ], ) @@ -142,72 +120,90 @@ def update_content(tab_switch, interval, n_clicks): ) -def generate_table_services(): - """Generates a HTML table with the service information +@app.callback( + Output("notifications-content", "children"), + Input("interval1", "n_intervals"), + Input("clear-notifications-button", "n_clicks"), +) +# pylint: disable=unused-argument +def update_notifications(interval, n_clicks): + return generate_table_notifications() - Returns: - html.Table: The html table with service information - """ - services = read_services() - headers = ["Main", "Service name", "Version", "State", "Logs", "Documentation"] - return html.Table( - # Header - [html.Tr([html.Th(col) for col in headers])] - + - # Body - [ - html.Tr( - # Main/Shadow - [ - html.Td("*", className="green-text") - if service["main"] - else html.Td("") - ] - + - # Name - [html.Td(get_service_link(service))] - # Version - + [html.Td(service["version"])] - # Service state - + [html.Td(get_service_state(service))] - # Log link - + [html.Td(get_service_log_link(service))] - # Docs link - + [html.Td(get_service_docs_link(service))] - ) - for service in services - ] - ) +def _service_since(inspection, running): + """Format the running/stopped 'since' timestamp from a service inspection.""" + key = "StartedAt" if running else "FinishedAt" + raw = inspection["State"][key].split(".")[0] + parsed = datetime.strptime(raw, "%Y-%m-%dT%H:%M:%S") + return parsed.strftime("%Y-%m-%d %H:%M:%S") -def get_service_state(service): - """Getter for the state of the service. - The statue of the service is collected from the inspect information. +def build_service_row(service): + """Build a single styled service row Div for the dashboard.""" + is_main = service["main"] + inspection = inspect_service(service["name"], service["version"]) + running = inspection["State"]["Running"] - Args: - service (dict): The service to get the state from. + if running and is_main: + dot_class, state_label = "sdot run live", "Running" + elif running: + dot_class, state_label = "sdot shadow", "Mirroring traffic" + else: + dot_class, state_label = "sdot stop", "Stopped" - Returns: - str: The state of the 'service' - """ - inspection = inspect_service(service["name"], service["version"]) + if is_main: + pin = html.Span("★", className="pin main", title="Main version") + elif not running: + pin = html.Span("○", className="pin", title="Stopped") + else: + pin = html.Span("↗", className="pin", title="Shadow version") - running = inspection["State"]["Running"] - if running: - timestamp = inspection["State"]["StartedAt"] - running_msg = "Running" + if not is_main and running: + name = html.Div( + [service["name"], html.Span("shadow", className="badge shadow")], + className="name", + ) else: - timestamp = inspection["State"]["FinishedAt"] - running_msg = "Stopped" + name = html.Div(service["name"], className="name") - timestamp = datetime.strptime(timestamp.split(".")[0], "%Y-%m-%dT%H:%M:%S") - timestamp = timestamp.strftime("%Y-%m-%d %H:%M:%S") - return f" {running_msg} (since {timestamp})" + id_div = html.Div( + [pin, html.Div([name, html.Div(service["version"], className="ver")])], + className="id", + ) + state_div = html.Div( + [ + html.Span(className=dot_class), + html.Div( + [ + html.Div( + state_label, className="lbl" if running else "lbl stopped" + ), + html.Div( + f"since {_service_since(inspection, running)}", + className="since", + ), + ] + ), + ], + className="state", + ) + actions_div = html.Div( + [get_service_log_link(service), get_service_docs_link(service)], + className="svc-actions", + ) + return html.Div([id_div, state_div, actions_div], className="svc") -def get_link_style(): - return {"color": "white"} +def generate_table_services(): + """Generates service rows with the service information. + + Returns: + html.Div: A Div containing styled service rows. + """ + services = read_services() + if not services: + return html.Div("No services deployed.", className="empty") + return html.Div([build_service_row(service) for service in services]) def get_service_docs_link(service): @@ -223,7 +219,7 @@ def get_service_docs_link(service): return html.A( "Docs", href=(f"{proxy_url}/services/{service['name']}_{service['version']}/docs"), - style=get_link_style(), + className="lnk", ) @@ -236,94 +232,120 @@ def get_service_log_link(service): Returns: html.A: Html.A object with a href to the logs for 'service' """ - logs_end_point = f"{get_external_proxy_url()}/services/~logs" + proxy_url = get_external_proxy_url() return html.A( "Logs", - href=f"{logs_end_point}?name={service['name']}&version={service['version']}" - f"&follow=true&tail={DEFAULT_NUMBER_OF_LOGS}", - style=get_link_style(), - ) - - -def get_service_link(service): - service_endpoint = ( - f"{get_external_proxy_url()}/services/{service['name']}_{service['version']}/" + href=f"{proxy_url}/services/~logs/view" + f"?name={service['name']}&version={service['version']}", + className="lnk", ) - return html.A(service["name"], href=service_endpoint, style=get_link_style()) def generate_table_notifications(): - """Generates a HTML table with the notifications + """Generates notification rows. Returns: - html.Table: The html table with notification information + html.Div: A Div containing styled notification rows. """ notifications = get_notifications() - headers = [ - "Latest Timestamp", - "Service name", - "Version", - "Message", - "Count", - "Severity", - ] - dict_keys = ["timestamp", "service_name", "service_version", "msg", "counter"] - severity_colors = get_severity_colors(notifications) - return html.Table( - # Header - [html.Tr([html.Th(col) for col in headers])] - + - # Body - [ - html.Tr( - [html.Td(notifications[index[0]][key]) for key in dict_keys] - + [severity_colors[index[0]]] - ) - for index in reversed( - sorted(notifications.items(), key=lambda item: item[1]["timestamp"]) - ) - ] - ) + if not notifications: + return html.Div("No notifications.", className="empty") + + severity_map = get_severity_colors(notifications) + rows = [] + for index, _ in reversed( + sorted(notifications.items(), key=lambda item: item[1]["timestamp"]) + ): + notification = notifications[index] + sev_class, sev_label = severity_map[index] + timestamp = notification["timestamp"] + msg = notification["msg"] + + row = html.Div( + [ + html.Span(className=f"sev {sev_class}"), + html.Div( + [ + html.Div(msg, className="msg"), + html.Div( + [ + html.Span(sev_label, className=f"sev-tag {sev_class}"), + html.Span(timestamp), + ], + className="meta", + ), + ] + ), + ], + className="note", + ) + rows.append(row) + + return html.Div(rows) def get_severity_colors(notifications): - """Get the correct color for a severity. + """Get the CSS class and label for each notification's severity. Args: notifications (dict): The notifications. Returns: dict: Dict with the notification hash as the key and - a html.Td object with correct color class for the severity - as value. + a (css_class, label) tuple as the value. + Severity mapping: 0=Info, 1=Warning, 2=Critical. """ - tds = {} + result = {} for index, notification in notifications.items(): - color_class = "severity-info" - text = "Info" - if notification["severity"] == 1: - color_class = "severity-warning" - text = "Warning" - elif notification["severity"] == 2: - color_class = "severity-critical" - text = "Critical" - tds[index] = html.Td(text, className=color_class) - return tds + sev = notification["severity"] + if sev == 1: + result[index] = ("warn", "Warning") + elif sev == 2: + result[index] = ("crit", "Critical") + else: + result[index] = ("info", "Info") + return result app.layout = html.Div( - id="big-app-container", children=[ - # reload intevarl dcc.Interval(id="interval1", interval=5 * 1000, n_intervals=0), - build_user_section(), + # Hidden tabs component kept so update_content callback still resolves + dcc.Tabs(id="app-tabs", value="tab1", style={"display": "none"}), build_banner(), html.Div( - id="app-container", + className="page", children=[ - build_tabs(), - # Main app - html.Div(id="app-content"), + html.Div( + className="grid", + children=[ + # Services panel (hero) + html.Section( + className="panel", + children=[ + html.Div( + className="panel-head", + children=[html.H2("Services")], + ), + html.Div(id="app-content"), + ], + ), + # Notifications panel + html.Section( + className="panel", + children=[ + html.Div( + className="panel-head", + children=[html.H2("Notifications")], + ), + html.Div( + id="notifications-content", + children=[generate_table_notifications()], + ), + ], + ), + ], + ), ], ), ], diff --git a/manager/routers/logging_api.py b/manager/routers/logging_api.py index 6159ed6..437c212 100644 --- a/manager/routers/logging_api.py +++ b/manager/routers/logging_api.py @@ -1,14 +1,17 @@ from datetime import datetime, timedelta import logging import sys -from typing import Union -from fastapi import APIRouter, HTTPException -from fastapi.responses import PlainTextResponse -from manager.constants import log_level, access_logs_enabled +from typing import Optional, Union +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import PlainTextResponse, StreamingResponse, HTMLResponse +from fastapi.templating import Jinja2Templates +from manager.constants import log_level, access_logs_enabled, get_manager_version from manager.runtime_connectors import RTE_CONN ROUTER = APIRouter() +TEMPLATES = Jinja2Templates(directory="manager/templates") + @ROUTER.get("/", response_class=PlainTextResponse) async def manager_logs(since: Union[datetime, None] = None) -> str: @@ -34,6 +37,43 @@ async def manager_logs(since: Union[datetime, None] = None) -> str: return logs +@ROUTER.get("/stream", response_class=StreamingResponse) +async def manager_logs_stream( + tail: Optional[int] = None, + follow: Optional[bool] = False, + since: Union[datetime, None] = None, +) -> str: + """Stream the manager logs (optionally following new output). + + \f + # noqa: DAR101,DAR201 + """ + return StreamingResponse( + RTE_CONN.manager_logs_stream(tail, follow, since), + media_type="text/plain", + headers={"X-Content-Type-Options": "nosniff"}, + ) + + +@ROUTER.get("/view", response_class=HTMLResponse) +def manager_logs_view(request: Request): + """HTML view that streams the manager logs with a follow/auto-scroll toggle. + + \f + # noqa: DAR101,DAR201 + """ + return TEMPLATES.TemplateResponse( + request=request, + name="logs.html", + context={ + "title": "manager", + "subtitle": f"v: {get_manager_version()}", + "stream_url": "/logs/stream?follow=true&tail=400", + "manager_version": get_manager_version(), + }, + ) + + def setup_logging(): root_logger = logging.getLogger() root_logger.setLevel(log_level()) diff --git a/manager/routers/service_api.py b/manager/routers/service_api.py index 35a059e..470f6a5 100644 --- a/manager/routers/service_api.py +++ b/manager/routers/service_api.py @@ -5,11 +5,13 @@ from datetime import datetime from typing import List, Optional from pathlib import Path +from urllib.parse import quote from pydantic import ValidationError, Json from cookiecutter.main import cookiecutter -from fastapi import APIRouter, HTTPException, File, UploadFile, Form -from fastapi.responses import StreamingResponse +from fastapi import APIRouter, HTTPException, File, Request, UploadFile, Form +from fastapi.responses import HTMLResponse, StreamingResponse +from fastapi.templating import Jinja2Templates from docker.errors import ImageNotFound, ImageLoadError from manager.exceptions import ( @@ -37,6 +39,7 @@ DAEPLOY_PREFIX, DAEPLOY_TAR_FILE_NAME, DAEPLOY_DEFAULT_INTERNAL_PORT, + get_manager_version, ) from manager.checks import ( check_service_exists_json_body, @@ -54,6 +57,7 @@ LOGGER = logging.getLogger(__name__) ROUTER = APIRouter() +TEMPLATES = Jinja2Templates(directory="manager/templates") @ROUTER.get("/", response_model=List[ServiceResponse]) @@ -87,7 +91,7 @@ def new_service_from_git_repo(service_request: ServiceGitRequest): """ check_service_exists(service_request.name, service_request.version) - image = build_service_image_s2i(service_request.git_url, service_request) + image = build_service_image_s2i(str(service_request.git_url), service_request) start_service_from_image(image, service_request) return "Accepted" @@ -325,6 +329,25 @@ def assign_main_service(service: BaseService): return "OK" +@ROUTER.get("/~logs/view", response_class=HTMLResponse) +def service_logs_view(request: Request, name: str, version: str): + """HTML view that streams a service's logs with a follow/auto-scroll toggle.""" + stream_url = ( + f"/services/~logs?name={quote(name)}&version={quote(version)}" + "&follow=true&tail=200" + ) + return TEMPLATES.TemplateResponse( + request=request, + name="logs.html", + context={ + "title": name, + "subtitle": f"v{version}", + "stream_url": stream_url, + "manager_version": get_manager_version(), + }, + ) + + @ROUTER.get("/~logs", response_class=StreamingResponse) @async_check_service_exists_query_parameters async def read_service_logs( diff --git a/manager/runtime_connectors.py b/manager/runtime_connectors.py index 75382b9..9e96a6d 100644 --- a/manager/runtime_connectors.py +++ b/manager/runtime_connectors.py @@ -73,7 +73,13 @@ async def service_logs(self, service, tail, follow, since): class LocalDockerConnector(ConnectorBase): CLIENT = docker.from_env() - AIO_CLIENT = aiodocker.Docker() + _AIO_CLIENT = None + + @classmethod + def _get_aio_client(cls): + if cls._AIO_CLIENT is None: + cls._AIO_CLIENT = aiodocker.Docker() + return cls._AIO_CLIENT def __init__(self): # Create our own docker network @@ -395,7 +401,7 @@ async def service_logs( AsyncGenerator[str, None]: Async infinite generator if following, else async finite generator. """ - container = await self.AIO_CLIENT.containers.get( + container = await self._get_aio_client().containers.get( create_container_name(service.name, service.version) ) @@ -429,5 +435,42 @@ def manager_logs(self, since: datetime) -> str: manager_container = self._get_manager_container() return manager_container.logs(since=since) + async def manager_logs_stream( + self, + tail: Optional[int] = None, + follow: Optional[bool] = False, + since: Optional[datetime] = None, + ) -> AsyncGenerator[str, None]: + """Stream the manager's own logs, optionally following new output. + + Args: + tail (Optional[int]): Number of lines from the end. Default all. + follow (Optional[bool]): If the logs should be followed. + since (Optional[datetime]): Get logs since given datetime. + + Yields: + AsyncGenerator[str, None]: Async infinite generator if following, + else async finite generator. + """ + manager = self._get_manager_container() + container = await self._get_aio_client().containers.get(manager.id) + + kwargs = { + "tail": tail if tail else "all", + "stdout": True, + "stderr": True, + "follow": follow, + } + if since: + kwargs["since"] = datetime_to_timestamp(since) + logs = container.log(**kwargs) + + if follow: + async for log in logs: + yield log + else: + for log in await logs: + yield log + RTE_CONN = LocalDockerConnector() diff --git a/manager/templates/daeploy_pickle_template/{{cookiecutter.project_name}}/requirements.txt b/manager/templates/daeploy_pickle_template/{{cookiecutter.project_name}}/requirements.txt index 7ee9e67..3bfd672 100644 --- a/manager/templates/daeploy_pickle_template/{{cookiecutter.project_name}}/requirements.txt +++ b/manager/templates/daeploy_pickle_template/{{cookiecutter.project_name}}/requirements.txt @@ -1,2 +1,2 @@ -daeploy==0.4.6 +daeploy==1.3.1 pandas diff --git a/manager/templates/daeploy_pickle_template/{{cookiecutter.project_name}}/service.py b/manager/templates/daeploy_pickle_template/{{cookiecutter.project_name}}/service.py index aeb3b3a..0cbb3e6 100644 --- a/manager/templates/daeploy_pickle_template/{{cookiecutter.project_name}}/service.py +++ b/manager/templates/daeploy_pickle_template/{{cookiecutter.project_name}}/service.py @@ -46,7 +46,7 @@ def predict(data: dict) -> List[Any]: logger.info(f"Recieved data: \n{data_df}") y_pred = model.predict(data_df) logger.info(f"Predicted: {y_pred}") - return list(y_pred) + return y_pred.tolist() if __name__ == "__main__": diff --git a/manager/templates/login.html b/manager/templates/login.html index 7f3168b..d786d63 100644 --- a/manager/templates/login.html +++ b/manager/templates/login.html @@ -1,38 +1,166 @@ - - - Daeploy - - - - -
-
- -
-
- -
-
- -
-
- -
-
-
-
+ + + + + + Daeploy — Sign in + + + + + + + + + diff --git a/manager/templates/logs.html b/manager/templates/logs.html new file mode 100644 index 0000000..47fc495 --- /dev/null +++ b/manager/templates/logs.html @@ -0,0 +1,203 @@ + + + + + Daeploy — {{ title }} logs + + + + + +
+
+
+ + daeploy +
+ manager v: {{ manager_version }} +
+ +
+
+
+
+
+ {{ title }}{{ subtitle }}
+
+ Live + +
+
+
+
+ +
+
+
+ + + diff --git a/pinned_results.json b/pinned_results.json index 5efd95c..672da14 100644 --- a/pinned_results.json +++ b/pinned_results.json @@ -55,6 +55,21 @@ "websecure" ] }, + { + "entryPoints": [ + "web", + "websecure" + ], + "name": "static_assets@file", + "provider": "file", + "rule": "Host(`localhost`) && PathPrefix(`/assets`)", + "service": "manager_service", + "status": "enabled", + "using": [ + "web", + "websecure" + ] + }, { "entryPoints": [ "web", @@ -106,7 +121,8 @@ "type": "loadbalancer", "usedBy": [ "login_page@file", - "manager@file" + "manager@file", + "static_assets@file" ] }, { @@ -226,6 +242,21 @@ "websecure" ] }, + { + "entryPoints": [ + "web", + "websecure" + ], + "name": "static_assets@file", + "provider": "file", + "rule": "Host(`localhost`) && PathPrefix(`/assets`)", + "service": "manager_service", + "status": "enabled", + "using": [ + "web", + "websecure" + ] + }, { "entryPoints": [ "web", diff --git a/pytest.ini b/pytest.ini index ad16166..c1fd712 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,3 @@ [pytest] -testpaths = tests/ \ No newline at end of file +testpaths = tests/ +timeout = 180 \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index 22749eb..b040166 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -5,11 +5,14 @@ pytest pytest-sphinx pytest-asyncio pytest-pinned -pylint==2.7.4 +pytest-timeout +pylint black flake8 darglint streamlit +httpx async_asgi_testclient scikit-learn -nbconvert \ No newline at end of file +nbconvert +ipykernel \ No newline at end of file diff --git a/requirements_docs.txt b/requirements_docs.txt index fe0c8ea..7dfd045 100644 --- a/requirements_docs.txt +++ b/requirements_docs.txt @@ -1,4 +1,6 @@ -sphinx +# sphinx-multiversion (unmaintained) breaks on Sphinx 8+ (Config.read signature +# change). Pin Sphinx < 8 until multiversion is replaced/updated. +sphinx<8 sphinx-rtd-theme sphinx-multiversion pip-licenses diff --git a/requirements_manager.txt b/requirements_manager.txt index 0cff61c..d8fad21 100644 --- a/requirements_manager.txt +++ b/requirements_manager.txt @@ -1,14 +1,14 @@ -fastapi==0.65.2 -uvicorn==0.13.3 -docker==4.4.1 -aiodocker==0.19.1 -semver==2.13.0 -python-multipart==0.0.5 +fastapi==0.135.3 +uvicorn==0.44.0 +docker==7.1.0 +aiodocker==0.26.0 +semver==3.0.4 +python-multipart==0.0.26 toml==0.10.2 -sqlalchemy==1.3.22 -dash==1.18.1 -pyjwt==2.0.0 -bcrypt==3.2.0 -jinja2==2.11.3 -cookiecutter==1.7.2 -cryptography==3.3.2 +sqlalchemy==2.0.49 +dash==4.1.0 +pyjwt==2.12.1 +bcrypt==5.0.0 +jinja2==3.1.6 +cookiecutter==2.7.1 +cryptography==46.0.7 diff --git a/setup.py b/setup.py index 310a4a8..1020618 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ "Programming Language :: Python :: 3", "Operating System :: OS Independent", ], - python_requires=">=3.6", + python_requires=">=3.9", install_requires=required, entry_points={ "console_scripts": ["daeploy=daeploy.cli.cli:app"], diff --git a/tests/conftest.py b/tests/conftest.py index 3b0218a..09e965e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,10 +44,10 @@ def test_client_logged_in(test_client: TestClient, auth_enabled, database): response = test_client.post( "/auth/login", data={"username": "admin", "password": "admin"}, - allow_redirects=False, + follow_redirects=False, ) # Check that we have access! - response = test_client.get("/auth/verify", allow_redirects=False) + response = test_client.get("/auth/verify", follow_redirects=False) assert response.status_code == 200 yield test_client # Logs out when removing cookies in parent fixture diff --git a/tests/e2e_test/downstream/downstream.py b/tests/e2e_test/downstream/downstream.py index 222014a..f30b704 100644 --- a/tests/e2e_test/downstream/downstream.py +++ b/tests/e2e_test/downstream/downstream.py @@ -1,6 +1,7 @@ """ File used as a service in e2e tests. """ + import logging import time from pydantic import BaseModel diff --git a/tests/e2e_test/e2e_test.py b/tests/e2e_test/e2e_test.py index 4c8cb42..2e4a4ac 100644 --- a/tests/e2e_test/e2e_test.py +++ b/tests/e2e_test/e2e_test.py @@ -1,5 +1,7 @@ import docker import pytest +import sys +import subprocess import time import requests import uuid @@ -9,7 +11,6 @@ import re import shutil -from setuptools import sandbox from pathlib import Path from typer.testing import CliRunner import nbformat @@ -72,22 +73,54 @@ def cli_auth_login(dummy_manager, cli_auth): @pytest.fixture(scope="module") -def pickle_service(cli_auth_login, headers): +def pickle_service(cli_auth_login, dummy_manager, headers): data = { "name": "pickle", "version": "0.1.0", "port": 8000, - "requirements": ["pandas", "sklearn"], + "requirements": ["pandas", "scikit-learn"], } - requests.request( + response = requests.request( "POST", url="http://localhost/services/~pickle", data=data, headers=headers, files={"file": ("filename", open(THIS_DIR / "pickle_e2e_testing.pkl", "rb"))}, ) - time.sleep(5) # Grace period + assert response.status_code == 202, response.text + + # Poll for the pickle service to be reachable; pandas + scikit-learn + # make the s2i build several minutes long, and the service still needs + # time to start up after the container appears. + deadline = time.time() + 800 + reachable = False + while time.time() < deadline: + try: + r = requests.get( + "http://localhost/services/pickle/openapi.json", + headers=headers, + timeout=5, + ) + if r.status_code == 200: + reachable = True + break + except requests.RequestException: + pass + time.sleep(5) + if not reachable: + client = docker.from_env() + print("Containers:", [c.name for c in client.containers.list(all=True)]) + print( + "dummy_manager logs:\n", + dummy_manager.logs(tail=200).decode(errors="replace"), + ) + for c in client.containers.list(all=True): + if "pickle" in c.name: + print( + f"{c.name} status={c.status} logs:\n", + c.logs(tail=200).decode(errors="replace"), + ) try: yield finally: @@ -168,9 +201,16 @@ def generate_requirements_file_for_service(service_folder): the path to the wheel file which contains the daeploy package. """ # TODO: No need to run the setup twice... - sandbox.run_setup( - str(THIS_DIR.parent.parent / "setup.py"), - ["bdist_wheel", "--dist-dir", str(service_folder)], + subprocess.run( + [ + sys.executable, + "setup.py", + "bdist_wheel", + "--dist-dir", + str(service_folder), + ], + cwd=str(THIS_DIR.parent.parent), + check=True, ) with (service_folder / "requirements.txt").open("w") as file_handle: file_handle.write(WHEEL_FILE_NAME) @@ -263,8 +303,8 @@ def test_disable_http_logs_in_entrypoint( logs = logs("downstream") # Check that the logs from inside the entrypoint are logged. assert "This is a correct log!" in logs - assert '"POST /services/downstream_0.1.0/http_logs_2 HTTP/1.1" 200 OK' in logs - assert '"POST /services/downstream_0.1.0/http_logs HTTP/1.1" 200 OK' not in logs + assert '"POST /http_logs_2 HTTP/1.1" 200 OK' in logs + assert '"POST /http_logs HTTP/1.1" 200 OK' not in logs def test_call_service_multiple_cases( @@ -479,6 +519,7 @@ def test_docs_page_from_service_shows_correct_docs( assert "0.1.0" in service_docs.text +@pytest.mark.timeout(900) def test_service_from_pickle_endpoint(dummy_manager, pickle_service, headers): client = docker.from_env() containers = [con.name for con in client.containers.list()] @@ -492,7 +533,14 @@ def test_service_from_pickle_endpoint(dummy_manager, pickle_service, headers): json=data, headers=headers, ) - assert resp.status_code == 200 + if resp.status_code != 200: + for c in client.containers.list(all=True): + if "pickle" in c.name: + print( + f"{c.name} status={c.status} logs:\n", + c.logs(tail=200).decode(errors="replace"), + ) + assert resp.status_code == 200, resp.text # Test documentation started properly response = requests.get( diff --git a/tests/e2e_test/pickle_e2e_testing.pkl b/tests/e2e_test/pickle_e2e_testing.pkl index 7fc5657..f712db5 100644 Binary files a/tests/e2e_test/pickle_e2e_testing.pkl and b/tests/e2e_test/pickle_e2e_testing.pkl differ diff --git a/tests/manager_test/admin_test.py b/tests/manager_test/admin_test.py index 07f2f30..a6b490f 100644 --- a/tests/manager_test/admin_test.py +++ b/tests/manager_test/admin_test.py @@ -38,7 +38,7 @@ def change_user(client, username, password): client.post( "/auth/login", data={"username": username, "password": password}, - allow_redirects=False, + follow_redirects=False, ) diff --git a/tests/manager_test/auth_test.py b/tests/manager_test/auth_test.py index 13c3e7a..159d201 100644 --- a/tests/manager_test/auth_test.py +++ b/tests/manager_test/auth_test.py @@ -33,7 +33,7 @@ def test_login_page(exclude_middleware): def test_verification_without_auth(database): - response = client.get("/auth/verify", allow_redirects=False) + response = client.get("/auth/verify", follow_redirects=False) assert response.status_code == 200 @@ -43,39 +43,39 @@ def test_failed_login(database, auth_enabled): response = client.post( "/auth/login", data={"username": "admin", "password": "wrongpassword"}, - allow_redirects=False, + follow_redirects=False, ) assert response.status_code == 303 # No access after - response = client.get("/auth/verify", allow_redirects=False) + response = client.get("/auth/verify", follow_redirects=False) assert response.status_code == 303 def test_cookie_token(database, auth_enabled): # No access from beginning - response = client.get("/auth/verify", allow_redirects=False) + response = client.get("/auth/verify", follow_redirects=False) assert response.status_code == 303 # Login response = client.post( "/auth/login", data={"username": "admin", "password": "admin"}, - allow_redirects=False, + follow_redirects=False, ) assert response.status_code == 303 # Check that we have access! - response = client.get("/auth/verify", allow_redirects=False) + response = client.get("/auth/verify", follow_redirects=False) assert response.status_code == 200 # Logout - response = client.get("/auth/logout", allow_redirects=False) + response = client.get("/auth/logout", follow_redirects=False) assert response.status_code == 303 # No access at the end - response = client.get("/auth/verify", allow_redirects=False) + response = client.get("/auth/verify", follow_redirects=False) assert response.status_code == 303 @@ -85,7 +85,7 @@ def test_API_token(clear_cookies, database, auth_enabled): response = client.get( "/auth/verify", headers={"Authorization": f"Bearer mumbojumbo"}, - allow_redirects=True, + follow_redirects=True, ) assert response.status_code == 401 @@ -111,7 +111,7 @@ def test_API_token(clear_cookies, database, auth_enabled): response = client.get( "/auth/verify", headers={"Authorization": f"Bearer {token}"}, - allow_redirects=False, + follow_redirects=False, ) assert response.status_code == 200 @@ -119,7 +119,7 @@ def test_API_token(clear_cookies, database, auth_enabled): response = client.get( "/auth/verify", headers={"Authorization": f"Bearer {token}"}, - allow_redirects=True, + follow_redirects=True, ) assert response.status_code == 200 @@ -131,6 +131,6 @@ def test_API_token(clear_cookies, database, auth_enabled): response = client.get( "/auth/verify", headers={"Authorization": f"Bearer {token}"}, - allow_redirects=True, + follow_redirects=True, ) assert response.status_code == 401 diff --git a/tests/manager_test/endpoint_test.py b/tests/manager_test/endpoint_test.py index 4098bcb..fa66180 100644 --- a/tests/manager_test/endpoint_test.py +++ b/tests/manager_test/endpoint_test.py @@ -10,6 +10,7 @@ import pytest from async_asgi_testclient import TestClient as AsyncTestClient from docker.errors import ImageNotFound +from fastapi.exceptions import ResponseValidationError from fastapi.testclient import TestClient from manager import proxy from manager.routers import logging_api, notification_api, service_api @@ -17,7 +18,6 @@ from manager.constants import DAEPLOY_DEFAULT_S2I_BUILD_IMAGE, get_manager_version from manager.data_models.request_models import BaseService - client = TestClient(app) async_client = AsyncTestClient(app) @@ -195,12 +195,12 @@ def test_post_services_git_request( response = client.post("/services/~git", json=req) assert response.status_code == 202 - service_api.run_s2i.assert_called_with( - url="https://github.com/sclorg/django-ex", - build_image=DAEPLOY_DEFAULT_S2I_BUILD_IMAGE, - name=SERVICE_NAME, - version=SERVICE_VERSION, - ) + service_api.run_s2i.assert_called_once() + call_kwargs = service_api.run_s2i.call_args[1] + assert str(call_kwargs["url"]) == "https://github.com/sclorg/django-ex" + assert call_kwargs["build_image"] == DAEPLOY_DEFAULT_S2I_BUILD_IMAGE + assert call_kwargs["name"] == SERVICE_NAME + assert call_kwargs["version"] == SERVICE_VERSION assert response.json() == "Accepted" @@ -228,12 +228,12 @@ def test_post_services_git_request_changed_builder_image( response = client.post("/services/~git", json=req) assert response.status_code == 202 - service_api.run_s2i.assert_called_with( - url="https://github.com/sclorg/django-ex", - build_image="centos/python-38-centos7", - name=SERVICE_NAME, - version=SERVICE_VERSION, - ) + service_api.run_s2i.assert_called_once() + call_kwargs = service_api.run_s2i.call_args[1] + assert str(call_kwargs["url"]) == "https://github.com/sclorg/django-ex" + assert call_kwargs["build_image"] == "centos/python-38-centos7" + assert call_kwargs["name"] == SERVICE_NAME + assert call_kwargs["version"] == SERVICE_VERSION assert response.json() == "Accepted" @@ -438,8 +438,10 @@ def test_service_delete(mocked_docker_connection, database): ) service_name = SERVICE_NAME service_version = SERVICE_VERSION - response = client.delete( - "/services/", json={"name": service_name, "version": service_version} + response = client.request( + "DELETE", + "/services/", + json={"name": service_name, "version": service_version}, ) assert response.status_code == 200 @@ -462,7 +464,8 @@ def test_service_delete_keep_image(mocked_docker_connection, database): ) service_name = SERVICE_NAME service_version = SERVICE_VERSION - response = client.delete( + response = client.request( + "DELETE", "/services/", json={"name": service_name, "version": service_version}, params={"remove_image": False}, @@ -576,7 +579,7 @@ def test_service_inspection(mocked_docker_connection): service_name = SERVICE_NAME service_version = SERVICE_VERSION - with pytest.raises(pydantic.ValidationError): + with pytest.raises((pydantic.ValidationError, ResponseValidationError)): client.get( f"/services/~inspection?name={service_name}&version={service_version}" ) diff --git a/tests/manager_test/local_docker_connection_test.py b/tests/manager_test/local_docker_connection_test.py index 98791e3..608b99a 100644 --- a/tests/manager_test/local_docker_connection_test.py +++ b/tests/manager_test/local_docker_connection_test.py @@ -211,13 +211,13 @@ async def test_service_logs(local_docker_connection): def check_required_inspection_keys(container_info): - assert set(InspectResponse.schema()["required"]).issubset( + assert set(InspectResponse.model_json_schema()["required"]).issubset( set(container_info.keys()) ) - assert set(NetworkSettingsResponse.schema()["required"]).issubset( + assert set(NetworkSettingsResponse.model_json_schema()["required"]).issubset( set(container_info["NetworkSettings"].keys()) ) - assert set(StateResponse.schema()["required"]).issubset( + assert set(StateResponse.model_json_schema()["required"]).issubset( set(container_info["State"].keys()) ) diff --git a/tests/manager_test/notifications_test.py b/tests/manager_test/notifications_test.py index 612c608..2877c8c 100644 --- a/tests/manager_test/notifications_test.py +++ b/tests/manager_test/notifications_test.py @@ -97,7 +97,7 @@ def test_email_notification_not_send_when_frozen(email_func, notifications_dict) notification_api.new_notification(notification_3) notification_api.new_notification(notification_3) # The email func is only called once! - email_func.called_once() + email_func.assert_called_once() @patch("manager.routers.notification_api._send_notification_as_email") diff --git a/tests/manager_test/test_ui_redesign.py b/tests/manager_test/test_ui_redesign.py new file mode 100644 index 0000000..f89834f --- /dev/null +++ b/tests/manager_test/test_ui_redesign.py @@ -0,0 +1,127 @@ +# tests/manager_test/test_ui_redesign.py +from pathlib import Path + +ASSETS = Path("manager/assets") + + +def test_tokens_css_defines_palette(): + css = (ASSETS / "tokens.css").read_text() + for var, val in [ + ("--ground", "#0E1320"), + ("--accent", "#5EE6D0"), + ("--text", "#E7ECF5"), + ("--crit", "#F2585B"), + ]: + assert f"{var}:{val}" in css.replace(" ", ""), f"missing {var}" + assert "@font-face" in css + assert "url(fonts/" in css.replace(" ", ""), "fonts must be referenced relatively" + assert "http://" not in css and "https://" not in css + + +def test_fonts_bundled(): + woff2 = list((ASSETS / "fonts").glob("*.woff2")) + assert len(woff2) >= 4, "expected Inter + JetBrains Mono weights" + assert all(f.stat().st_size > 5000 for f in woff2), "woff2 files look empty" + + +def test_logo_is_local_svg(): + svg = (ASSETS / "daeploy_mark.svg").read_text() + assert " "ValueError: I/O operation on closed file". + killer.join() assert len(streamed_logs.stdout) > len(first_logs.stdout) diff --git a/tests/sdk_test/daeploy_test.py b/tests/sdk_test/daeploy_test.py index d16c186..1cac8ca 100644 --- a/tests/sdk_test/daeploy_test.py +++ b/tests/sdk_test/daeploy_test.py @@ -508,7 +508,7 @@ def test_local_invocation_pydantic_validation(): assert valid_entrypoint_method_args(32, "Urban") == "hello" # Args of wrong type! - with pytest.raises(pydantic.error_wrappers.ValidationError): + with pytest.raises(pydantic.ValidationError): wrapped(32, "Urban") assert wrapped("Urban", 32) == "hello" @@ -589,7 +589,8 @@ def test_entrypoint_get(): client = TestClient(service.app) req = {"name": "Rune", "age": 100} - response = client.get( + response = client.request( + "GET", "/valid_entrypoint_method_args", json=req, headers={"accept": "application/json"},