feat: s3-compatible storage support

backend/api/alembic/versions/c1d2e3f4a5b6_add_s3_destination_storage.py

@@ -0,0 +1,39 @@
+"""add s3 destination storage
+
+Revision ID: c1d2e3f4a5b6
+Revises: a1b2c3d4e5f6
+Create Date: 2026-05-23 12:00:00.000000
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+
+revision: str = "c1d2e3f4a5b6"
+down_revision: Union[str, Sequence[str], None] = "a1b2c3d4e5f6"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ALTER TYPE ... ADD VALUE cannot run inside a transaction block.
+    # See a1b2c3d4e5f6 for the same pattern.
+    with op.get_context().autocommit_block():
+        op.execute("ALTER TYPE storagetype ADD VALUE IF NOT EXISTS 'S3'")
+
+    op.add_column(
+        "destinations",
+        sa.Column("config_data", sa.JSON(), nullable=True),
+    )
+    op.add_column(
+        "destinations",
+        sa.Column("quota_bytes", sa.BigInteger(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("destinations", "quota_bytes")
+    op.drop_column("destinations", "config_data")
+    # Postgres cannot remove an enum value; the 'S3' value remains.

backend/api/app/routers/destinations.py

@@ -55,3 +55,12 @@ async def delete_destination(
     user: User = Depends(require_admin),
 ) -> None:
     await destination_service.delete_destination(db, str(destination_id))
+
+
+@router.post("/{destination_id}/test")
+async def test_destination(
+    destination_id: uuid.UUID,
+    db: AsyncSession = Depends(get_db),
+    user: User = Depends(require_admin),
+) -> dict:
+    return await destination_service.test_destination(db, str(destination_id))

backend/api/app/services/destination_service.py

@@ -8,13 +8,31 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.repositories import destination_repo
+from shared.crypto import encrypt_field
 from shared.enums import StorageType
 from shared.models import Destination, User
 from shared.schemas import DestinationCreate, DestinationRead, DestinationUpdate
+from shared.storage import (
+    SECRET_KEYS,
+    StorageConfigError,
+    validate_destination_config,
+)
+from shared.storage_backends import StorageBackendError, get_storage_backend
 
 
-async def _get_available_bytes(destination: Destination) -> int | None:
-    """Get available disk space for a destination. Returns None if unknown."""
+_APP_SECRET = os.environ.get("JWT_SECRET", "")
+
+
+async def _get_available_bytes(
+    destination: Destination, used_bytes: int
+) -> int | None:
+    """Available capacity for a destination.
+
+    If a quota is set, returns `quota - used` regardless of storage type
+    (clamped to >=0). Otherwise falls back to filesystem free space for LOCAL
+    and `None` for remote storage where there is no portable signal.
+    """
+    quota = destination.quota_bytes
     if destination.storage_type == StorageType.LOCAL:
         path = Path(destination.path)
         try:
@@ -23,10 +41,21 @@ def _check() -> int:
                 while not check_path.exists() and check_path.parent != check_path:
                     check_path = check_path.parent
                 return shutil.disk_usage(check_path).free
-            return await asyncio.to_thread(_check)
+            disk_free = await asyncio.to_thread(_check)
         except OSError:
-            return None
-    # S3, GCS, etc. — no simple way to know
+            disk_free = None
+        if quota:
+            quota_remaining = max(0, quota - used_bytes)
+            return (
+                min(disk_free, quota_remaining)
+                if disk_free is not None
+                else quota_remaining
+            )
+        return disk_free
+
+    # S3, GCS, etc. — no portable free-space API.
+    if quota:
+        return max(0, quota - used_bytes)
     return None
 
 
@@ -45,14 +74,25 @@ async def _enrich_destinations(
         dest_stats = stats.get(dest.id, {})
         read.repo_count = dest_stats.get("repo_count", 0)
         read.used_bytes = dest_stats.get("used_bytes", 0)
-        read.available_bytes = await _get_available_bytes(dest)
+        read.available_bytes = await _get_available_bytes(dest, read.used_bytes)
         result.append(read)
     return result
 
 
 BACKUP_ROOT = Path(os.environ.get("BACKUP_DIR", "/data/backups"))
 
 
+def _encrypt_config_secrets(config: dict | None) -> dict | None:
+    """Encrypt sensitive keys in a config dict before storing in DB."""
+    if not config:
+        return config
+    out = dict(config)
+    for key in SECRET_KEYS:
+        if key in out and out[key] and _APP_SECRET:
+            out[key] = encrypt_field(out[key], _APP_SECRET)
+    return out
+
+
 async def create_destination(
     db: AsyncSession, user: User, body: DestinationCreate
 ) -> DestinationRead:
@@ -72,7 +112,9 @@ async def create_destination(
     destination = Destination(
         alias=body.alias,
         storage_type=body.storage_type,
-        path=body.path,
+        path=body.path or "",
+        config_data=_encrypt_config_secrets(body.config_data),
+        quota_bytes=body.quota_bytes,
         is_default=body.is_default,
         created_by=user.id,
     )
@@ -113,6 +155,40 @@ async def update_destination(
                 detail=f"Local path must be under {BACKUP_ROOT}",
             )
         path.mkdir(parents=True, exist_ok=True)
+
+    if "config_data" in update_data:
+        # Merge partial config with the stored config so callers can omit
+        # fields they don't want to change — notably secrets, which the read
+        # schema strips and the UI therefore won't echo back on edit.
+        existing = dict(destination.config_data or {})
+        incoming = dict(update_data["config_data"] or {})
+        merged = {**existing, **{k: v for k, v in incoming.items() if v not in ("", None)}}
+
+        # Re-encrypt only secrets the user explicitly provided this round.
+        for key in SECRET_KEYS:
+            if key in incoming and incoming[key] and _APP_SECRET:
+                merged[key] = encrypt_field(incoming[key], _APP_SECRET)
+
+        # Validate the merged shape. Since stored secrets are ciphertext,
+        # substitute placeholders so the "non-empty" checks still pass without
+        # the validator needing to know about encryption.
+        validate_input = dict(merged)
+        for key in SECRET_KEYS:
+            if validate_input.get(key):
+                validate_input[key] = "<encrypted>"
+        try:
+            validate_destination_config(
+                destination.storage_type,
+                validate_input,
+                update_data.get("path", destination.path),
+            )
+        except StorageConfigError as e:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)
+            ) from e
+
+        update_data["config_data"] = merged
+
     if update_data.get("is_default"):
         await destination_repo.clear_default(db)
 
@@ -136,3 +212,23 @@ async def delete_destination(db: AsyncSession, dest_id: str) -> None:
 
     await destination_repo.delete(db, destination)
     await db.commit()
+
+
+async def test_destination(db: AsyncSession, dest_id: str) -> dict:
+    """Probe a destination's connectivity / permissions. Returns {ok, message}."""
+    destination = await destination_repo.get_by_id(db, uuid.UUID(dest_id))
+    if not destination:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Destination not found")
+
+    def _probe() -> tuple[bool, str]:
+        try:
+            backend = get_storage_backend(destination)
+            backend.validate_config()
+            return True, "Destination is reachable"
+        except StorageBackendError as e:
+            return False, str(e)
+        except Exception as e:
+            return False, f"Unexpected error: {e}"
+
+    ok, message = await asyncio.to_thread(_probe)
+    return {"ok": ok, "message": message}

backend/api/pyproject.toml

@@ -16,6 +16,7 @@ dependencies = [
     "python-dotenv>=1.0",
     "croniter>=6.0",
     "apprise>=1.9",
+    "boto3>=1.34",
     "shared",
 ]
 

backend/api/tests/test_destination_service.py

@@ -80,3 +80,71 @@ async def test_delete_destination(db_session, admin_user):
     results = await destination_service.list_destinations(db_session)
     aliases = {d.alias for d in results}
     assert "ToDelete" not in aliases
+
+
+# --- S3 destinations + secret handling ---
+
+
+_S3_CONFIG = {
+    "bucket": "my-backups",
+    "region": "us-east-1",
+    "access_key_id": "AKIA1234",
+    "secret_access_key": "TOPSECRET",
+}
+
+
+@patch("app.services.destination_service._APP_SECRET", "test-secret-32bytes-padding-pad")
+async def test_create_s3_encrypts_secret(db_session, admin_user):
+    body = DestinationCreate(
+        alias="My S3", storage_type=StorageType.S3, config_data=dict(_S3_CONFIG)
+    )
+    result = await destination_service.create_destination(db_session, admin_user, body)
+
+    # Read schema must strip the secret — even though we just wrote it.
+    assert "secret_access_key" not in (result.config_data or {})
+    # Access key id IS still visible (it's an identifier, not a secret).
+    assert result.config_data["access_key_id"] == "AKIA1234"
+
+    # Inspect the raw row: stored secret is ciphertext, not the plaintext.
+    db_session.expire_all()
+    raw = await db_session.get(Destination, result.id)
+    assert raw.config_data["secret_access_key"] != "TOPSECRET"
+    assert raw.config_data["secret_access_key"]  # but not empty
+
+
+@patch("app.services.destination_service._APP_SECRET", "test-secret-32bytes-padding-pad")
+async def test_update_preserves_existing_secret_when_omitted(
+    db_session, admin_user
+):
+    body = DestinationCreate(
+        alias="My S3", storage_type=StorageType.S3, config_data=dict(_S3_CONFIG)
+    )
+    created = await destination_service.create_destination(db_session, admin_user, body)
+
+    raw_before = await db_session.get(Destination, created.id)
+    stored_secret = raw_before.config_data["secret_access_key"]
+
+    # Update bucket only — omit secret_access_key entirely.
+    from shared.schemas import DestinationUpdate
+    new_cfg = {k: v for k, v in _S3_CONFIG.items() if k != "secret_access_key"}
+    new_cfg["bucket"] = "new-bucket"
+    await destination_service.update_destination(
+        db_session, str(created.id),
+        DestinationUpdate(config_data=new_cfg),
+    )
+
+    db_session.expire_all()
+    raw_after = await db_session.get(Destination, created.id)
+    assert raw_after.config_data["bucket"] == "new-bucket"
+    # Stored secret unchanged — same ciphertext, same plaintext after decrypt.
+    assert raw_after.config_data["secret_access_key"] == stored_secret
+
+
+async def test_quota_field_round_trip(db_session, admin_user):
+    body = DestinationCreate(
+        alias="With Quota",
+        path=f"{tempfile.gettempdir()}/quota",
+        quota_bytes=10 * 1024**3,
+    )
+    result = await destination_service.create_destination(db_session, admin_user, body)
+    assert result.quota_bytes == 10 * 1024**3

backend/backup-core/pyproject.toml

@@ -10,11 +10,12 @@ dependencies = [
     "python-dotenv>=1.0",
     "croniter>=6.0",
     "apprise>=1.9",
+    "boto3>=1.34",
     "shared",
 ]
 
 [project.optional-dependencies]
-dev = ["pytest>=8.0"]
+dev = ["pytest>=8.0", "moto[s3]>=5.0"]
 
 [tool.uv.sources]
 shared = { path = "../shared", editable = true }