From 52ff0b4be05c21103c60c15f6e5d8300906eca2b Mon Sep 17 00:00:00 2001 From: Roman Hrokholskyi Date: Sat, 23 May 2026 10:36:49 +0400 Subject: [PATCH 1/2] feat: s3-compatible storage support --- ...c1d2e3f4a5b6_add_s3_destination_storage.py | 39 ++ backend/api/app/routers/destinations.py | 9 + .../api/app/services/destination_service.py | 110 +++- backend/api/pyproject.toml | 1 + backend/api/tests/test_destination_service.py | 68 +++ backend/api/uv.lock | 51 ++ backend/backup-core/pyproject.toml | 3 +- .../backup-core/services/backup_service.py | 93 +-- .../backup-core/services/restore_service.py | 32 +- .../backup-core/tests/test_storage_local.py | 51 ++ backend/backup-core/tests/test_storage_s3.py | 76 +++ .../tests/test_storage_validation.py | 81 +++ backend/backup-core/uv.lock | 184 ++++++ backend/shared/shared/destination_stats.py | 30 + backend/shared/shared/enums.py | 1 + backend/shared/shared/models.py | 2 + backend/shared/shared/schemas.py | 33 +- backend/shared/shared/storage.py | 112 ++++ .../shared/storage_backends/__init__.py | 74 +++ .../shared/shared/storage_backends/local.py | 60 ++ backend/shared/shared/storage_backends/s3.py | 148 +++++ frontend/src/app/destinations/page.tsx | 565 +++++++++++++++--- frontend/src/lib/api.ts | 32 +- 23 files changed, 1699 insertions(+), 156 deletions(-) create mode 100644 backend/api/alembic/versions/c1d2e3f4a5b6_add_s3_destination_storage.py create mode 100644 backend/backup-core/tests/test_storage_local.py create mode 100644 backend/backup-core/tests/test_storage_s3.py create mode 100644 backend/backup-core/tests/test_storage_validation.py create mode 100644 backend/shared/shared/destination_stats.py create mode 100644 backend/shared/shared/storage.py create mode 100644 backend/shared/shared/storage_backends/__init__.py create mode 100644 backend/shared/shared/storage_backends/local.py create mode 100644 backend/shared/shared/storage_backends/s3.py diff --git a/backend/api/alembic/versions/c1d2e3f4a5b6_add_s3_destination_storage.py b/backend/api/alembic/versions/c1d2e3f4a5b6_add_s3_destination_storage.py new file mode 100644 index 0000000..fd06cf1 --- /dev/null +++ b/backend/api/alembic/versions/c1d2e3f4a5b6_add_s3_destination_storage.py @@ -0,0 +1,39 @@ +"""add s3 destination storage + +Revision ID: c1d2e3f4a5b6 +Revises: a1b2c3d4e5f6 +Create Date: 2026-05-23 12:00:00.000000 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + + +revision: str = "c1d2e3f4a5b6" +down_revision: Union[str, Sequence[str], None] = "a1b2c3d4e5f6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ALTER TYPE ... ADD VALUE cannot run inside a transaction block. + # See a1b2c3d4e5f6 for the same pattern. + with op.get_context().autocommit_block(): + op.execute("ALTER TYPE storagetype ADD VALUE IF NOT EXISTS 'S3'") + + op.add_column( + "destinations", + sa.Column("config_data", sa.JSON(), nullable=True), + ) + op.add_column( + "destinations", + sa.Column("quota_bytes", sa.BigInteger(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("destinations", "quota_bytes") + op.drop_column("destinations", "config_data") + # Postgres cannot remove an enum value; the 'S3' value remains. diff --git a/backend/api/app/routers/destinations.py b/backend/api/app/routers/destinations.py index 401fc6d..b4f705d 100644 --- a/backend/api/app/routers/destinations.py +++ b/backend/api/app/routers/destinations.py @@ -55,3 +55,12 @@ async def delete_destination( user: User = Depends(require_admin), ) -> None: await destination_service.delete_destination(db, str(destination_id)) + + +@router.post("/{destination_id}/test") +async def test_destination( + destination_id: uuid.UUID, + db: AsyncSession = Depends(get_db), + user: User = Depends(require_admin), +) -> dict: + return await destination_service.test_destination(db, str(destination_id)) diff --git a/backend/api/app/services/destination_service.py b/backend/api/app/services/destination_service.py index 4986cae..7be3f11 100644 --- a/backend/api/app/services/destination_service.py +++ b/backend/api/app/services/destination_service.py @@ -8,13 +8,31 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.repositories import destination_repo +from shared.crypto import encrypt_field from shared.enums import StorageType from shared.models import Destination, User from shared.schemas import DestinationCreate, DestinationRead, DestinationUpdate +from shared.storage import ( + SECRET_KEYS, + StorageConfigError, + validate_destination_config, +) +from shared.storage_backends import StorageBackendError, get_storage_backend -async def _get_available_bytes(destination: Destination) -> int | None: - """Get available disk space for a destination. Returns None if unknown.""" +_APP_SECRET = os.environ.get("JWT_SECRET", "") + + +async def _get_available_bytes( + destination: Destination, used_bytes: int +) -> int | None: + """Available capacity for a destination. + + If a quota is set, returns `quota - used` regardless of storage type + (clamped to >=0). Otherwise falls back to filesystem free space for LOCAL + and `None` for remote storage where there is no portable signal. + """ + quota = destination.quota_bytes if destination.storage_type == StorageType.LOCAL: path = Path(destination.path) try: @@ -23,10 +41,21 @@ def _check() -> int: while not check_path.exists() and check_path.parent != check_path: check_path = check_path.parent return shutil.disk_usage(check_path).free - return await asyncio.to_thread(_check) + disk_free = await asyncio.to_thread(_check) except OSError: - return None - # S3, GCS, etc. — no simple way to know + disk_free = None + if quota: + quota_remaining = max(0, quota - used_bytes) + return ( + min(disk_free, quota_remaining) + if disk_free is not None + else quota_remaining + ) + return disk_free + + # S3, GCS, etc. — no portable free-space API. + if quota: + return max(0, quota - used_bytes) return None @@ -45,7 +74,7 @@ async def _enrich_destinations( dest_stats = stats.get(dest.id, {}) read.repo_count = dest_stats.get("repo_count", 0) read.used_bytes = dest_stats.get("used_bytes", 0) - read.available_bytes = await _get_available_bytes(dest) + read.available_bytes = await _get_available_bytes(dest, read.used_bytes) result.append(read) return result @@ -53,6 +82,17 @@ async def _enrich_destinations( BACKUP_ROOT = Path(os.environ.get("BACKUP_DIR", "/data/backups")) +def _encrypt_config_secrets(config: dict | None) -> dict | None: + """Encrypt sensitive keys in a config dict before storing in DB.""" + if not config: + return config + out = dict(config) + for key in SECRET_KEYS: + if key in out and out[key] and _APP_SECRET: + out[key] = encrypt_field(out[key], _APP_SECRET) + return out + + async def create_destination( db: AsyncSession, user: User, body: DestinationCreate ) -> DestinationRead: @@ -72,7 +112,9 @@ async def create_destination( destination = Destination( alias=body.alias, storage_type=body.storage_type, - path=body.path, + path=body.path or "", + config_data=_encrypt_config_secrets(body.config_data), + quota_bytes=body.quota_bytes, is_default=body.is_default, created_by=user.id, ) @@ -113,6 +155,40 @@ async def update_destination( detail=f"Local path must be under {BACKUP_ROOT}", ) path.mkdir(parents=True, exist_ok=True) + + if "config_data" in update_data: + # Merge partial config with the stored config so callers can omit + # fields they don't want to change — notably secrets, which the read + # schema strips and the UI therefore won't echo back on edit. + existing = dict(destination.config_data or {}) + incoming = dict(update_data["config_data"] or {}) + merged = {**existing, **{k: v for k, v in incoming.items() if v not in ("", None)}} + + # Re-encrypt only secrets the user explicitly provided this round. + for key in SECRET_KEYS: + if key in incoming and incoming[key] and _APP_SECRET: + merged[key] = encrypt_field(incoming[key], _APP_SECRET) + + # Validate the merged shape. Since stored secrets are ciphertext, + # substitute placeholders so the "non-empty" checks still pass without + # the validator needing to know about encryption. + validate_input = dict(merged) + for key in SECRET_KEYS: + if validate_input.get(key): + validate_input[key] = "" + try: + validate_destination_config( + destination.storage_type, + validate_input, + update_data.get("path", destination.path), + ) + except StorageConfigError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(e) + ) from e + + update_data["config_data"] = merged + if update_data.get("is_default"): await destination_repo.clear_default(db) @@ -136,3 +212,23 @@ async def delete_destination(db: AsyncSession, dest_id: str) -> None: await destination_repo.delete(db, destination) await db.commit() + + +async def test_destination(db: AsyncSession, dest_id: str) -> dict: + """Probe a destination's connectivity / permissions. Returns {ok, message}.""" + destination = await destination_repo.get_by_id(db, uuid.UUID(dest_id)) + if not destination: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Destination not found") + + def _probe() -> tuple[bool, str]: + try: + backend = get_storage_backend(destination) + backend.validate_config() + return True, "Destination is reachable" + except StorageBackendError as e: + return False, str(e) + except Exception as e: + return False, f"Unexpected error: {e}" + + ok, message = await asyncio.to_thread(_probe) + return {"ok": ok, "message": message} diff --git a/backend/api/pyproject.toml b/backend/api/pyproject.toml index 9256be5..94b6c0b 100644 --- a/backend/api/pyproject.toml +++ b/backend/api/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "python-dotenv>=1.0", "croniter>=6.0", "apprise>=1.9", + "boto3>=1.34", "shared", ] diff --git a/backend/api/tests/test_destination_service.py b/backend/api/tests/test_destination_service.py index e8bd92d..55ac384 100644 --- a/backend/api/tests/test_destination_service.py +++ b/backend/api/tests/test_destination_service.py @@ -80,3 +80,71 @@ async def test_delete_destination(db_session, admin_user): results = await destination_service.list_destinations(db_session) aliases = {d.alias for d in results} assert "ToDelete" not in aliases + + +# --- S3 destinations + secret handling --- + + +_S3_CONFIG = { + "bucket": "my-backups", + "region": "us-east-1", + "access_key_id": "AKIA1234", + "secret_access_key": "TOPSECRET", +} + + +@patch("app.services.destination_service._APP_SECRET", "test-secret-32bytes-padding-pad") +async def test_create_s3_encrypts_secret(db_session, admin_user): + body = DestinationCreate( + alias="My S3", storage_type=StorageType.S3, config_data=dict(_S3_CONFIG) + ) + result = await destination_service.create_destination(db_session, admin_user, body) + + # Read schema must strip the secret — even though we just wrote it. + assert "secret_access_key" not in (result.config_data or {}) + # Access key id IS still visible (it's an identifier, not a secret). + assert result.config_data["access_key_id"] == "AKIA1234" + + # Inspect the raw row: stored secret is ciphertext, not the plaintext. + db_session.expire_all() + raw = await db_session.get(Destination, result.id) + assert raw.config_data["secret_access_key"] != "TOPSECRET" + assert raw.config_data["secret_access_key"] # but not empty + + +@patch("app.services.destination_service._APP_SECRET", "test-secret-32bytes-padding-pad") +async def test_update_preserves_existing_secret_when_omitted( + db_session, admin_user +): + body = DestinationCreate( + alias="My S3", storage_type=StorageType.S3, config_data=dict(_S3_CONFIG) + ) + created = await destination_service.create_destination(db_session, admin_user, body) + + raw_before = await db_session.get(Destination, created.id) + stored_secret = raw_before.config_data["secret_access_key"] + + # Update bucket only — omit secret_access_key entirely. + from shared.schemas import DestinationUpdate + new_cfg = {k: v for k, v in _S3_CONFIG.items() if k != "secret_access_key"} + new_cfg["bucket"] = "new-bucket" + await destination_service.update_destination( + db_session, str(created.id), + DestinationUpdate(config_data=new_cfg), + ) + + db_session.expire_all() + raw_after = await db_session.get(Destination, created.id) + assert raw_after.config_data["bucket"] == "new-bucket" + # Stored secret unchanged — same ciphertext, same plaintext after decrypt. + assert raw_after.config_data["secret_access_key"] == stored_secret + + +async def test_quota_field_round_trip(db_session, admin_user): + body = DestinationCreate( + alias="With Quota", + path=f"{tempfile.gettempdir()}/quota", + quota_bytes=10 * 1024**3, + ) + result = await destination_service.create_destination(db_session, admin_user, body) + assert result.quota_bytes == 10 * 1024**3 diff --git a/backend/api/uv.lock b/backend/api/uv.lock index a2c286a..3bff3e5 100644 --- a/backend/api/uv.lock +++ b/backend/api/uv.lock @@ -201,6 +201,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" }, ] +[[package]] +name = "boto3" +version = "1.43.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/4b/616367e871ce3f1cb3e8545a97736b6331b9fb081497f2d44c5b2aa6959d/boto3-1.43.14.tar.gz", hash = "sha256:5c0a994b3182061ee101812e721100717a4d664f9f4ceaf4a86b6d032ce9fc2d", size = 113142, upload-time = "2026-05-22T19:28:47.861Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/00/59cb9329c18e2d3aa23062ceaa87d065f2e81e7d2931df24d64e9a7815aa/boto3-1.43.14-py3-none-any.whl", hash = "sha256:574335744656cfed0b362a0a0467aaf2eb2bf15526edcd02d31d3c661f4b09e4", size = 140536, upload-time = "2026-05-22T19:28:46.49Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/3c/798d2f7deb118241930c7c6bcfb0b970d3f0245bf580700663199aeed2c3/botocore-1.43.14.tar.gz", hash = "sha256:b9e500737e43d2f147c9d4e23b54360335e77d4c0ba90a318f51b65e06cb8516", size = 15382604, upload-time = "2026-05-22T19:28:36.363Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/7e/6e64821077cd2efc4aa51b7d638fb6d48e1c7c450201c529fbaf1de8bfd3/botocore-1.43.14-py3-none-any.whl", hash = "sha256:1f4a2a95ea78c10398e78431e98c1fe47adb54a7b10a32975144c1f541186658", size = 15061424, upload-time = "2026-05-22T19:28:32.682Z" }, +] + [[package]] name = "celery" version = "5.6.2" @@ -531,6 +559,7 @@ dependencies = [ { name = "apprise" }, { name = "asyncpg" }, { name = "bcrypt" }, + { name = "boto3" }, { name = "celery" }, { name = "croniter" }, { name = "fastapi" }, @@ -558,6 +587,7 @@ requires-dist = [ { name = "apprise", specifier = ">=1.9" }, { name = "asyncpg", specifier = ">=0.30" }, { name = "bcrypt", specifier = ">=4.2" }, + { name = "boto3", specifier = ">=1.34" }, { name = "celery", specifier = ">=5.4" }, { name = "croniter", specifier = ">=6.0" }, { name = "fastapi", specifier = ">=0.115" }, @@ -701,6 +731,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + [[package]] name = "kombu" version = "5.6.2" @@ -1099,6 +1138,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, ] +[[package]] +name = "s3transfer" +version = "0.17.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/ec/7c692cde9125b77e84b307354d4fb705f98b8ccad59a036d5957ca75bfc3/s3transfer-0.17.0.tar.gz", hash = "sha256:9edeb6d1c3c2f89d6050348548834ad8289610d886e5bf7b7207728bd43ce33a", size = 155337, upload-time = "2026-04-29T22:07:36.33Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/72/c6c32d2b657fa3dad1de340254e14390b1e334ce38268b7ad51abda3c8c2/s3transfer-0.17.0-py3-none-any.whl", hash = "sha256:ce3801712acf4ad3e89fb9990df97b4972e93f4b3b0004d214be5bce12814c20", size = 86811, upload-time = "2026-04-29T22:07:34.966Z" }, +] + [[package]] name = "shared" version = "0.1.0" diff --git a/backend/backup-core/pyproject.toml b/backend/backup-core/pyproject.toml index 24c4ccf..1aaaf05 100644 --- a/backend/backup-core/pyproject.toml +++ b/backend/backup-core/pyproject.toml @@ -10,11 +10,12 @@ dependencies = [ "python-dotenv>=1.0", "croniter>=6.0", "apprise>=1.9", + "boto3>=1.34", "shared", ] [project.optional-dependencies] -dev = ["pytest>=8.0"] +dev = ["pytest>=8.0", "moto[s3]>=5.0"] [tool.uv.sources] shared = { path = "../shared", editable = true } diff --git a/backend/backup-core/services/backup_service.py b/backend/backup-core/services/backup_service.py index ddb1b92..47b16fd 100644 --- a/backend/backup-core/services/backup_service.py +++ b/backend/backup-core/services/backup_service.py @@ -24,7 +24,9 @@ from services.git_service import scrub_credentials from services.encryption import get_encryption_provider from services.notifications import NotificationEvent -from shared.enums import ArchiveFormat, JobStatus, RepoStatus +from shared.destination_stats import sum_destination_used_bytes +from shared.storage_backends import StorageBackendError, get_storage_backend +from shared.enums import ArchiveFormat, JobStatus, RepoStatus, StorageType from shared.models import BackupSnapshot logger = logging.getLogger(__name__) @@ -100,12 +102,15 @@ def run_backup(session: Session, job_id: str) -> dict: session.commit() return {"error": "Destination not found"} - # Validate destination path - dest_path = Path(destination.path).resolve() - if not dest_path.is_absolute(): - backup_job_repo.mark_failed(session, job, datetime.now(timezone.utc), None, "Invalid destination path") + # Build the storage backend for this destination (encrypted secrets decrypted here) + try: + backend = get_storage_backend(destination) + except (StorageBackendError, KeyError, ValueError) as e: + backup_job_repo.mark_failed( + session, job, datetime.now(timezone.utc), None, f"Storage backend error: {e}" + ) session.commit() - return {"error": "Invalid destination path"} + return {"error": f"Storage backend error: {e}"} # Mark as running started_at = datetime.now(timezone.utc) @@ -201,13 +206,8 @@ def run_backup(session: Session, job_id: str) -> dict: encryption_key_id = enc_key.id log_lines.append("Encryption complete") - # Move final file to destination - dest_dir = dest_path - dest_dir.mkdir(parents=True, exist_ok=True) - final_path = dest_dir / archive_name - shutil.move(str(tmp_archive), str(final_path)) - - backup_size = final_path.stat().st_size + # Hand the final archive to the storage backend (local move or S3 upload) + backup_size = backend.upload(tmp_archive, archive_name) log_lines.append(f"Archive saved: {backup_size} bytes") now = datetime.now(timezone.utc) @@ -257,27 +257,52 @@ def run_backup(session: Session, job_id: str) -> dict: ), ) elif job.status == JobStatus.SUCCEEDED: - # Check disk space on destination - try: - usage = shutil.disk_usage(str(dest_path)) - free_pct = (usage.free / usage.total) * 100 - if free_pct < 10: - send_notifications( - session, - NotificationEvent( - event_type="disk_space_low", - title="Disk space low", - repo_name=repo.name, - repo_url=repo.url, - message=( - f"Destination '{destination.alias}' has " - f"{free_pct:.1f}% free space " - f"({usage.free // (1024 ** 3)} GB remaining)" + # Two independent capacity signals — LOCAL free-space % (existing) + # and any-type user-configured quota_bytes (new). + if destination.storage_type == StorageType.LOCAL: + try: + usage = shutil.disk_usage(str(destination.path)) + free_pct = (usage.free / usage.total) * 100 + if free_pct < 10: + send_notifications( + session, + NotificationEvent( + event_type="disk_space_low", + title="Disk space low", + repo_name=repo.name, + repo_url=repo.url, + message=( + f"Destination '{destination.alias}' has " + f"{free_pct:.1f}% free space " + f"({usage.free // (1024 ** 3)} GB remaining)" + ), + timestamp=datetime.now(timezone.utc), ), - timestamp=datetime.now(timezone.utc), - ), - ) - except OSError: - pass # Destination path may be remote / unmounted + ) + except OSError: + pass # Destination path may be remote / unmounted + + if destination.quota_bytes: + try: + used = sum_destination_used_bytes(session, destination.id) + pct = (used / destination.quota_bytes) * 100 + if pct >= 90: + send_notifications( + session, + NotificationEvent( + event_type="disk_space_low", + title="Storage quota nearly full", + repo_name=repo.name, + repo_url=repo.url, + message=( + f"Destination '{destination.alias}' is at " + f"{pct:.1f}% of its {destination.quota_bytes // (1024 ** 3)} GB quota " + f"({used // (1024 ** 3)} GB used)" + ), + timestamp=datetime.now(timezone.utc), + ), + ) + except Exception as e: + logger.warning("Quota check failed for destination %s: %s", destination.id, e) return {"job_id": job_id, "status": job.status.value} diff --git a/backend/backup-core/services/restore_service.py b/backend/backup-core/services/restore_service.py index 71f41e6..315904c 100644 --- a/backend/backup-core/services/restore_service.py +++ b/backend/backup-core/services/restore_service.py @@ -20,6 +20,7 @@ from services.encryption import get_encryption_provider from services.notifications import NotificationEvent from shared.enums import ArchiveFormat, JobStatus +from shared.storage_backends import get_storage_backend logger = logging.getLogger(__name__) @@ -48,12 +49,7 @@ def run_restore(session: Session, restore_job_id: str) -> dict: if not destination: raise RuntimeError("Destination no longer exists") - dest_root = Path(destination.path).resolve() - archive_path = (dest_root / snapshot.artifact_filename).resolve() - if not archive_path.is_relative_to(dest_root): - raise RuntimeError("Invalid archive path") - if not archive_path.is_file(): - raise RuntimeError(f"Archive file not found: {archive_path}") + backend = get_storage_backend(destination) log_lines.append(f"Loaded snapshot artifact: {snapshot.artifact_filename}") log_lines.append(f"Format: {snapshot.archive_format.value}") @@ -61,6 +57,10 @@ def run_restore(session: Session, restore_job_id: str) -> dict: with tempfile.TemporaryDirectory() as tmpdir: tmp_path = Path(tmpdir) + # Fetch the archive from its storage backend into the temp dir. + archive_path = tmp_path / snapshot.artifact_filename + backend.download(snapshot.artifact_filename, archive_path) + # Decrypt if needed if snapshot.archive_format == ArchiveFormat.TAR_GZ_GPG: if not snapshot.encryption_key_id: @@ -235,16 +235,14 @@ def run_restore_preview(session: Session, preview_id: str) -> dict: if not destination: raise RuntimeError("Destination no longer exists") - dest_root = Path(destination.path).resolve() - archive_path = (dest_root / snapshot.artifact_filename).resolve() - if not archive_path.is_relative_to(dest_root): - raise RuntimeError("Invalid archive path") - if not archive_path.is_file(): - raise RuntimeError(f"Archive file not found: {archive_path}") + backend = get_storage_backend(destination) with tempfile.TemporaryDirectory() as tmpdir: tmp_path = Path(tmpdir) + archive_path = tmp_path / snapshot.artifact_filename + backend.download(snapshot.artifact_filename, archive_path) + # Decrypt if needed if snapshot.archive_format == ArchiveFormat.TAR_GZ_GPG: if not snapshot.encryption_key_id: @@ -335,16 +333,14 @@ def run_detailed_preview(session: Session, preview_id: str) -> dict: if not destination: raise RuntimeError("Destination no longer exists") - dest_root = Path(destination.path).resolve() - archive_path = (dest_root / snapshot.artifact_filename).resolve() - if not archive_path.is_relative_to(dest_root): - raise RuntimeError("Invalid archive path") - if not archive_path.is_file(): - raise RuntimeError(f"Archive file not found: {archive_path}") + backend = get_storage_backend(destination) with tempfile.TemporaryDirectory() as tmpdir: tmp_path = Path(tmpdir) + archive_path = tmp_path / snapshot.artifact_filename + backend.download(snapshot.artifact_filename, archive_path) + # Decrypt if needed if snapshot.archive_format == ArchiveFormat.TAR_GZ_GPG: if not snapshot.encryption_key_id: diff --git a/backend/backup-core/tests/test_storage_local.py b/backend/backup-core/tests/test_storage_local.py new file mode 100644 index 0000000..e0ce241 --- /dev/null +++ b/backend/backup-core/tests/test_storage_local.py @@ -0,0 +1,51 @@ +import pytest + +from shared.storage_backends import StorageBackendError +from shared.storage_backends.local import LocalStorageBackend + + +@pytest.fixture +def backend(tmp_path): + return LocalStorageBackend(tmp_path) + + +def test_round_trip(tmp_path, backend): + src = tmp_path / "input.bin" + src.write_bytes(b"hello world") + + size = backend.upload(src, "out.bin") + assert size == len(b"hello world") + assert not src.exists() # move semantics + assert (tmp_path / "out.bin").read_bytes() == b"hello world" + + dst = tmp_path / "downloaded.bin" + backend.download("out.bin", dst) + assert dst.read_bytes() == b"hello world" + # download is copy — source still there. + assert (tmp_path / "out.bin").exists() + + backend.delete("out.bin") + assert not (tmp_path / "out.bin").exists() + + +def test_download_missing_raises(tmp_path, backend): + with pytest.raises(StorageBackendError, match="not found"): + backend.download("nope", tmp_path / "x") + + +def test_delete_missing_is_noop(backend): + backend.delete("nope") # must not raise + + +@pytest.mark.parametrize("bad_key", ["../escape", "/abs/path", "a/../b"]) +def test_traversal_rejected(tmp_path, backend, bad_key): + src = tmp_path / "src" + src.write_bytes(b"x") + with pytest.raises(StorageBackendError, match="Invalid storage key"): + backend.upload(src, bad_key) + + +def test_validate_config_writable_dir(backend, tmp_path): + backend.validate_config() + # Probe file should be cleaned up. + assert list(tmp_path.iterdir()) == [] diff --git a/backend/backup-core/tests/test_storage_s3.py b/backend/backup-core/tests/test_storage_s3.py new file mode 100644 index 0000000..371f731 --- /dev/null +++ b/backend/backup-core/tests/test_storage_s3.py @@ -0,0 +1,76 @@ +import boto3 +import pytest +from moto import mock_aws + +from shared.storage_backends import StorageBackendError +from shared.storage_backends.s3 import S3StorageBackend + + +BUCKET = "gitbacker-test" + + +@pytest.fixture +def s3_backend(): + with mock_aws(): + # Provision the bucket the backend will write into. + boto3.client("s3", region_name="us-east-1").create_bucket(Bucket=BUCKET) + yield S3StorageBackend( + bucket=BUCKET, + prefix="archives", + region="us-east-1", + endpoint_url=None, + access_key_id="test", + secret_access_key="test", + ) + + +def test_round_trip(tmp_path, s3_backend): + src = tmp_path / "in.bin" + payload = b"a" * 1024 + src.write_bytes(payload) + + size = s3_backend.upload(src, "repo_2026.tar.gz") + assert size == len(payload) + + dst = tmp_path / "out.bin" + s3_backend.download("repo_2026.tar.gz", dst) + assert dst.read_bytes() == payload + + s3_backend.delete("repo_2026.tar.gz") + with pytest.raises(StorageBackendError): + s3_backend.download("repo_2026.tar.gz", tmp_path / "x") + + +def test_validate_config(s3_backend): + s3_backend.validate_config() + + +def test_validate_config_missing_bucket(): + with mock_aws(): + backend = S3StorageBackend( + bucket="does-not-exist", + region="us-east-1", + access_key_id="test", + secret_access_key="test", + ) + with pytest.raises(StorageBackendError, match="does not exist"): + backend.validate_config() + + +def test_traversal_rejected(tmp_path, s3_backend): + src = tmp_path / "in" + src.write_bytes(b"x") + with pytest.raises(StorageBackendError, match="Invalid storage key"): + s3_backend.upload(src, "../escape") + + +def test_prefix_join(tmp_path, s3_backend): + src = tmp_path / "in.bin" + src.write_bytes(b"x") + s3_backend.upload(src, "myfile") + + # Verify the object actually landed under the configured prefix. + raw = boto3.client("s3", region_name="us-east-1").get_object( + Bucket=BUCKET, Key="archives/myfile" + ) + assert raw["Body"].read() == b"x" diff --git a/backend/backup-core/tests/test_storage_validation.py b/backend/backup-core/tests/test_storage_validation.py new file mode 100644 index 0000000..fc14fb8 --- /dev/null +++ b/backend/backup-core/tests/test_storage_validation.py @@ -0,0 +1,81 @@ +import pytest + +from shared.enums import StorageType +from shared.storage import StorageConfigError, validate_destination_config + + +def test_local_requires_path(): + with pytest.raises(StorageConfigError, match="path is required"): + validate_destination_config(StorageType.LOCAL, None, None) + + +def test_local_rejects_config(): + with pytest.raises(StorageConfigError, match="must be empty"): + validate_destination_config(StorageType.LOCAL, {"bucket": "x"}, "/data") + + +def test_s3_requires_config(): + with pytest.raises(StorageConfigError, match="config_data is required"): + validate_destination_config(StorageType.S3, None, None) + + +@pytest.mark.parametrize("bucket", ["AB", "Capital", "with_underscore", "..bad", ""]) +def test_s3_bucket_name_validation(bucket): + cfg = { + "bucket": bucket, + "access_key_id": "k", + "secret_access_key": "s", + } + with pytest.raises(StorageConfigError): + validate_destination_config(StorageType.S3, cfg, "") + + +def test_s3_minimal_valid(): + validate_destination_config( + StorageType.S3, + { + "bucket": "my-backups", + "access_key_id": "AKIA1234", + "secret_access_key": "secret", + }, + "", + ) + + +def test_s3_missing_secret_rejected(): + cfg = {"bucket": "my-backups", "access_key_id": "AKIA"} + with pytest.raises(StorageConfigError, match="secret_access_key"): + validate_destination_config(StorageType.S3, cfg, "") + + +def test_s3_private_endpoint_blocked(): + cfg = { + "bucket": "my-backups", + "endpoint_url": "http://127.0.0.1:9000", + "access_key_id": "k", + "secret_access_key": "s", + } + with pytest.raises(StorageConfigError, match="private/internal"): + validate_destination_config(StorageType.S3, cfg, "") + + +def test_s3_private_endpoint_allowed_with_escape_hatch(): + cfg = { + "bucket": "my-backups", + "endpoint_url": "http://127.0.0.1:9000", + "allow_private_endpoint": True, + "access_key_id": "k", + "secret_access_key": "s", + } + validate_destination_config(StorageType.S3, cfg, "") + + +def test_s3_bad_prefix(): + cfg = { + "bucket": "my-backups", + "prefix": "/leading/slash", + "access_key_id": "k", + "secret_access_key": "s", + } + with pytest.raises(StorageConfigError, match="prefix must not start"): + validate_destination_config(StorageType.S3, cfg, "") diff --git a/backend/backup-core/uv.lock b/backend/backup-core/uv.lock index c461f28..aeb534b 100644 --- a/backend/backup-core/uv.lock +++ b/backend/backup-core/uv.lock @@ -50,6 +50,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" }, ] +[[package]] +name = "boto3" +version = "1.43.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/4b/616367e871ce3f1cb3e8545a97736b6331b9fb081497f2d44c5b2aa6959d/boto3-1.43.14.tar.gz", hash = "sha256:5c0a994b3182061ee101812e721100717a4d664f9f4ceaf4a86b6d032ce9fc2d", size = 113142, upload-time = "2026-05-22T19:28:47.861Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/00/59cb9329c18e2d3aa23062ceaa87d065f2e81e7d2931df24d64e9a7815aa/boto3-1.43.14-py3-none-any.whl", hash = "sha256:574335744656cfed0b362a0a0467aaf2eb2bf15526edcd02d31d3c661f4b09e4", size = 140536, upload-time = "2026-05-22T19:28:46.49Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/3c/798d2f7deb118241930c7c6bcfb0b970d3f0245bf580700663199aeed2c3/botocore-1.43.14.tar.gz", hash = "sha256:b9e500737e43d2f147c9d4e23b54360335e77d4c0ba90a318f51b65e06cb8516", size = 15382604, upload-time = "2026-05-22T19:28:36.363Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/7e/6e64821077cd2efc4aa51b7d638fb6d48e1c7c450201c529fbaf1de8bfd3/botocore-1.43.14-py3-none-any.whl", hash = "sha256:1f4a2a95ea78c10398e78431e98c1fe47adb54a7b10a32975144c1f541186658", size = 15061424, upload-time = "2026-05-22T19:28:32.682Z" }, +] + [[package]] name = "celery" version = "5.6.2" @@ -361,6 +389,7 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "apprise" }, + { name = "boto3" }, { name = "celery" }, { name = "croniter" }, { name = "psycopg2-binary" }, @@ -372,14 +401,17 @@ dependencies = [ [package.optional-dependencies] dev = [ + { name = "moto", extra = ["s3"] }, { name = "pytest" }, ] [package.metadata] requires-dist = [ { name = "apprise", specifier = ">=1.9" }, + { name = "boto3", specifier = ">=1.34" }, { name = "celery", specifier = ">=5.4" }, { name = "croniter", specifier = ">=6.0" }, + { name = "moto", extras = ["s3"], marker = "extra == 'dev'", specifier = ">=5.0" }, { name = "psycopg2-binary", specifier = ">=2.9" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, { name = "python-dotenv", specifier = ">=1.0" }, @@ -450,6 +482,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + [[package]] name = "kombu" version = "5.6.2" @@ -474,6 +515,93 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36", size = 108180, upload-time = "2026-02-09T14:57:25.787Z" }, ] +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "moto" +version = "5.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "boto3" }, + { name = "botocore" }, + { name = "cryptography" }, + { name = "requests" }, + { name = "responses" }, + { name = "werkzeug" }, + { name = "xmltodict" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f6/e9/c38202162db2e76623176be9f1dbc9aa41228ffa91ee8da2d3986082c3e3/moto-5.2.1.tar.gz", hash = "sha256:ccb2f3e1dfa82e50e054bda98b0be708d244d2668364dcc1d45e8d3de6091bde", size = 8634437, upload-time = "2026-05-10T19:11:57.286Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/79/8085b7c1ecd48d0535c3c8444a1d8df2926e457dce8e55fabc332a382c9c/moto-5.2.1-py3-none-any.whl", hash = "sha256:19d2fbd6e613aa5b4e364c52cd5d3cea371643a0f4210689a703227bd2924c5c", size = 6671379, upload-time = "2026-05-10T19:11:53.543Z" }, +] + +[package.optional-dependencies] +s3 = [ + { name = "py-partiql-parser" }, + { name = "pyyaml" }, +] + [[package]] name = "oauthlib" version = "3.3.1" @@ -554,6 +682,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/36/9c0c326fe3a4227953dfb29f5d0c8ae3b8eb8c1cd2967aa569f50cb3c61f/psycopg2_binary-2.9.11-cp314-cp314-win_amd64.whl", hash = "sha256:4012c9c954dfaccd28f94e84ab9f94e12df76b4afb22331b1f0d3154893a6316", size = 2803913, upload-time = "2025-10-10T11:13:57.058Z" }, ] +[[package]] +name = "py-partiql-parser" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/56/7a/a0f6bda783eb4df8e3dfd55973a1ac6d368a89178c300e1b5b91cd181e5e/py_partiql_parser-0.6.3.tar.gz", hash = "sha256:09cecf916ce6e3da2c050f0cb6106166de42c33d34a078ec2eb19377ea70389a", size = 17456, upload-time = "2025-10-18T13:56:13.441Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/33/a7cbfccc39056a5cf8126b7aab4c8bafbedd4f0ca68ae40ecb627a2d2cd3/py_partiql_parser-0.6.3-py2.py3-none-any.whl", hash = "sha256:deb0769c3346179d2f590dcbde556f708cdb929059fb654bad75f4cf6e07f582", size = 23752, upload-time = "2025-10-18T13:56:12.256Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -792,6 +929,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, ] +[[package]] +name = "responses" +version = "0.26.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/58/1fb6de3503428196df78638f991ec8095274f1ee9723e272ee4d9ff0092b/responses-0.26.1.tar.gz", hash = "sha256:2eb3218553cc8f79b57d257bac23af5e1bf381f5b9390b1767816f0843e01dc2", size = 83088, upload-time = "2026-05-21T19:56:39.747Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/31/6a620b4427d546b9e7cca8b3b8c5f0559d9cef2bb9eedcda7f73c1473c19/responses-0.26.1-py3-none-any.whl", hash = "sha256:8aacc4586eb08fb2208ef64a9eb4258d9b0c6e6f4260845f2f018ab847495345", size = 35502, upload-time = "2026-05-21T19:56:38.046Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.17.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/ec/7c692cde9125b77e84b307354d4fb705f98b8ccad59a036d5957ca75bfc3/s3transfer-0.17.0.tar.gz", hash = "sha256:9edeb6d1c3c2f89d6050348548834ad8289610d886e5bf7b7207728bd43ce33a", size = 155337, upload-time = "2026-04-29T22:07:36.33Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/72/c6c32d2b657fa3dad1de340254e14390b1e334ce38268b7ad51abda3c8c2/s3transfer-0.17.0-py3-none-any.whl", hash = "sha256:ce3801712acf4ad3e89fb9990df97b4972e93f4b3b0004d214be5bce12814c20", size = 86811, upload-time = "2026-04-29T22:07:34.966Z" }, +] + [[package]] name = "shared" version = "0.1.0" @@ -936,3 +1099,24 @@ sdist = { url = "https://files.pythonhosted.org/packages/35/a2/8e3becb46433538a3 wheels = [ { url = "https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", size = 94189, upload-time = "2026-02-06T19:19:39.646Z" }, ] + +[[package]] +name = "werkzeug" +version = "3.1.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/b2/381be8cfdee792dd117872481b6e378f85c957dd7c5bca38897b08f765fd/werkzeug-3.1.8.tar.gz", hash = "sha256:9bad61a4268dac112f1c5cd4630a56ede601b6ed420300677a869083d70a4c44", size = 875852, upload-time = "2026-04-02T18:49:14.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/8c/2e650f2afeb7ee576912636c23ddb621c91ac6a98e66dc8d29c3c69446e1/werkzeug-3.1.8-py3-none-any.whl", hash = "sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50", size = 226459, upload-time = "2026-04-02T18:49:12.72Z" }, +] + +[[package]] +name = "xmltodict" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/70/80f3b7c10d2630aa66414bf23d210386700aa390547278c789afa994fd7e/xmltodict-1.0.4.tar.gz", hash = "sha256:6d94c9f834dd9e44514162799d344d815a3a4faec913717a9ecbfa5be1bb8e61", size = 26124, upload-time = "2026-02-22T02:21:22.074Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/34/98a2f52245f4d47be93b580dae5f9861ef58977d73a79eb47c58f1ad1f3a/xmltodict-1.0.4-py3-none-any.whl", hash = "sha256:a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a", size = 13580, upload-time = "2026-02-22T02:21:21.039Z" }, +] diff --git a/backend/shared/shared/destination_stats.py b/backend/shared/shared/destination_stats.py new file mode 100644 index 0000000..acf2732 --- /dev/null +++ b/backend/shared/shared/destination_stats.py @@ -0,0 +1,30 @@ +"""Single-destination usage stats — shared between API (async) and worker (sync). + +The API's `destination_repo.get_stats` does the same calculation in a batched, +async form for the dashboard. This module exposes the equivalent computation +for the worker's per-backup quota check so both sides answer "how much space +is this destination using?" with the same SQL. +""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from .enums import JobStatus +from .models import BackupJob, Repository + + +def sum_destination_used_bytes(session: Session, dest_id: uuid.UUID) -> int: + """Sum bytes of all successful backup artifacts for a destination.""" + result = session.execute( + select(func.coalesce(func.sum(BackupJob.backup_size_bytes), 0)) + .join(Repository, Repository.id == BackupJob.repository_id) + .where( + Repository.destination_id == dest_id, + BackupJob.status == JobStatus.SUCCEEDED, + ) + ) + return int(result.scalar() or 0) diff --git a/backend/shared/shared/enums.py b/backend/shared/shared/enums.py index 3fa9616..c3504f0 100644 --- a/backend/shared/shared/enums.py +++ b/backend/shared/shared/enums.py @@ -34,6 +34,7 @@ class TriggerType(str, enum.Enum): class StorageType(str, enum.Enum): LOCAL = "local" + S3 = "s3" class RepoPermission(enum.IntEnum): diff --git a/backend/shared/shared/models.py b/backend/shared/shared/models.py index ba54a69..9f4c98d 100644 --- a/backend/shared/shared/models.py +++ b/backend/shared/shared/models.py @@ -104,6 +104,8 @@ class Destination(Base): Enum(StorageType), nullable=False, default=StorageType.LOCAL ) path: Mapped[str] = mapped_column(String(1024), nullable=False) + config_data: Mapped[dict | None] = mapped_column(JSON, nullable=True) + quota_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) is_default: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) created_by: Mapped[uuid.UUID] = mapped_column( UUID(as_uuid=True), ForeignKey("users.id", ondelete="RESTRICT"), nullable=False diff --git a/backend/shared/shared/schemas.py b/backend/shared/shared/schemas.py index d9bb702..ecb8a4f 100644 --- a/backend/shared/shared/schemas.py +++ b/backend/shared/shared/schemas.py @@ -15,6 +15,7 @@ TriggerType, UserRole, ) +from .storage import SECRET_KEYS, StorageConfigError, validate_destination_config # --- Auth --- @@ -101,15 +102,29 @@ def validate_password(cls, v: str) -> str: class DestinationCreate(BaseModel): alias: str storage_type: StorageType = StorageType.LOCAL - path: str + path: str | None = None + config_data: dict | None = None + quota_bytes: int | None = None is_default: bool = False + @model_validator(mode="after") + def _validate_storage_config(self) -> "DestinationCreate": + if self.quota_bytes is not None and self.quota_bytes <= 0: + raise ValueError("quota_bytes must be positive") + try: + validate_destination_config(self.storage_type, self.config_data, self.path) + except StorageConfigError as e: + raise ValueError(str(e)) from e + return self + class DestinationRead(BaseModel): id: uuid.UUID alias: str storage_type: StorageType path: str + config_data: dict | None = None + quota_bytes: int | None = None is_default: bool created_by: uuid.UUID created_at: datetime @@ -119,12 +134,28 @@ class DestinationRead(BaseModel): model_config = {"from_attributes": True} + @model_validator(mode="after") + def _scrub_secrets(self) -> "DestinationRead": + if self.config_data: + self.config_data = { + k: v for k, v in self.config_data.items() if k not in SECRET_KEYS + } + return self + class DestinationUpdate(BaseModel): alias: str | None = None path: str | None = None + config_data: dict | None = None + quota_bytes: int | None = None is_default: bool | None = None + @model_validator(mode="after") + def _check_quota(self) -> "DestinationUpdate": + if self.quota_bytes is not None and self.quota_bytes <= 0: + raise ValueError("quota_bytes must be positive (use null to clear)") + return self + # --- Repositories --- diff --git a/backend/shared/shared/storage.py b/backend/shared/shared/storage.py new file mode 100644 index 0000000..56aee72 --- /dev/null +++ b/backend/shared/shared/storage.py @@ -0,0 +1,112 @@ +"""Destination-type-aware config validation. + +Pure-Python validation; does not import boto3. Both the API service (POST/PATCH +validation + test endpoint) and backup-core (boot-time backend construction) +consume these helpers so the per-type config shapes have one source of truth. +""" + +from __future__ import annotations + +import ipaddress +import re +import socket +from urllib.parse import urlparse + +from .enums import StorageType + + +class StorageConfigError(ValueError): + """Raised when a destination's config_data is malformed for its storage_type.""" + + +_BUCKET_RE = re.compile(r"^[a-z0-9][a-z0-9.\-]{1,61}[a-z0-9]$") +_BUCKET_IP_RE = re.compile(r"^(\d{1,3}\.){3}\d{1,3}$") + + +def _validate_bucket_name(name: str) -> None: + if not name: + raise StorageConfigError("bucket is required") + if not _BUCKET_RE.match(name): + raise StorageConfigError( + "bucket must be 3-63 chars, lowercase letters/digits/dots/hyphens, " + "starting and ending with a letter or digit" + ) + if ".." in name or ".-" in name or "-." in name: + raise StorageConfigError("bucket must not contain adjacent dots/hyphens") + if _BUCKET_IP_RE.match(name): + raise StorageConfigError("bucket must not be formatted as an IP address") + + +def _validate_prefix(prefix: str) -> None: + if prefix.startswith("/"): + raise StorageConfigError("prefix must not start with '/'") + if ".." in prefix.split("/"): + raise StorageConfigError("prefix must not contain '..' segments") + + +def _check_not_private(url: str, *, field: str) -> None: + """SSRF guard: reject URLs that resolve to private/loopback/link-local IPs.""" + parsed = urlparse(url) + hostname = parsed.hostname or "" + if not hostname: + raise StorageConfigError(f"{field} has no hostname") + try: + addr = ipaddress.ip_address(socket.gethostbyname(hostname)) + except socket.gaierror as e: + raise StorageConfigError(f"Cannot resolve hostname: {hostname}") from e + if addr.is_private or addr.is_loopback or addr.is_link_local: + raise StorageConfigError( + f"{field} must not point to a private/internal address ({hostname}). " + "For MinIO on localhost, set allow_private_endpoint=true." + ) + + +def validate_destination_config( + storage_type: StorageType, + config: dict | None, + path: str | None, +) -> None: + """Validate config_data + path against the schema for storage_type. + + Raises StorageConfigError on failure. Successful return implies the + inputs are well-formed enough for the storage backend constructor. + """ + match storage_type: + case StorageType.LOCAL: + if not path: + raise StorageConfigError("path is required for LOCAL destinations") + if config: + raise StorageConfigError( + "config_data must be empty for LOCAL destinations" + ) + + case StorageType.S3: + if not isinstance(config, dict) or not config: + raise StorageConfigError("config_data is required for S3 destinations") + + _validate_bucket_name(str(config.get("bucket", ""))) + + prefix = config.get("prefix") or "" + if prefix: + _validate_prefix(str(prefix)) + + endpoint_url = config.get("endpoint_url") or "" + if endpoint_url: + if not endpoint_url.startswith(("http://", "https://")): + raise StorageConfigError( + "endpoint_url must start with http:// or https://" + ) + if not config.get("allow_private_endpoint"): + _check_not_private(endpoint_url, field="endpoint_url") + + if not config.get("access_key_id"): + raise StorageConfigError("access_key_id is required for S3") + if not config.get("secret_access_key"): + raise StorageConfigError("secret_access_key is required for S3") + + case _: + raise StorageConfigError(f"Unsupported storage type: {storage_type}") + + +SECRET_KEYS: tuple[str, ...] = ("secret_access_key",) +"""Config keys whose values are encrypted at rest and must never appear in read schemas.""" diff --git a/backend/shared/shared/storage_backends/__init__.py b/backend/shared/shared/storage_backends/__init__.py new file mode 100644 index 0000000..ef25e87 --- /dev/null +++ b/backend/shared/shared/storage_backends/__init__.py @@ -0,0 +1,74 @@ +"""Storage backend Protocol and factory. + +Storage backends are responsible for the *transport* of backup archives between +the worker's temp dir and a destination (local FS, S3-compatible bucket, ...). +Encryption and archive packaging happen before upload; restore downloads into +a temp dir and decrypts/extracts there. + +Used-bytes accounting deliberately lives in `shared.destination_stats`, not on +this interface — both the API and the worker derive used_bytes from BackupJob +rows so storage-type-specific bucket walks are not required. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Protocol + +from shared.crypto import decrypt_field +from shared.enums import StorageType +from shared.models import Destination + + +_APP_SECRET = os.environ.get("JWT_SECRET", "") + + +class StorageBackendError(RuntimeError): + """Raised on user-actionable storage failures (bucket missing, auth wrong, etc.).""" + + +class StorageBackend(Protocol): + """Minimal interface for destination storage.""" + + def upload(self, local_path: Path, remote_key: str) -> int: + """Upload local_path under remote_key. Returns bytes written.""" + ... + + def download(self, remote_key: str, local_path: Path) -> None: + """Download remote_key to local_path. Raises StorageBackendError if missing.""" + ... + + def delete(self, remote_key: str) -> None: + """Delete remote_key. No-op if not present.""" + ... + + def validate_config(self) -> None: + """Connectivity / permission probe. Raises StorageBackendError on failure.""" + ... + + +def get_storage_backend(destination: Destination) -> StorageBackend: + """Build the right backend for a destination, decrypting secrets as needed.""" + if destination.storage_type == StorageType.LOCAL: + from .local import LocalStorageBackend + + return LocalStorageBackend(Path(destination.path)) + + if destination.storage_type == StorageType.S3: + from .s3 import S3StorageBackend + + cfg = destination.config_data or {} + secret = cfg.get("secret_access_key", "") + if _APP_SECRET and secret: + secret = decrypt_field(secret, _APP_SECRET) + return S3StorageBackend( + bucket=cfg["bucket"], + prefix=cfg.get("prefix") or "", + region=cfg.get("region") or None, + endpoint_url=cfg.get("endpoint_url") or None, + access_key_id=cfg["access_key_id"], + secret_access_key=secret, + ) + + raise ValueError(f"Unknown storage type: {destination.storage_type}") diff --git a/backend/shared/shared/storage_backends/local.py b/backend/shared/shared/storage_backends/local.py new file mode 100644 index 0000000..fd29391 --- /dev/null +++ b/backend/shared/shared/storage_backends/local.py @@ -0,0 +1,60 @@ +"""Local-filesystem storage backend. + +Wraps the inline shutil.move logic that used to live in backup_service.run_backup, +plus a defensive traversal guard inherited from the previous restore_service +inline `is_relative_to` check. +""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +from . import StorageBackend, StorageBackendError + + +class LocalStorageBackend: + """Atomic-rename-based storage on a local directory.""" + + def __init__(self, root: Path) -> None: + self._root = root.resolve() + if not self._root.is_absolute(): + raise StorageBackendError(f"Destination path must be absolute: {root}") + + def _resolve_key(self, remote_key: str) -> Path: + if remote_key.startswith("/") or ".." in remote_key.split("/"): + raise StorageBackendError(f"Invalid storage key: {remote_key!r}") + return self._root / remote_key + + def upload(self, local_path: Path, remote_key: str) -> int: + target = self._resolve_key(remote_key) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(local_path), str(target)) + return target.stat().st_size + + def download(self, remote_key: str, local_path: Path) -> None: + source = self._resolve_key(remote_key) + if not source.is_file(): + raise StorageBackendError(f"Archive not found at destination: {remote_key}") + local_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(str(source), str(local_path)) + + def delete(self, remote_key: str) -> None: + target = self._resolve_key(remote_key) + target.unlink(missing_ok=True) + + def validate_config(self) -> None: + if not self._root.exists(): + try: + self._root.mkdir(parents=True, exist_ok=True) + except OSError as e: + raise StorageBackendError(f"Cannot create destination dir: {e}") from e + if not self._root.is_dir(): + raise StorageBackendError(f"Destination path is not a directory: {self._root}") + # Probe writability. + probe = self._root / ".gitbacker_write_probe" + try: + probe.write_text("ok") + probe.unlink() + except OSError as e: + raise StorageBackendError(f"Destination not writable: {e}") from e diff --git a/backend/shared/shared/storage_backends/s3.py b/backend/shared/shared/storage_backends/s3.py new file mode 100644 index 0000000..8c29a4e --- /dev/null +++ b/backend/shared/shared/storage_backends/s3.py @@ -0,0 +1,148 @@ +"""S3-compatible storage backend (AWS S3, MinIO, R2, Wasabi, Backblaze B2).""" + +from __future__ import annotations + +import logging +from functools import lru_cache +from pathlib import Path +from typing import Any + +import boto3 +from botocore.config import Config +from botocore.exceptions import ( + ClientError, + EndpointConnectionError, + NoCredentialsError, +) + +from . import StorageBackendError + +logger = logging.getLogger(__name__) + + +@lru_cache(maxsize=16) +def _build_client( + region: str | None, + endpoint_url: str | None, + access_key_id: str, + secret_access_key: str, +) -> Any: + """Cache boto3 clients by connection params. Prefork-safe (each Celery worker + process has its own cache).""" + return boto3.client( + "s3", + region_name=region or "us-east-1", + endpoint_url=endpoint_url or None, + aws_access_key_id=access_key_id, + aws_secret_access_key=secret_access_key, + config=Config( + retries={"max_attempts": 3, "mode": "standard"}, + signature_version="s3v4", + ), + ) + + +def _translate_client_error( + e: ClientError, *, bucket: str, op: str = "" +) -> StorageBackendError: + code = e.response.get("Error", {}).get("Code", "") + msg = e.response.get("Error", {}).get("Message", str(e)) + if code == "NoSuchBucket": + return StorageBackendError(f"Bucket '{bucket}' does not exist") + if code in ("AccessDenied", "403"): + return StorageBackendError(f"Access denied to bucket '{bucket}': {msg}") + if code == "InvalidAccessKeyId": + return StorageBackendError("Invalid AWS access key ID") + if code == "SignatureDoesNotMatch": + return StorageBackendError("Invalid AWS secret access key (signature mismatch)") + if code in ("NoSuchKey", "404"): + # 404 from download_file / head_object means the key is missing — + # use the op hint when present to give a more specific message. + verb = op or "Object" + return StorageBackendError(f"{verb} not found in bucket '{bucket}'") + return StorageBackendError(f"S3 error ({code}): {msg}") + + +class S3StorageBackend: + """Stores backup archives in an S3-compatible bucket.""" + + def __init__( + self, + *, + bucket: str, + prefix: str = "", + region: str | None = None, + endpoint_url: str | None = None, + access_key_id: str, + secret_access_key: str, + ) -> None: + self._bucket = bucket + self._prefix = prefix.strip("/") + self._client = _build_client( + region, endpoint_url, access_key_id, secret_access_key + ) + + def _full_key(self, remote_key: str) -> str: + if remote_key.startswith("/") or ".." in remote_key.split("/"): + raise StorageBackendError(f"Invalid storage key: {remote_key!r}") + return f"{self._prefix}/{remote_key}".lstrip("/") if self._prefix else remote_key + + def upload(self, local_path: Path, remote_key: str) -> int: + key = self._full_key(remote_key) + try: + # upload_file uses the boto3 TransferManager which handles + # multipart upload + parallel parts for large files automatically. + self._client.upload_file(str(local_path), self._bucket, key) + except ClientError as e: + raise _translate_client_error(e, bucket=self._bucket) from e + except EndpointConnectionError as e: + raise StorageBackendError(f"Cannot reach S3 endpoint: {e}") from e + return local_path.stat().st_size + + def download(self, remote_key: str, local_path: Path) -> None: + key = self._full_key(remote_key) + local_path.parent.mkdir(parents=True, exist_ok=True) + try: + self._client.download_file(self._bucket, key, str(local_path)) + except ClientError as e: + raise _translate_client_error( + e, bucket=self._bucket, op=f"Archive '{remote_key}'" + ) from e + except EndpointConnectionError as e: + raise StorageBackendError(f"Cannot reach S3 endpoint: {e}") from e + + def delete(self, remote_key: str) -> None: + key = self._full_key(remote_key) + try: + self._client.delete_object(Bucket=self._bucket, Key=key) + except ClientError as e: + raise _translate_client_error(e, bucket=self._bucket) from e + + def validate_config(self) -> None: + # Prefer head_bucket (cheapest); fall back to list_objects_v2 when the + # credentials lack s3:ListBucket on the bucket but can still read/write. + # At this call site a 404 unambiguously means the bucket is missing — + # the translator can't tell 404-missing-bucket from 404-missing-key + # because both surface as the same HTTP code, so handle it inline. + try: + self._client.head_bucket(Bucket=self._bucket) + return + except ClientError as e: + code = e.response.get("Error", {}).get("Code", "") + if code == "404": + raise StorageBackendError( + f"Bucket '{self._bucket}' does not exist" + ) from e + if code not in ("AccessDenied", "403"): + raise _translate_client_error(e, bucket=self._bucket) from e + except EndpointConnectionError as e: + raise StorageBackendError(f"Cannot reach S3 endpoint: {e}") from e + except NoCredentialsError as e: + raise StorageBackendError("No AWS credentials provided") from e + + try: + self._client.list_objects_v2(Bucket=self._bucket, MaxKeys=1) + except ClientError as e: + raise _translate_client_error(e, bucket=self._bucket) from e + except EndpointConnectionError as e: + raise StorageBackendError(f"Cannot reach S3 endpoint: {e}") from e diff --git a/frontend/src/app/destinations/page.tsx b/frontend/src/app/destinations/page.tsx index 73ddc0a..f82db5f 100644 --- a/frontend/src/app/destinations/page.tsx +++ b/frontend/src/app/destinations/page.tsx @@ -1,17 +1,20 @@ "use client"; -import { useState } from "react"; +import { useMemo, useState } from "react"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { toast } from "sonner"; -import { HardDrive, GitBranch, Trash2, Info } from "lucide-react"; +import { Cloud, GitBranch, HardDrive, Info, Pencil, Trash2 } from "lucide-react"; import { useAuth } from "@/lib/auth"; import { formatBytes } from "@/lib/utils"; import { AppShell } from "@/components/app-shell"; import { createDestination, deleteDestination, + Destination, + DestinationCreateInput, listDestinations, listRepositories, + testDestination, updateDestination, } from "@/lib/api"; import { Button } from "@/components/ui/button"; @@ -25,6 +28,13 @@ import { DialogTitle, DialogTrigger, } from "@/components/ui/dialog"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; import { Table, TableBody, @@ -34,29 +44,160 @@ import { TableRow, } from "@/components/ui/table"; -function StorageBar({ - used, - available, +// --------------------------------------------------------------------------- +// Registry: drives the per-storage-type form. Mirror of the notifications +// CHANNEL_TYPES pattern (frontend/src/app/settings/notifications/page.tsx). +// Adding a new backend = one entry here + one branch in shared/storage.py + +// one new XYZStorageBackend in shared/storage_backends/. +// --------------------------------------------------------------------------- +type FieldDef = { + key: string; + label: string; + placeholder?: string; + helperText?: string; + inputType?: "text" | "password"; + required?: boolean; + defaultValue?: string; +}; + +type StorageTypeDef = { + label: string; + description: string; + fields: FieldDef[]; + /** Build the user-facing "location" string for the table cell. */ + describe: (dest: Destination) => string; +}; + +const STORAGE_TYPES: Record<"local" | "s3", StorageTypeDef> = { + local: { + label: "Local volume", + description: "Write archives to a subdirectory under /data/backups.", + fields: [ + { + key: "path", + label: "Subdirectory", + placeholder: "e.g. critical", + helperText: + "Created automatically under /data/backups/. Leave empty to use the root.", + }, + ], + describe: (dest) => dest.path || "/data/backups", + }, + s3: { + label: "S3-compatible", + description: + "AWS S3, MinIO, Cloudflare R2, Wasabi, Backblaze B2 — anything that speaks S3 v4.", + fields: [ + { key: "bucket", label: "Bucket", placeholder: "my-backups", required: true }, + { + key: "region", + label: "Region", + placeholder: "us-east-1", + defaultValue: "us-east-1", + }, + { + key: "endpoint_url", + label: "Custom endpoint (optional)", + placeholder: "https://minio.example.com or http://localhost:9000", + helperText: + "Leave blank for AWS. For MinIO on localhost, also set 'allow_private_endpoint' (not yet exposed).", + }, + { + key: "prefix", + label: "Prefix (optional)", + placeholder: "gitbacker/", + helperText: + "Archives land under this key prefix. Changing later breaks references to existing snapshots.", + }, + { + key: "access_key_id", + label: "Access key ID", + placeholder: "AKIA…", + required: true, + }, + { + key: "secret_access_key", + label: "Secret access key", + inputType: "password", + required: true, + }, + ], + describe: (dest) => { + const bucket = dest.config_data?.bucket ?? ""; + const prefix = dest.config_data?.prefix ?? ""; + return prefix ? `s3://${bucket}/${prefix}` : `s3://${bucket}`; + }, + }, +}; + +const QUOTA_UNITS = [ + { label: "MB", bytes: 1024 ** 2 }, + { label: "GB", bytes: 1024 ** 3 }, + { label: "TB", bytes: 1024 ** 4 }, +]; + +function quotaToFormParts(quotaBytes: number | null): { + value: string; + unit: string; +} { + if (!quotaBytes) return { value: "", unit: "GB" }; + for (const u of [...QUOTA_UNITS].reverse()) { + if (quotaBytes >= u.bytes && quotaBytes % u.bytes === 0) { + return { value: String(quotaBytes / u.bytes), unit: u.label }; + } + } + return { value: String(quotaBytes / 1024 ** 2), unit: "MB" }; +} + +function partsToQuotaBytes(value: string, unit: string): number | null { + const v = parseFloat(value); + if (!value || isNaN(v) || v <= 0) return null; + const u = QUOTA_UNITS.find((u) => u.label === unit) ?? QUOTA_UNITS[1]; + return Math.round(v * u.bytes); +} + +// --------------------------------------------------------------------------- +// Capacity rendering: branches on (quota set?, available_bytes known?) +// --------------------------------------------------------------------------- +function CapacityCell({ + dest, + onSetQuota, }: { - used: number; - available: number | null; + dest: Destination; + onSetQuota: () => void; }) { - if (available === null || available === 0) { + const hasQuota = !!dest.quota_bytes; + const hasAvailable = dest.available_bytes !== null; + + // S3 with no quota: show used + an actionable "Set a quota" link. + if (!hasQuota && !hasAvailable) { return ( - - {formatBytes(used)} used - +
+
+ {formatBytes(dest.used_bytes)} used +
+ +
); } - const total = used + available; - const pct = Math.min((used / total) * 100, 100); + // Total: quota when set, otherwise used + free (the existing LOCAL behavior). + const total = hasQuota + ? dest.quota_bytes! + : dest.used_bytes + (dest.available_bytes ?? 0); + const pct = total > 0 ? Math.min((dest.used_bytes / total) * 100, 100) : 0; const tone = - pct > 85 ? "var(--err)" : pct > 70 ? "var(--warn)" : "var(--mint)"; + pct > 90 ? "var(--err)" : pct > 75 ? "var(--warn)" : "var(--mint)"; const pctColor = - pct > 85 + pct > 90 ? "var(--err)" - : pct > 70 + : pct > 75 ? "var(--warn)" : "var(--muted-foreground)"; @@ -65,12 +206,15 @@ function StorageBar({
- {formatBytes(used)} + {formatBytes(dest.used_bytes)} {" "} - used ·{" "} - {pct.toFixed(1)}% + used · {pct.toFixed(1)}% + + + {hasQuota + ? `of ${formatBytes(dest.quota_bytes!)} quota` + : `${formatBytes(dest.available_bytes ?? 0)} free`} - {formatBytes(available)} free
= {}; + export default function DestinationsPage() { const { token } = useAuth(); const queryClient = useQueryClient(); + const [open, setOpen] = useState(false); + const [editingId, setEditingId] = useState(null); + const isEditing = editingId !== null; + const [alias, setAlias] = useState(""); - const [path, setPath] = useState(""); + const [storageType, setStorageType] = useState<"local" | "s3">("local"); + const [fields, setFields] = useState>(EMPTY_FIELDS); + const [quotaValue, setQuotaValue] = useState(""); + const [quotaUnit, setQuotaUnit] = useState("GB"); const { data: destinations = [], isLoading, isError } = useQuery({ queryKey: ["destinations"], @@ -105,36 +259,120 @@ export default function DestinationsPage() { enabled: !!token, }); - // Only aggregate destinations with known capacity. Treating null - // available_bytes as zero would inflate the "% used" figure for remote or - // unmounted destinations. Matches the guard StorageBar already uses per-row. + // Only aggregate destinations with known capacity for the global tiles. const withCapacity = destinations.filter( (d) => d.available_bytes != null && d.available_bytes > 0, ); - const totalUsed = withCapacity.reduce((s, d) => s + d.used_bytes, 0); + const totalUsed = destinations.reduce((s, d) => s + d.used_bytes, 0); const totalCapacity = withCapacity.reduce( - (s, d) => s + d.used_bytes + (d.available_bytes ?? 0), + (s, d) => + s + + (d.quota_bytes ?? d.used_bytes + (d.available_bytes ?? 0)), 0, ); const usedPct = totalCapacity > 0 ? (totalUsed / totalCapacity) * 100 : 0; - const BACKUP_ROOT = "/data/backups"; - const fullPath = path ? `${BACKUP_ROOT}/${path}` : BACKUP_ROOT; + function resetForm() { + setAlias(""); + setStorageType("local"); + setFields(EMPTY_FIELDS); + setQuotaValue(""); + setQuotaUnit("GB"); + setEditingId(null); + } + + function openForCreate() { + resetForm(); + setOpen(true); + } + + function openForEdit(dest: Destination) { + setEditingId(dest.id); + setAlias(dest.alias); + setStorageType(dest.storage_type); + + const initial: Record = {}; + if (dest.storage_type === "local") { + // Strip the /data/backups/ prefix for the subdirectory input. + initial.path = dest.path.startsWith(`${BACKUP_ROOT}/`) + ? dest.path.slice(BACKUP_ROOT.length + 1) + : dest.path === BACKUP_ROOT + ? "" + : dest.path; + } else { + for (const [k, v] of Object.entries(dest.config_data ?? {})) { + initial[k] = String(v); + } + } + setFields(initial); + + const q = quotaToFormParts(dest.quota_bytes); + setQuotaValue(q.value); + setQuotaUnit(q.unit); + setOpen(true); + } + + function buildSubmitPayload(): DestinationCreateInput { + const quota_bytes = partsToQuotaBytes(quotaValue, quotaUnit); + + if (storageType === "local") { + const sub = (fields.path ?? "").replace(/^\/+/, ""); + return { + alias, + storage_type: "local", + path: sub ? `${BACKUP_ROOT}/${sub}` : BACKUP_ROOT, + quota_bytes, + }; + } + + // S3: drop empty fields so the server treats them as unset (relevant on + // edit, where omitting secret_access_key means "keep existing"). + const config: Record = {}; + for (const f of STORAGE_TYPES.s3.fields) { + const value = fields[f.key]?.trim() ?? ""; + if (value) config[f.key] = value; + } + return { + alias, + storage_type: "s3", + path: "", + config_data: config, + quota_bytes, + }; + } const createMutation = useMutation({ - mutationFn: () => - createDestination(token!, { alias, path: fullPath }), + mutationFn: () => createDestination(token!, buildSubmitPayload()), onSuccess: () => { queryClient.invalidateQueries({ queryKey: ["destinations"] }); setOpen(false); - setAlias(""); - setPath(""); + resetForm(); toast.success("Destination created"); }, onError: (err: Error) => toast.error(err.message), }); + const updateMutation = useMutation({ + mutationFn: () => { + const payload = buildSubmitPayload(); + // On update, only send fields that make sense as a PATCH. + return updateDestination(token!, editingId!, { + alias: payload.alias, + path: payload.path ?? undefined, + config_data: payload.config_data, + quota_bytes: payload.quota_bytes, + }); + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["destinations"] }); + setOpen(false); + resetForm(); + toast.success("Destination updated"); + }, + onError: (err: Error) => toast.error(err.message), + }); + const deleteMutation = useMutation({ mutationFn: (id: string) => deleteDestination(token!, id), onSuccess: () => { @@ -154,6 +392,18 @@ export default function DestinationsPage() { onError: (err: Error) => toast.error(err.message), }); + const testMutation = useMutation({ + mutationFn: (id: string) => testDestination(token!, id), + onSuccess: (data) => { + if (data.ok) toast.success(data.message); + else toast.error(data.message); + }, + onError: (err: Error) => toast.error(err.message), + }); + + const def = STORAGE_TYPES[storageType]; + const visibleFields = useMemo(() => def.fields, [def]); + return (
@@ -166,22 +416,31 @@ export default function DestinationsPage() { Destinations

- Where archives land. Paths must exist on disk before saving — - the directory will be created if missing. + Where archives land. Local volumes auto-create their subdirectory; + S3-compatible buckets are tested on save.

- + { + setOpen(o); + if (!o) resetForm(); + }} + > - + - + - Add destination + + {isEditing ? "Edit destination" : "Add destination"} +
{ e.preventDefault(); - createMutation.mutate(); + if (isEditing) updateMutation.mutate(); + else createMutation.mutate(); }} className="space-y-4" > @@ -191,43 +450,156 @@ export default function DestinationsPage() { id="alias" value={alias} onChange={(e) => setAlias(e.target.value)} - placeholder="e.g. External SSD" + placeholder="e.g. External SSD or Production S3" maxLength={64} required />
+
- -
- - /data/backups/ - - - setPath( - e.target.value - .replace(/^\/+/, "") - .replace(/[^\w./\-]/g, ""), - ) + + +

+ {isEditing + ? "Type cannot be changed. Delete and recreate to switch." + : def.description} +

+
+ + {storageType === "local" ? ( +
+ +
+ + /data/backups/ + + + setFields((f) => ({ + ...f, + path: e.target.value + .replace(/^\/+/, "") + .replace(/[^\w./\-]/g, ""), + })) + } + maxLength={128} + placeholder="e.g. critical" + className="flex-1 bg-transparent px-3 py-2 text-sm font-mono outline-none placeholder:text-muted-foreground" + /> +
+

+ Optional subfolder within the backup volume. Created + automatically. +

+
+ ) : ( + visibleFields.map((f) => ( +
+ + + setFields((prev) => ({ + ...prev, + [f.key]: e.target.value, + })) + } + placeholder={ + f.inputType === "password" && isEditing + ? "••••••••• (leave blank to keep)" + : f.placeholder + } + required={f.required && !isEditing} + /> + {f.helperText && ( +

+ {f.helperText} +

+ )} +
+ )) + )} + +
+ +
+ setQuotaValue(e.target.value)} + placeholder="e.g. 100" + className="flex-1" /> +

- Optional subfolder within the backup volume. Leave empty to use - the root. The directory will be created automatically. + Get notified when usage exceeds 90% of this limit. Leave + blank for no quota alerts. + {storageType === "local" && + " (Free-disk alerts still fire independently.)"}

+ @@ -242,10 +614,14 @@ export default function DestinationsPage() {
- {formatBytes(totalCapacity).replace(/ .*/, "")} + {totalCapacity > 0 + ? formatBytes(totalCapacity).replace(/ .*/, "") + : "—"} - {formatBytes(totalCapacity).replace(/^[\d.]+ /, "")} + {totalCapacity > 0 + ? formatBytes(totalCapacity).replace(/^[\d.]+ /, "") + : "no quota set on remote destinations"}
@@ -288,8 +664,7 @@ export default function DestinationsPage() {

) : destinations.length === 0 ? (

- No destinations configured. Restart the API to provision the default - local destination. + No destinations configured. Click "Add destination" to create one.

) : (
@@ -300,7 +675,7 @@ export default function DestinationsPage() { Alias - Path + Location Repos @@ -311,25 +686,13 @@ export default function DestinationsPage() { Status - + {destinations.map((dest) => { - const cap = - dest.available_bytes != null - ? dest.used_bytes + dest.available_bytes - : null; - const pct = - cap != null && cap > 0 - ? (dest.used_bytes / cap) * 100 - : null; - const health = - pct != null && pct > 85 - ? { color: "var(--err)", label: "Low space" } - : pct != null && pct > 70 - ? { color: "var(--warn)", label: "High usage" } - : { color: "var(--mint)", label: "Mounted" }; + const def = STORAGE_TYPES[dest.storage_type] ?? STORAGE_TYPES.local; + const Icon = dest.storage_type === "s3" ? Cloud : HardDrive; return (
- +
{dest.alias}
- Local volume + {def.label}
- {dest.path} + {def.describe(dest)} @@ -362,9 +725,9 @@ export default function DestinationsPage() { - openForEdit(dest)} /> @@ -372,12 +735,13 @@ export default function DestinationsPage() { - {health.label} + {dest.storage_type === "s3" ? "Reachable" : "Mounted"}
{dest.is_default && ( @@ -387,6 +751,23 @@ export default function DestinationsPage() { )} + + {!dest.is_default && (