diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d51d69b..9b554c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,19 @@ jobs: - name: Clone uses: actions/checkout@v6 + - name: Start MinIO + run: | + docker run -d --name minio \ + -p 9090:9000 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio server /data + timeout 30 bash -c 'until curl -sf http://127.0.0.1:9090/minio/health/live; do sleep 1; done' + curl -sL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc + chmod +x /usr/local/bin/mc + mc alias set ci http://127.0.0.1:9090 minioadmin minioadmin + mc mb ci/vms-media + - name: Find tests run: | echo "Finding tests" @@ -96,6 +109,10 @@ jobs: env: CI: 'Yes' + - name: Configure VMS Settings for MinIO + working-directory: /home/runner/frappe-bench + run: bench --site test_site execute vms.seed.seed_test_settings + - name: Run Tests working-directory: /home/runner/frappe-bench run: | @@ -103,3 +120,4 @@ jobs: bench --site test_site run-tests --app vms env: TYPE: server + VMS_S3_ENDPOINT_URL: http://127.0.0.1:9090 diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 76a294f..eaecf37 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -37,6 +37,21 @@ jobs: - name: Clone uses: actions/checkout@v4 + - name: Start MinIO + run: | + docker run -d --name minio \ + -p 9090:9000 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio server /data + # Wait for MinIO to be ready + timeout 30 bash -c 'until curl -sf http://127.0.0.1:9090/minio/health/live; do sleep 1; done' + # Create test bucket using mc + curl -sL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc + chmod +x /usr/local/bin/mc + mc alias set ci http://127.0.0.1:9090 minioadmin minioadmin + mc mb ci/vms-media + - name: Setup Python uses: actions/setup-python@v5 with: @@ -102,6 +117,10 @@ jobs: bench --site vms.test set-config allow_tests true bench --site vms.test set-config host_name "http://vms.test:8000" + - name: Configure VMS Settings for MinIO + working-directory: /home/runner/frappe-bench + run: bench --site vms.test execute vms.seed.seed_test_settings + - name: Start Frappe Server working-directory: /home/runner/frappe-bench run: | @@ -111,6 +130,8 @@ jobs: echo "Waiting for Frappe server to start..." timeout 60 bash -c 'until curl -s http://vms.test:8000 > /dev/null; do sleep 2; done' echo "Frappe server is ready!" + env: + VMS_S3_ENDPOINT_URL: http://127.0.0.1:9090 - name: Install Playwright run: | diff --git a/e2e/helpers/vms.ts b/e2e/helpers/vms.ts index 40e0083..5bd12b1 100644 --- a/e2e/helpers/vms.ts +++ b/e2e/helpers/vms.ts @@ -1,5 +1,5 @@ import { APIRequestContext } from "@playwright/test"; -import { createDoc, deleteDoc, getDoc, getList } from "./frappe"; +import { callMethod, createDoc, deleteDoc, getDoc, getList } from "./frappe"; /** * VMS Project document interface. @@ -151,3 +151,162 @@ export async function cleanupTestProjects( } } } + +// --------------------------------------------------------------------------- +// Upload helpers +// --------------------------------------------------------------------------- + +interface UploadUrlResponse { + upload_url: string; + r2_key: string; + asset_name: string; +} + +interface ConfirmUploadResponse { + status: string; + asset_name: string; +} + +interface ViewUrlResponse { + url: string; +} + +interface DownloadUrlResponse { + url: string; +} + +/** + * Request a presigned upload URL from the VMS API. + */ +export async function getUploadUrl( + request: APIRequestContext, + options: { + file_name: string; + content_type: string; + project?: string; + category?: string; + }, +): Promise { + return callMethod(request, "vms.api.get_upload_url", { + file_name: options.file_name, + content_type: options.content_type, + project: options.project, + category: options.category, + }); +} + +/** + * Upload a buffer to the presigned URL (PUT request directly to object storage). + */ +export async function uploadToPresignedUrl( + request: APIRequestContext, + uploadUrl: string, + content: Buffer, + contentType: string, +): Promise { + const response = await request.put(uploadUrl, { + data: content, + headers: { + "Content-Type": contentType, + }, + }); + + if (!response.ok()) { + throw new Error( + `Upload to presigned URL failed: ${response.status()} ${response.statusText()}`, + ); + } +} + +/** + * Confirm an upload after the file has been PUT to object storage. + */ +export async function confirmUpload( + request: APIRequestContext, + assetName: string, + fileSize: number, +): Promise { + return callMethod( + request, + "vms.api.confirm_upload", + { + asset_name: assetName, + file_size: fileSize, + }, + ); +} + +/** + * Get a presigned view URL for an asset. + */ +export async function getViewUrl( + request: APIRequestContext, + assetName: string, +): Promise { + return callMethod(request, "vms.api.get_view_url", { + asset_name: assetName, + }); +} + +/** + * Get a presigned download URL for an asset. + */ +export async function getDownloadUrl( + request: APIRequestContext, + assetName: string, +): Promise { + return callMethod( + request, + "vms.api.get_download_url", + { + asset_name: assetName, + }, + ); +} + +/** + * Full upload flow: get presigned URL → PUT file → confirm upload. + * Returns the asset name and r2 key. + */ +export async function uploadTestFile( + request: APIRequestContext, + options: { + file_name?: string; + content?: Buffer; + content_type?: string; + project?: string; + category?: string; + } = {}, +): Promise<{ asset_name: string; r2_key: string }> { + const fileName = options.file_name || `test-file-${Date.now()}.mp4`; + const contentType = options.content_type || "video/mp4"; + const content = options.content || Buffer.from("E2E test file content"); + + // Step 1: Get presigned upload URL + const { upload_url, r2_key, asset_name } = await getUploadUrl(request, { + file_name: fileName, + content_type: contentType, + project: options.project, + category: options.category, + }); + + // Step 2: PUT file to object storage + await uploadToPresignedUrl(request, upload_url, content, contentType); + + // Step 3: Confirm the upload + await confirmUpload(request, asset_name, content.length); + + return { asset_name, r2_key }; +} + +/** + * Delete an asset via the VMS API (cleans up both DB record and object storage). + */ +export async function deleteAsset( + request: APIRequestContext, + assetName: string, +): Promise { + await callMethod(request, "vms.api.delete_asset", { + asset_name: assetName, + }); +} diff --git a/e2e/tests/uploads.spec.ts b/e2e/tests/uploads.spec.ts new file mode 100644 index 0000000..4094d9d --- /dev/null +++ b/e2e/tests/uploads.spec.ts @@ -0,0 +1,199 @@ +import { test, expect } from "@playwright/test"; +import { + createTestProject, + cleanupTestProjects, + uploadTestFile, + getUploadUrl, + uploadToPresignedUrl, + confirmUpload, + getViewUrl, + getDownloadUrl, + deleteAsset, +} from "../helpers/vms"; +import { callMethod } from "../helpers/frappe"; + +test.describe("Uploads", () => { + let projectName: string; + + test.beforeAll(async ({ request }) => { + const project = await createTestProject(request, { + project_name: `E2E Upload Project ${Date.now()}`, + }); + projectName = project.name; + }); + + test.afterAll(async ({ request }) => { + await cleanupTestProjects(request, "E2E Upload Project"); + }); + + test("should get a presigned upload URL", async ({ request }) => { + const result = await getUploadUrl(request, { + file_name: "test-video.mp4", + content_type: "video/mp4", + project: projectName, + }); + + expect(result.upload_url).toBeTruthy(); + expect(result.r2_key).toBeTruthy(); + expect(result.asset_name).toBeTruthy(); + expect(result.r2_key).toContain(".mp4"); + }); + + test("should upload a file via presigned URL and confirm", async ({ + request, + }) => { + const content = Buffer.from("test video content for E2E"); + + // Step 1: Get presigned URL + const { upload_url, asset_name } = await getUploadUrl(request, { + file_name: "e2e-test.mp4", + content_type: "video/mp4", + project: projectName, + }); + + // Step 2: Upload to presigned URL + await uploadToPresignedUrl(request, upload_url, content, "video/mp4"); + + // Step 3: Confirm upload + const confirmResult = await confirmUpload( + request, + asset_name, + content.length, + ); + expect(confirmResult.status).toBe("ok"); + + // Cleanup + await deleteAsset(request, asset_name); + }); + + test("should complete full upload flow with helper", async ({ + request, + }) => { + const { asset_name, r2_key } = await uploadTestFile(request, { + file_name: "full-flow-test.mp4", + content: Buffer.from("full flow test content"), + content_type: "video/mp4", + project: projectName, + }); + + expect(asset_name).toBeTruthy(); + expect(r2_key).toBeTruthy(); + + // Cleanup + await deleteAsset(request, asset_name); + }); + + test("should generate a view URL for an uploaded asset", async ({ + request, + }) => { + const testContent = "view test content"; + const { asset_name } = await uploadTestFile(request, { + file_name: "view-test.mp4", + content: Buffer.from(testContent), + content_type: "video/mp4", + project: projectName, + }); + + const { url } = await getViewUrl(request, asset_name); + expect(url).toBeTruthy(); + expect(url).toContain("X-Amz-Signature"); + + // Verify the presigned URL actually returns the file + const response = await request.get(url); + expect(response.ok()).toBeTruthy(); + const body = await response.text(); + expect(body).toBe(testContent); + + // Cleanup + await deleteAsset(request, asset_name); + }); + + test("should generate a download URL for an uploaded asset", async ({ + request, + }) => { + const testContent = "download test content"; + const { asset_name } = await uploadTestFile(request, { + file_name: "download-test.mp4", + content: Buffer.from(testContent), + content_type: "video/mp4", + project: projectName, + }); + + const { url } = await getDownloadUrl(request, asset_name); + expect(url).toBeTruthy(); + expect(url).toContain("X-Amz-Signature"); + + // Verify the presigned URL returns the file + const response = await request.get(url); + expect(response.ok()).toBeTruthy(); + const body = await response.text(); + expect(body).toBe(testContent); + + // Cleanup + await deleteAsset(request, asset_name); + }); + + test("should delete an uploaded asset and its R2 object", async ({ + request, + }) => { + const { asset_name } = await uploadTestFile(request, { + file_name: "delete-test.mp4", + content: Buffer.from("delete test content"), + content_type: "video/mp4", + project: projectName, + }); + + // Delete the asset + await deleteAsset(request, asset_name); + + // Verify the asset record is gone + const response = await request.get( + `/api/resource/VMS Asset/${encodeURIComponent(asset_name)}`, + ); + expect(response.status()).toBe(404); + }); + + test("should upload to inbox when no project specified", async ({ + request, + }) => { + const { asset_name, r2_key } = await uploadTestFile(request, { + file_name: "inbox-test.mp4", + content: Buffer.from("inbox test content"), + content_type: "video/mp4", + }); + + expect(r2_key).toMatch(/^inbox\//); + + // Cleanup + await deleteAsset(request, asset_name); + }); + + test("should respect file category on upload", async ({ request }) => { + const { asset_name } = await uploadTestFile(request, { + file_name: "final-cut.mp4", + content: Buffer.from("final cut content"), + content_type: "video/mp4", + project: projectName, + category: "Final", + }); + + // Verify the category was set + const assetResponse = await request.get( + `/api/resource/VMS Asset/${encodeURIComponent(asset_name)}`, + ); + expect(assetResponse.ok()).toBeTruthy(); + const assetData = await assetResponse.json(); + expect(assetData.data.category).toBe("Final"); + + // Cleanup + await deleteAsset(request, asset_name); + }); + + test("should test R2 connection successfully", async ({ request }) => { + const result = await callMethod<{ status: string }>( + request, + "vms.api.test_r2_connection", + ); + expect(result.status).toBe("ok"); + }); +}); diff --git a/vms/api.py b/vms/api.py index 799b382..5c22e4b 100644 --- a/vms/api.py +++ b/vms/api.py @@ -63,7 +63,13 @@ def fail_upload(asset_name: str): @frappe.whitelist() -def get_upload_url(file_name: str, content_type: str, project: str | None = None, category: str = "Source", folder: str | None = None): +def get_upload_url( + file_name: str, + content_type: str, + project: str | None = None, + category: str = "Source", + folder: str | None = None, +): """Generate a presigned upload URL for direct upload to R2. Returns dict with upload_url, r2_key, and asset_name. @@ -237,9 +243,7 @@ def create_folder(folder_name: str, project: str): frappe.throw(_("Project {0} does not exist").format(project)) # Check for duplicate folder name in the same project - existing = frappe.db.exists( - "VMS Folder", {"folder_name": folder_name, "project": project} - ) + existing = frappe.db.exists("VMS Folder", {"folder_name": folder_name, "project": project}) if existing: frappe.throw(_("A folder named '{0}' already exists in this project").format(folder_name)) diff --git a/vms/r2.py b/vms/r2.py index f7e1912..5e92126 100644 --- a/vms/r2.py +++ b/vms/r2.py @@ -1,3 +1,4 @@ +import os import uuid import boto3 @@ -6,9 +7,13 @@ def get_r2_client(): settings = frappe.get_single("VMS Settings") + endpoint_url = os.environ.get( + "VMS_S3_ENDPOINT_URL", + f"https://{settings.r2_account_id}.r2.cloudflarestorage.com", + ) return boto3.client( "s3", - endpoint_url=f"https://{settings.r2_account_id}.r2.cloudflarestorage.com", + endpoint_url=endpoint_url, aws_access_key_id=settings.r2_access_key_id, aws_secret_access_key=settings.get_password("r2_secret_access_key"), region_name="auto", diff --git a/vms/seed.py b/vms/seed.py new file mode 100644 index 0000000..577f7ec --- /dev/null +++ b/vms/seed.py @@ -0,0 +1,15 @@ +import frappe + + +def seed_test_settings(): + """Configure VMS Settings for MinIO in CI/test environments. + + Usage: + bench --site execute vms.seed.seed_test_settings + """ + settings = frappe.get_doc("VMS Settings") + settings.r2_account_id = "test" + settings.r2_access_key_id = "minioadmin" + settings.r2_bucket_name = "vms-media" + settings.r2_secret_access_key = "minioadmin" + settings.save(ignore_permissions=True) diff --git a/vms/video_management_solution/doctype/vms_folder/test_vms_folder.py b/vms/video_management_solution/doctype/vms_folder/test_vms_folder.py index e827a17..a277e94 100644 --- a/vms/video_management_solution/doctype/vms_folder/test_vms_folder.py +++ b/vms/video_management_solution/doctype/vms_folder/test_vms_folder.py @@ -4,7 +4,6 @@ # import frappe from frappe.tests import IntegrationTestCase - # On IntegrationTestCase, the doctype test records and all # link-field test record dependencies are recursively loaded # Use these module variables to add/remove to/from that list @@ -12,7 +11,6 @@ IGNORE_TEST_RECORD_DEPENDENCIES = [] # eg. ["User"] - class IntegrationTestVMSFolder(IntegrationTestCase): """ Integration tests for VMSFolder.