diff --git a/src/handlers/get.js b/src/handlers/get.js index 87916a3b..3f16c13a 100644 --- a/src/handlers/get.js +++ b/src/handlers/get.js @@ -11,6 +11,7 @@ */ import { getSource } from '../routes/source.js'; import getList from '../routes/list.js'; +import getListPaginated from '../routes/list-paginated.js'; import logout from '../routes/logout.js'; import { getConfig } from '../routes/config.js'; import { getVersionSource, getVersionList } from '../routes/version.js'; @@ -24,13 +25,14 @@ function getRobots() { return { body, status: 200 }; } -export default async function getHandler({ env, daCtx }) { +export default async function getHandler({ req, env, daCtx }) { const { path } = daCtx; if (path.startsWith('/favicon.ico')) return get404(); if (path.startsWith('/robots.txt')) return getRobots(); if (path.startsWith('/source')) return getSource({ env, daCtx }); + if (path.startsWith('/list-paginated')) return getListPaginated({ req, env, daCtx }); if (path.startsWith('/list')) return getList({ env, daCtx }); if (path.startsWith('/config')) return getConfig({ env, daCtx }); if (path.startsWith('/versionlist')) return getVersionList({ env, daCtx }); diff --git a/src/index.js b/src/index.js index ca62928d..47016bf3 100644 --- a/src/index.js +++ b/src/index.js @@ -36,7 +36,7 @@ export default { respObj = await headHandler({ env, daCtx }); break; case 'GET': - respObj = await getHandler({ env, daCtx }); + respObj = await getHandler({ req, env, daCtx }); break; case 'PUT': respObj = await postHandler({ req, env, daCtx }); diff --git a/src/routes/list-paginated.js b/src/routes/list-paginated.js new file mode 100644 index 00000000..121ef39f --- /dev/null +++ b/src/routes/list-paginated.js @@ -0,0 +1,52 @@ +/* + * Copyright 2024 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +import listBuckets from '../storage/bucket/list.js'; +import { listObjectsPaginated } from '../storage/object/list.js'; +import { getChildRules, hasPermission } from '../utils/auth.js'; + +async function handleBuckets({ env, daCtx }) { + const res = await listBuckets(env, daCtx); + + if (res.status !== 200) return { body: '', status: res.status }; + + try { + const parsedBody = JSON.parse(res.body); + return { + body: JSON.stringify({ + data: parsedBody, + limit: 1000, + offset: 0, + }), + status: 200, + }; + } catch (e) { + return { body: '', status: 500 }; + } +} + +export default async function getListPaginated({ req, env, daCtx }) { + if (!daCtx.org) return handleBuckets({ env, daCtx }); + if (!hasPermission(daCtx, daCtx.key, 'read')) return { status: 403 }; + + // Get the child rules of the current folder and store this in daCtx.aclCtx + getChildRules(daCtx); + + const { searchParams } = new URL(req.url); + const limit = Number.parseInt(searchParams.get('limit'), 10) ?? null; + const offset = Number.parseInt(searchParams.get('offset'), 10) ?? null; + + function numOrUndef(num) { + return Number.isNaN(num) ? undefined : num; + } + + return /* await */ listObjectsPaginated(env, daCtx, numOrUndef(limit), numOrUndef(offset)); +} diff --git a/src/storage/object/list.js b/src/storage/object/list.js index 9132fef2..572b5d61 100644 --- a/src/storage/object/list.js +++ b/src/storage/object/list.js @@ -15,18 +15,72 @@ import { } from '@aws-sdk/client-s3'; import getS3Config from '../utils/config.js'; -import formatList from '../utils/list.js'; +import formatList, { formatPaginatedList } from '../utils/list.js'; -function buildInput({ org, key, maxKeys }) { +const LIST_LIMIT = 5000; + +function buildInput({ + org, key, maxKeys, continuationToken, +}) { const input = { Bucket: `${org}-content`, Prefix: key ? `${key}/` : null, Delimiter: '/', }; if (maxKeys) input.MaxKeys = maxKeys; + if (continuationToken) input.ContinuationToken = continuationToken; return input; } +async function scanFiles({ + daCtx, env, offset, limit, +}) { + const config = getS3Config(env); + const client = new S3Client(config); + + let continuationToken = null; + const visibleFiles = []; + + while (visibleFiles.length < offset + limit) { + const remainingKeys = offset + limit - visibleFiles.length; + // fetch 25 extra to account for some hidden files + const numKeysToFetch = Math.min(1000, remainingKeys + 25); + + const input = buildInput({ ...daCtx, maxKeys: numKeysToFetch, continuationToken }); + const command = new ListObjectsV2Command(input); + + const resp = await client.send(command); + continuationToken = resp.NextContinuationToken; + visibleFiles.push(...formatPaginatedList(resp, daCtx)); + + if (!continuationToken) break; + } + + return visibleFiles.slice(offset, offset + limit); +} + +export async function listObjectsPaginated(env, daCtx, maxKeys = 1000, offset = 0) { + if (offset + maxKeys > LIST_LIMIT) { + return { status: 400 }; + } + + try { + const files = await scanFiles({ + daCtx, env, limit: maxKeys, offset, + }); + return { + body: JSON.stringify({ + offset, + limit: maxKeys, + data: files, + }), + status: 200, + }; + } catch (e) { + return { body: '', status: 404 }; + } +} + export default async function listObjects(env, daCtx, maxKeys) { const config = getS3Config(env); const client = new S3Client(config); @@ -35,7 +89,6 @@ export default async function listObjects(env, daCtx, maxKeys) { const command = new ListObjectsV2Command(input); try { const resp = await client.send(command); - // console.log(resp); const body = formatList(resp, daCtx); return { body: JSON.stringify(body), diff --git a/src/storage/utils/list.js b/src/storage/utils/list.js index 5d6e5ee9..fa21c128 100644 --- a/src/storage/utils/list.js +++ b/src/storage/utils/list.js @@ -13,6 +13,68 @@ import { ListObjectsV2Command, } from '@aws-sdk/client-s3'; +function mapPrefixes(CommonPrefixes, daCtx) { + return CommonPrefixes?.map((prefix) => { + const name = prefix.Prefix.slice(0, -1).split('/').pop(); + const splitName = name.split('.'); + + // Do not add any extension folders + if (splitName.length > 1) return null; + + const path = `/${daCtx.org}/${prefix.Prefix.slice(0, -1)}`; + + return { path, name }; + }).filter((x) => !!x) ?? []; +} + +function mapContents(Contents, folders, daCtx) { + return Contents?.map((content) => { + const itemName = content.Key.split('/').pop(); + const splitName = itemName.split('.'); + // file.jpg.props should not be a part of the list + // hidden files (.props) should not be a part of this list + if (splitName.length !== 2) return null; + + const [name, ext, props] = splitName; + + // Do not show any props sidecar files + if (props) return null; + + // See if the folder is already in the list + if (ext === 'props') { + if (folders.some((item) => item.name === name)) return null; + + // Remove props from the key so it can look like a folder + // eslint-disable-next-line no-param-reassign + content.Key = content.Key.replace('.props', ''); + } + + // Do not show any hidden files. + if (!name) return null; + const item = { path: `/${daCtx.org}/${content.Key}`, name }; + if (ext !== 'props') { + item.ext = ext; + item.lastModified = content.LastModified.getTime(); + } + + return item; + }).filter((x) => !!x) ?? []; +} + +// Performs the same as formatList, but doesn't sort +// This prevents bugs when sorting across pages of the paginated api response +// However, the order is slightly different to the formatList return value +export function formatPaginatedList(resp, daCtx) { + const { Contents } = resp; + + const combined = []; + + const files = mapContents(Contents, [], daCtx); + combined.push(...files); + + return combined; +} + export default function formatList(resp, daCtx) { function compare(a, b) { if (a.name < b.name) return -1; @@ -24,52 +86,11 @@ export default function formatList(resp, daCtx) { const combined = []; - if (CommonPrefixes) { - CommonPrefixes.forEach((prefix) => { - const name = prefix.Prefix.slice(0, -1).split('/').pop(); - const splitName = name.split('.'); + const folders = mapPrefixes(CommonPrefixes, daCtx); + combined.push(...folders); - // Do not add any extension folders - if (splitName.length > 1) return; - - const path = `/${daCtx.org}/${prefix.Prefix.slice(0, -1)}`; - combined.push({ path, name }); - }); - } - - if (Contents) { - Contents.forEach((content) => { - const itemName = content.Key.split('/').pop(); - const splitName = itemName.split('.'); - // file.jpg.props should not be a part of the list - // hidden files (.props) should not be a part of this list - if (splitName.length !== 2) return; - - const [name, ext, props] = splitName; - - // Do not show any props sidecar files - if (props) return; - - // See if the folder is already in the list - if (ext === 'props') { - if (combined.some((item) => item.name === name)) return; - - // Remove props from the key so it can look like a folder - // eslint-disable-next-line no-param-reassign - content.Key = content.Key.replace('.props', ''); - } - - // Do not show any hidden files. - if (!name) return; - const item = { path: `/${daCtx.org}/${content.Key}`, name }; - if (ext !== 'props') { - item.ext = ext; - item.lastModified = content.LastModified.getTime(); - } - - combined.push(item); - }); - } + const files = mapContents(Contents, folders, daCtx); + combined.push(...files); return combined.sort(compare); } diff --git a/test/routes/list-paginated.test.js b/test/routes/list-paginated.test.js new file mode 100644 index 00000000..f94998c4 --- /dev/null +++ b/test/routes/list-paginated.test.js @@ -0,0 +1,101 @@ +/* + * Copyright 2025 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +import assert from 'assert'; +import { describe, it, afterEach, vi, beforeAll } from 'vitest'; +import getListPaginated from '../../src/routes/list-paginated.js'; +import { listObjectsPaginated } from '../../src/storage/object/list.js'; +import { hasPermission, getChildRules } from '../../src/utils/auth.js'; + +describe('List Route', () => { + beforeAll(() => { + vi.mock('../../src/storage/object/list.js', () => ({ + listObjectsPaginated: vi.fn() + })); + vi.mock('../../src/utils/auth.js', async () => { + const actual = await vi.importActual('../../src/utils/auth.js'); + return { + ...actual, + hasPermission: vi.fn(), + getChildRules: vi.fn() + }; + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('Test getListPaginated with permissions', async () => { + const loCalled = []; + listObjectsPaginated.mockImplementation((e, c) => { + loCalled.push({ e, c }); + return {}; + }); + + const ctx = { org: 'foo', key: 'q/q/q' }; + hasPermission.mockImplementation((c, k, a) => { + if (k === 'q/q/q' && a === 'read') { + return false; + } + return true; + }); + + const req = { + url: new URL('https://admin.da.live/list/foo/bar'), + }; + + const resp = await getListPaginated({ req, env: {}, daCtx: ctx, aclCtx: {} }); + assert.strictEqual(403, resp.status); + assert.strictEqual(0, loCalled.length); + + const aclCtx = { pathLookup: new Map() }; + const daCtx = { org: 'bar', key: 'q/q', users: [], aclCtx }; + + // Mock getChildRules to set childRules on the aclCtx + getChildRules.mockImplementation((ctx) => { + ctx.aclCtx.childRules = ['/q/q/**=read,write']; + }); + + await getListPaginated({ req, env: {}, daCtx }); + assert.strictEqual(1, loCalled.length); + assert.strictEqual('q/q', loCalled[0].c.key); + + const childRules = aclCtx.childRules; + assert.strictEqual(1, childRules.length); + assert(childRules[0].startsWith('/q/q/**='), 'Should have defined some child rule'); + }); + + it('parses request params', async () => { + const loCalled = []; + listObjectsPaginated.mockImplementation((e, c, limit, offset) => { + console.log({offset, limit}); + loCalled.push({ offset, limit }); + return {}; + }); + + hasPermission.mockImplementation(() => true); + getChildRules.mockImplementation(() => {}); + + const ctx = { org: 'foo' }; + const reqs = [ + { url: 'https://admin.da.live/list/foo/bar?limit=12&offset=1' }, + { url: 'https://admin.da.live/list/foo/bar?limit=asdf&offset=17' }, + { url: 'https://admin.da.live/list/foo/bar?limit=12&offset=asdf' }, + ]; + await getListPaginated({ req: reqs[0], env: {}, daCtx: ctx, aclCtx: {} }); + assert.deepStrictEqual(loCalled[0], { limit: 12, offset: 1 }); + await getListPaginated({ req: reqs[1], env: {}, daCtx: ctx, aclCtx: {} }); + assert.deepStrictEqual(loCalled[1], { limit: undefined, offset: 17 }); + await getListPaginated({ req: reqs[2], env: {}, daCtx: ctx, aclCtx: {} }); + assert.deepStrictEqual(loCalled[2], { limit: 12, offset: undefined }); + }); +}); diff --git a/test/storage/object/list.test.js b/test/storage/object/list.test.js index 9d805e61..974e975d 100644 --- a/test/storage/object/list.test.js +++ b/test/storage/object/list.test.js @@ -11,14 +11,17 @@ */ import { ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3'; +import assert from "assert"; import { mockClient } from 'aws-sdk-client-mock'; import { describe, it, beforeEach, expect, afterEach, vi } from 'vitest'; const s3Mock = mockClient(S3Client); -import listObjects from '../../../src/storage/object/list.js'; +import listObjects, {listObjectsPaginated} from '../../../src/storage/object/list.js'; const Contents = [ + { Key: 'wknd/abc1234.html', LastModified: new Date() }, + { Key: 'wknd/abc123.html', LastModified: new Date() }, { Key: 'wknd/index.html', LastModified: new Date() }, { Key: 'wknd/nav.html', LastModified: new Date() }, { Key: 'wknd/footer.html', LastModified: new Date() }, @@ -44,7 +47,7 @@ describe('List Objects', () => { const daCtx = { org: 'adobe', key: 'wknd' }; const resp = await listObjects({}, daCtx); const data = JSON.parse(resp.body); - expect(data.length).toBe(3); + expect(data.length).toBe(Contents.length); expect(data.every((item) => item.ext && item.lastModified)).toBe(true); }); @@ -61,4 +64,123 @@ describe('List Objects', () => { const data = JSON.parse(resp.body); expect(data.length).toBe(2); }); + + it('sorts the results', async () => { + s3Mock.on(ListObjectsV2Command, { + Bucket: 'adobe-content', + Prefix: 'wknd/', + Delimiter: '/', + }).resolves({ $metadata: { httpStatusCode: 200 }, Contents }); + + const daCtx = { org: 'adobe', key: 'wknd' }; + const resp = await listObjects({}, daCtx); + const data = JSON.parse(resp.body); + + const firstIndex = data.findIndex((x) => x.name === 'abc123'); + const secondIndex = data.findIndex((x) => x.name === 'abc1234'); + expect(firstIndex).to.be.lessThan(secondIndex); + }); +}); + +describe('list paginated objects', async () => { + it('correctly handles continuation token', async () => { + s3Mock.on(ListObjectsV2Command, { + Bucket: 'adobe-content', + Prefix: 'wknd/', + Delimiter: '/', + }).resolves({ + $metadata: { httpStatusCode: 200 }, + Contents: [Contents[0], Contents[1]], + NextContinuationToken: 'token' + }); + + s3Mock.on(ListObjectsV2Command, { + Bucket: 'adobe-content', + Prefix: 'wknd/', + Delimiter: '/', + ContinuationToken: 'token' + }).resolves({ $metadata: { httpStatusCode: 200 }, Contents: [Contents[2], Contents[3]] }); + + const daCtx = { org: 'adobe', key: 'wknd' }; + const resp = await listObjectsPaginated({}, daCtx); + const { data, limit, offset } = JSON.parse(resp.body); + assert.strictEqual(data.length, 4, 'Should return all items'); + assert.strictEqual(limit, 1000, 'Should use default limit if no limit passed'); + assert.strictEqual(offset, 0, 'Should use default offset if no limit passed'); + }); + + it('correctly passes limit and offset', async () => { + s3Mock.on(ListObjectsV2Command, { + Bucket: 'adobe-content', + Prefix: 'wknd/', + Delimiter: '/', + MaxKeys: 27, + }).resolves({ + $metadata: { httpStatusCode: 200 }, + Contents: Contents, + NextContinuationToken: 'token', + }); + + const daCtx = { org: 'adobe', key: 'wknd' }; + const resp = await listObjectsPaginated({}, daCtx, 2, 1); + const { data, limit, offset } = JSON.parse(resp.body); + assert.strictEqual(data.length, 2, 'Should return 2 items'); + assert.strictEqual(data[1].name, 'index', 'Should return correct items'); + assert.strictEqual(limit, 2, 'Should use default limit if no limit passed'); + assert.strictEqual(offset, 1, 'Should use default offset if no limit passed'); + }); + + it('fetches more until enough files are present', async () => { + s3Mock.on(ListObjectsV2Command, { + Bucket: 'adobe-content', + Prefix: 'wknd/', + Delimiter: '/', + MaxKeys: 29, + }).resolves({ + $metadata: { httpStatusCode: 200 }, + Contents: new Array(29).fill({ Key: '.ignored', LastModified: new Date() }), + NextContinuationToken: 'token', + }); + + s3Mock.on(ListObjectsV2Command, { + Bucket: 'adobe-content', + Prefix: 'wknd/', + Delimiter: '/', + MaxKeys: 29, + ContinuationToken: 'token', + }).resolves({ + $metadata: { httpStatusCode: 200 }, + Contents: Contents, + NextContinuationToken: 'token', + }); + + const daCtx = { org: 'adobe', key: 'wknd' }; + const resp = await listObjectsPaginated({}, daCtx, 4, 0); + const { data } = JSON.parse(resp.body); + assert.strictEqual(data.length, 4, 'Should return 4 items'); + }); + + it('doesn\'t sort the results', async () => { + s3Mock.on(ListObjectsV2Command, { + Bucket: 'adobe-content', + Prefix: 'wknd/', + Delimiter: '/', + }).resolves({ $metadata: { httpStatusCode: 200 }, Contents }); + + const daCtx = { org: 'adobe', key: 'wknd' }; + const resp = await listObjectsPaginated({}, daCtx); + const data = JSON.parse(resp.body).data; + + const firstIndex = data.findIndex((x) => x.name === 'abc1234'); + const secondIndex = data.findIndex((x) => x.name === 'abc123'); + assert.strictEqual(true, firstIndex < secondIndex); + }); + + it('enforces size limit', async () => { + const daCtx = { org: 'adobe', key: 'wknd' }; + const resp1 = await listObjectsPaginated({}, daCtx, 5001, 0); + assert.strictEqual(resp1.status, 400); + const resp2 = await listObjectsPaginated({}, daCtx, 500, 4501); + assert.strictEqual(resp2.status, 400); + }); }); diff --git a/test/storage/utils.test.js b/test/storage/utils.test.js index 5e8d2515..69def289 100644 --- a/test/storage/utils.test.js +++ b/test/storage/utils.test.js @@ -1,51 +1,80 @@ /* eslint-env mocha */ import assert from 'assert'; +import formatList, { formatPaginatedList } from '../../src/storage/utils/list.js'; + import { describe, it } from 'vitest'; -import formatList from '../../src/storage/utils/list.js'; - -const MOCK = { - CommonPrefixes: [ - { Prefix: 'blog/' }, - { Prefix: 'da-aem-boilerplate/' }, - { Prefix: 'da/' }, - { Prefix: 'dac/' }, - { Prefix: 'milo/' }, - { Prefix: 'dark-alley.jpg/' }, - ], - Contents: [ - { - Key: 'blog.props', - LastModified: new Date(), - }, - { - Key: 'da.props', - LastModified: new Date(), - }, - { - Key: 'folder-only.props', - LastModified: new Date(), - }, - { - Key: 'test.html', - LastModified: new Date(), - }, - { - Key: 'dark-alley.jpg.props', - LastModified: new Date(), - }, - { - Key: 'dark-alley.jpg', - LastModified: new Date(), - } - ], -}; +function getMock() { + return { + CommonPrefixes: [ + { Prefix: 'blog/' }, + { Prefix: 'da-aem-boilerplate/' }, + { Prefix: 'da/' }, + { Prefix: 'dac/' }, + { Prefix: 'milo/' }, + { Prefix: 'dark-alley.jpg/' }, + ], + Contents: [ + { + Key: 'blog.props', + LastModified: new Date(), + }, + { + Key: 'da.props', + LastModified: new Date(), + }, + { + Key: 'folder-only.props', + LastModified: new Date(), + }, + { + Key: 'test.html', + LastModified: new Date(), + }, + { + Key: 'dark-alley.jpg.props', + LastModified: new Date(), + }, + { + Key: 'dark-alley.jpg', + LastModified: new Date(), + } + ], + }; +} const daCtx = { url: 'https://admin.da.live/list/foo/bar' }; describe('Format object list', () => { - const list = formatList(MOCK, daCtx); + const list = formatList(getMock(), daCtx); + + it('should return a true folder / common prefix', () => { + assert.strictEqual(list[0].name, 'blog'); + }); + + it('should return a contents-based folder', () => { + const folderOnly = list.find((item) => { return item.name === 'folder-only' }); + assert.strictEqual(folderOnly.name, 'folder-only'); + }); + + it('should not return a props file of same folder name', () => { + const found = list.reduce((acc, item) => { + if (item.name === 'blog') acc.push(item); + return acc; + },[]); + + assert.strictEqual(found.length, 1); + }); + + it('should not have a filename props file in the list', () => { + const propsSidecar = list.find((item) => { return item.name === 'dark-alley.jpg.props' }); + assert.strictEqual(propsSidecar, undefined); + }); +}); + +describe('format paginated object list', () => { + const list = formatPaginatedList(getMock(), daCtx); it('should return a true folder / common prefix', () => { assert.strictEqual(list[0].name, 'blog');