diff --git a/docs/CNAME b/docs/CNAME deleted file mode 100644 index 7ad473bf..00000000 --- a/docs/CNAME +++ /dev/null @@ -1 +0,0 @@ -docs.da.live \ No newline at end of file diff --git a/src/storage/object/put.js b/src/storage/object/put.js index 36f2611b..06c9513c 100644 --- a/src/storage/object/put.js +++ b/src/storage/object/put.js @@ -52,20 +52,6 @@ function createBucketIfMissing(client) { ); } -/** - * Check to see if the org is in the existing list of orgs - * - * @param {Object} env the cloud provider environment - * @param {*} org the org associated with the bucket - * @returns null - */ -async function checkOrgIndex(env, org) { - const orgs = await env.DA_AUTH.get('orgs', { type: 'json' }); - if (orgs.some((existingOrg) => existingOrg.name === org)) return; - orgs.push({ name: org, created: new Date().toISOString() }); - await env.DA_AUTH.put('orgs', JSON.stringify(orgs)); -} - export default async function putObject(env, daCtx, obj) { const config = getS3Config(env); const client = new S3Client(config); @@ -74,8 +60,6 @@ export default async function putObject(env, daCtx, obj) { // Only allow creating a new bucket for orgs and repos if (key.split('/').length <= 1) { - // await checkOrgIndex(env, org); - // R2 ONLY FEATURE createBucketIfMissing(client); } diff --git a/src/storage/utils/list.js b/src/storage/utils/list.js index 5d6e5ee9..ee8b962c 100644 --- a/src/storage/utils/list.js +++ b/src/storage/utils/list.js @@ -50,9 +50,9 @@ export default function formatList(resp, daCtx) { // Do not show any props sidecar files if (props) return; - // See if the folder is already in the list if (ext === 'props') { - if (combined.some((item) => item.name === name)) return; + // Do not add if it already exists as a folder (does not have an extension) + if (combined.some((item) => item.name === name && !item.ext)) return; // Remove props from the key so it can look like a folder // eslint-disable-next-line no-param-reassign diff --git a/src/utils/daResp.js b/src/utils/daResp.js index e534d7f7..c52ac4fb 100644 --- a/src/utils/daResp.js +++ b/src/utils/daResp.js @@ -29,6 +29,10 @@ export default function daResp({ headers.append('X-da-id', metadata.id); } + if (metadata?.timestamp) { + headers.append('Last-Modified', new Date(parseInt(metadata.timestamp, 10)).toUTCString()); + } + if (ctx?.aclCtx && status < 500) { headers.append('X-da-actions', `/${ctx.key}=${[...ctx.aclCtx.actionSet]}`); diff --git a/test/storage/object/delete.test.js b/test/storage/object/delete.test.js index c6259da0..63fe3171 100644 --- a/test/storage/object/delete.test.js +++ b/test/storage/object/delete.test.js @@ -12,8 +12,7 @@ import assert from 'node:assert'; import esmock from 'esmock'; import { mockClient } from 'aws-sdk-client-mock'; -import { DeleteObjectCommand, ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3'; -import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3'; const s3Mock = mockClient(S3Client); @@ -252,11 +251,6 @@ describe('Object delete', () => { getSignedUrl: mockSignedUrl, } }, - { - import: { - fetch: async () => ({ status: 200 }), - } - } ); s3Mock.on(ListObjectsV2Command).resolves({ Contents: [{ Key: 'foo/bar.html' }] }); const resp = await deleteObjects(env, daCtx, {}); @@ -287,11 +281,6 @@ describe('Object delete', () => { getSignedUrl: mockSignedUrl, } }, - { - import: { - fetch: async () => ({ status: 200 }), - } - } ); s3Mock.on(ListObjectsV2Command).resolves({ Contents: [{ Key: 'foo/bar.html' }], NextContinuationToken: 'token' }); const resp = await deleteObjects(env, daCtx, {}); diff --git a/test/storage/utils.t.js b/test/storage/utils.t.js deleted file mode 100644 index 6148f275..00000000 --- a/test/storage/utils.t.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-env mocha */ -import assert from 'assert'; - -import { getDaCtx } from '../../src/utils/daCtx.js'; -import formatList from '../../src/storage/utils/list.js'; - -const MOCK = { - CommonPrefixes: [ - { Prefix: 'blog/' }, - { Prefix: 'da-aem-boilerplate/' }, - { Prefix: 'da/' }, - { Prefix: 'dac/' }, - { Prefix: 'milo/' }, - { Prefix: 'dark-alley.jpg/' }, - ], - Contents: [ - { - Key: 'blog.props', - }, - { - Key: 'da.props', - }, - { - Key: 'folder-only.props', - }, - { - Key: 'test.html', - }, - { - Key: 'dark-alley.jpg.props', - }, - { - Key: 'dark-alley.jpg', - } - ], -}; - -const daCtx = getDaCtx('/source/adobecom'); - -describe('Format object list', () => { - const list = formatList(MOCK, daCtx); - - it('should return a true folder / common prefix', () => { - assert.strictEqual(list[0].name, 'blog'); - }); - - it('should return a contents-based folder', () => { - const folderOnly = list.find((item) => { return item.name === 'folder-only' }); - assert.strictEqual(folderOnly.name, 'folder-only'); - }); - - it('should not return a props file of same folder name', () => { - const found = list.reduce((acc, item) => { - if (item.name === 'blog') acc.push(item); - return acc; - },[]); - - assert.strictEqual(found.length, 1); - }); - - it('should not have a filename props file in the list', () => { - const propsSidecar = list.find((item) => { return item.name === 'dark-alley.jpg.props' }); - assert.strictEqual(propsSidecar, undefined); - }); -}); diff --git a/test/storage/utils/list.test.js b/test/storage/utils/list.test.js new file mode 100644 index 00000000..12da216b --- /dev/null +++ b/test/storage/utils/list.test.js @@ -0,0 +1,299 @@ +/* eslint-env mocha */ +import assert from 'assert'; +import sinon from 'sinon'; + +import getDaCtx from '../../../src/utils/daCtx.js'; +import formatList, { listCommand } from '../../../src/storage/utils/list.js'; + +const MOCK = { + CommonPrefixes: [ + { Prefix: 'da-aem-boilerplate/' }, + { Prefix: 'blog/' }, + { Prefix: 'da/' }, + { Prefix: 'dac/' }, + { Prefix: 'milo/' }, + { Prefix: 'dark-alley.jpg/' }, + ], + Contents: [ + { + Key: 'blog.props', + LastModified: new Date('2025-01-01'), + }, + { + Key: 'da.props', + LastModified: new Date('2025-01-01'), + }, + { + Key: 'folder-only.props', + LastModified: new Date('2025-01-01'), + }, + { + Key: 'test.html', + LastModified: new Date('2025-01-01'), + }, + { + Key: 'dark-alley.jpg.props', + LastModified: new Date('2025-01-01'), + }, + { + Key: 'dark-alley.jpg', + LastModified: new Date('2025-01-01'), + }, + { + Key: 'empty-folder-with-sibling-file.props', + LastModified: new Date('2025-01-01'), + }, + { + Key: 'empty-folder-with-sibling-file.html', + LastModified: new Date('2025-01-01'), + } + ], +}; + +const req = new Request('https://example.com/source/adobecom'); + +const daCtx = getDaCtx(req, {}); + +describe('Format object list', () => { + const list = formatList(MOCK, daCtx); + + it('should return a true folder / common prefix', () => { + assert.strictEqual(list[0].name, 'blog'); + }); + + it('should return a contents-based folder', () => { + const folderOnly = list.find((item) => { return item.name === 'folder-only' }); + assert.strictEqual(folderOnly.name, 'folder-only'); + }); + + it('should not return a props file of same folder name', () => { + const found = list.reduce((acc, item) => { + if (item.name === 'blog') acc.push(item); + return acc; + },[]); + + assert.strictEqual(found.length, 1); + }); + + it('should not have a filename props file in the list', () => { + const propsSidecar = list.find((item) => { return item.name === 'dark-alley.jpg.props' }); + assert.strictEqual(propsSidecar, undefined); + }); + + it('should handle empty folders with sibling file names of same name', () => { + const filtered = list.filter((item) => { return item.name === 'empty-folder-with-sibling-file' }); + assert.strictEqual(filtered.length, 2); + }); + + it('should handle empty CommonPrefixes', () => { + const emptyMock = { Contents: MOCK.Contents }; + const result = formatList(emptyMock, daCtx); + assert(Array.isArray(result)); + assert(result.length > 0); + }); + + it('should handle empty Contents', () => { + const emptyMock = { CommonPrefixes: MOCK.CommonPrefixes }; + const result = formatList(emptyMock, daCtx); + assert(Array.isArray(result)); + assert(result.length > 0); + }); + + it('should handle both empty CommonPrefixes and Contents', () => { + const emptyMock = {}; + const result = formatList(emptyMock, daCtx); + assert(Array.isArray(result)); + assert.strictEqual(result.length, 0); + }); + + it('should filter out extension folders from CommonPrefixes', () => { + const mockWithExtensionFolder = { + CommonPrefixes: [ + { Prefix: 'file.jpg/' }, + { Prefix: 'normal-folder/' } + ] + }; + const result = formatList(mockWithExtensionFolder, daCtx); + const extensionFolder = result.find(item => item.name === 'file.jpg'); + assert.strictEqual(extensionFolder, undefined); + const normalFolder = result.find(item => item.name === 'normal-folder'); + assert(normalFolder); + }); + + it('should handle files with more than 2 dot separators', () => { + const mockWithComplexFile = { + Contents: [ + { + Key: 'file.name.with.multiple.dots', + LastModified: new Date('2025-01-01'), + } + ] + }; + const result = formatList(mockWithComplexFile, daCtx); + assert.strictEqual(result.length, 0); + }); + + it('should handle hidden files (starting with dot)', () => { + const mockWithHiddenFile = { + Contents: [ + { + Key: '.hidden-file', + LastModified: new Date('2025-01-01'), + } + ] + }; + const result = formatList(mockWithHiddenFile, daCtx); + assert.strictEqual(result.length, 0); + }); + + it('should handle files with props extension correctly', () => { + const mockWithProps = { + Contents: [ + { + Key: 'test.props', + LastModified: new Date('2025-01-01'), + } + ] + }; + const result = formatList(mockWithProps, daCtx); + const propsItem = result.find(item => item.name === 'test'); + assert(propsItem); + assert.strictEqual(propsItem.ext, undefined); + assert.strictEqual(propsItem.lastModified, undefined); + }); + + it('should not add props file if folder already exists', () => { + const mockWithBoth = { + CommonPrefixes: [{ Prefix: 'test/' }], + Contents: [ + { + Key: 'test.props', + LastModified: new Date('2025-01-01'), + } + ] + }; + const result = formatList(mockWithBoth, daCtx); + const testItems = result.filter(item => item.name === 'test'); + assert.strictEqual(testItems.length, 1); + }); + + it('should sort results alphabetically', () => { + const mockForSorting = { + Contents: [ + { Key: 'zebra.html', LastModified: new Date('2025-01-01') }, + { Key: 'alpha.html', LastModified: new Date('2025-01-01') }, + { Key: 'beta.html', LastModified: new Date('2025-01-01') } + ] + }; + const result = formatList(mockForSorting, daCtx); + assert.strictEqual(result[0].name, 'alpha'); + assert.strictEqual(result[1].name, 'beta'); + assert.strictEqual(result[2].name, 'zebra'); + }); +}); + +describe('listCommand', () => { + let mockS3Client; + let testDaCtx; + + beforeEach(() => { + mockS3Client = { + send: sinon.stub() + }; + + // Create a proper daCtx object for testing + testDaCtx = { + bucket: 'test-bucket', + org: 'adobecom', + key: 'test', + ext: undefined + }; + }); + + afterEach(() => { + sinon.restore(); + }); + + it('should return sourceKeys array when item has extension', async () => { + const daCtxWithExt = { ...testDaCtx, ext: 'html' }; + const result = await listCommand(daCtxWithExt, {}, mockS3Client); + + assert.deepStrictEqual(result, { sourceKeys: [testDaCtx.key] }); + assert.strictEqual(mockS3Client.send.callCount, 0); + }); + + it('should call S3 list command when no extension', async () => { + const mockResponse = { + Contents: [ + { Key: 'adobecom/test/file1.html' }, + { Key: 'adobecom/test/file2.html' } + ], + NextContinuationToken: 'next-token' + }; + + mockS3Client.send.resolves(mockResponse); + + const result = await listCommand(testDaCtx, {}, mockS3Client); + + assert.strictEqual(mockS3Client.send.callCount, 1); + assert.deepStrictEqual(result, { + sourceKeys: [testDaCtx.key, `${testDaCtx.key}.props`, 'adobecom/test/file1.html', 'adobecom/test/file2.html'], + continuationToken: 'next-token' + }); + }); + + it('should handle continuation token', async () => { + const mockResponse = { + Contents: [ + { Key: 'adobecom/test/file3.html' } + ] + }; + + mockS3Client.send.resolves(mockResponse); + + const details = { continuationToken: 'prev-token' }; + const result = await listCommand(testDaCtx, details, mockS3Client); + + assert.strictEqual(mockS3Client.send.callCount, 1); + const callArgs = mockS3Client.send.firstCall.args[0]; + console.log('Call args:', JSON.stringify(callArgs, null, 2)); + // The command should have the continuation token + assert.strictEqual(callArgs.input.ContinuationToken, 'prev-token'); + assert.deepStrictEqual(result, { + sourceKeys: ['adobecom/test/file3.html'], + continuationToken: undefined + }); + }); + + it('should handle empty Contents response', async () => { + const mockResponse = { + Contents: [] + }; + + mockS3Client.send.resolves(mockResponse); + + const result = await listCommand(testDaCtx, {}, mockS3Client); + + assert.deepStrictEqual(result, { + sourceKeys: [testDaCtx.key, `${testDaCtx.key}.props`], + continuationToken: undefined + }); + }); + + it('should handle response without NextContinuationToken', async () => { + const mockResponse = { + Contents: [ + { Key: 'adobecom/test/file1.html' } + ] + }; + + mockS3Client.send.resolves(mockResponse); + + const result = await listCommand(testDaCtx, {}, mockS3Client); + + assert.deepStrictEqual(result, { + sourceKeys: [testDaCtx.key, `${testDaCtx.key}.props`, 'adobecom/test/file1.html'], + continuationToken: undefined + }); + }); +}); diff --git a/test/utils/auth.test.js b/test/utils/auth.test.js index 551ef7ce..485aabfa 100644 --- a/test/utils/auth.test.js +++ b/test/utils/auth.test.js @@ -29,7 +29,17 @@ import { const { setUser, getUsers, -} = await esmock('../../src/utils/auth.js', { jose, import: { fetch } }); +} = await esmock('../../src/utils/auth.js', { jose }); + +async function withMockedFetch(act) { + const savedFetch = globalThis.fetch; + globalThis.fetch = fetch; + try { + await act(); + } finally { + globalThis.fetch = savedFetch; + } +} describe('DA auth', () => { describe('get user', async () => { @@ -49,14 +59,18 @@ describe('DA auth', () => { }); it('authorized if email matches', async () => { - const users = await getUsers(reqs.site, env); - assert.strictEqual(users[0].email, 'aparker@geometrixx.info'); + await withMockedFetch(async () => { + const users = await getUsers(reqs.site, env); + assert.strictEqual(users[0].email, 'aparker@geometrixx.info'); + }); }); it('authorized with user if email matches and anonymous if present', async () => { - const users = await getUsers(reqs.siteMulti, env); - assert.strictEqual(users[0].email, 'anonymous') - assert.strictEqual(users[1].email, 'aparker@geometrixx.info'); + await withMockedFetch(async () => { + const users = await getUsers(reqs.siteMulti, env); + assert.strictEqual(users[0].email, 'anonymous') + assert.strictEqual(users[1].email, 'aparker@geometrixx.info'); + }); }); it('anonymous if ims fails', async () => { @@ -71,8 +85,12 @@ describe('DA auth', () => { 'Authorization': `Bearer aparker@geometrixx.info`, }); - const userValStr = await setUser('aparker@geometrixx.info', 100, headers, env); - const userValue = JSON.parse(userValStr); + let userValue; + + await withMockedFetch(async () => { + const userValStr = await setUser('aparker@geometrixx.info', 100, headers, env); + userValue = JSON.parse(userValStr); + }); assert.strictEqual('aparker@geometrixx.info', userValue.email); assert.strictEqual('123', userValue.ident); diff --git a/test/utils/daResp.test.js b/test/utils/daResp.test.js index 062cdfd6..dde77221 100644 --- a/test/utils/daResp.test.js +++ b/test/utils/daResp.test.js @@ -19,7 +19,7 @@ describe('DA Resp', () => { const aclCtx = { actionSet: ['read', 'write'], pathLookup: new Map() }; const ctx = { key: 'foo/bar.html', aclCtx }; const body = 'foobar'; - const metadata = { id: '1234' }; + const metadata = { id: '1234', timestamp: '1719235200000' }; const resp = daResp({status: 200, body, contentType: 'text/plain', contentLength: 777, metadata}, ctx); assert.strictEqual(body, await resp.text()); @@ -32,6 +32,7 @@ describe('DA Resp', () => { assert.strictEqual('777', resp.headers.get('Content-Length')); assert.strictEqual('/foo/bar.html=read,write', resp.headers.get('X-da-actions')); assert.strictEqual('1234', resp.headers.get('X-da-id')); + assert.strictEqual('Mon, 24 Jun 2024 13:20:00 GMT', resp.headers.get('Last-Modified')); assert(resp.headers.get('X-da-acltrace') === null); });