diff --git a/apps/backend/src/__tests__/conversations.cache.test.ts b/apps/backend/src/__tests__/conversations.cache.test.ts index 7eb854c..48408a4 100644 --- a/apps/backend/src/__tests__/conversations.cache.test.ts +++ b/apps/backend/src/__tests__/conversations.cache.test.ts @@ -198,10 +198,14 @@ describe('GET /conversations/:id/search', () => { vi.clearAllMocks(); }); - it('returns 501 for E2EE environments', async () => { + it('returns 410 Gone for E2EE environments', async () => { const res = await request(makeApp()).get('/conversations/conv-1/search?q=hello'); - expect(res.status).toBe(501); + expect(res.status).toBe(410); + expect(res.body).toEqual({ + error: 'Server-side search removed; search is now client-side over decrypted messages', + docs: 'https://github.com/DripWave/clicked/blob/main/docs/message-encryption-migration.md' + }); }); }); diff --git a/apps/backend/src/routes/conversations.ts b/apps/backend/src/routes/conversations.ts index 252503c..83bc2dd 100644 --- a/apps/backend/src/routes/conversations.ts +++ b/apps/backend/src/routes/conversations.ts @@ -512,7 +512,10 @@ conversationsRouter.get('/:id/messages', async (req: AuthRequest, res) => { }); conversationsRouter.get('/:id/search', async (req: AuthRequest, res) => { - res.status(501).json({ error: 'Search is not supported in E2EE conversations' }); + res.status(410).json({ + error: 'Server-side search removed; search is now client-side over decrypted messages', + docs: 'https://github.com/DripWave/clicked/blob/main/docs/message-encryption-migration.md' + }); }); // PATCH /conversations/:id/settings — update muted/archived state for the authenticated user diff --git a/apps/backend/src/routes/devices.ts b/apps/backend/src/routes/devices.ts index 0051370..a64a660 100644 --- a/apps/backend/src/routes/devices.ts +++ b/apps/backend/src/routes/devices.ts @@ -388,7 +388,8 @@ async function emitDeviceChangeEvent(userId: string, change: 'device_added' | 'd .values({ conversationId: m.conversationId, senderId: userId, - content: JSON.stringify({ userId, change }), + contentType: 'system', + ciphertext: JSON.stringify({ userId, change }), }) .returning(); diff --git a/apps/backend/src/routes/messages.ts b/apps/backend/src/routes/messages.ts index f09c4ba..eff7cb7 100644 --- a/apps/backend/src/routes/messages.ts +++ b/apps/backend/src/routes/messages.ts @@ -61,50 +61,51 @@ messagesRouter.post('/', validate(SendMessageSchema), async (req: AuthRequest, r return; } - if (fileId) { - const fileRecord = await db.query.files.findFirst({ - where: eq(files.id, fileId), + // ── persist in transaction ───────────────────────────────────────────────── + let message; + try { + message = await db.transaction(async (tx) => { + const [insertedMessage] = await tx + .insert(messages) + .values({ + id: messageId, + conversationId, + senderId: userId, + senderDeviceId: deviceId ?? null, + contentType: contentType?.trim().toLowerCase() || 'text', + ciphertext: ciphertext || null, + fileId: fileId ?? null, + }) + .returning(); + + if (envelopes && envelopes.length > 0) { + const deviceIds = envelopes.map((e) => e.recipientDeviceId); + const devicesList = await tx.query.userDevices.findMany({ + where: inArray(userDevices.id, deviceIds), + columns: { id: true, userId: true }, + }); + const deviceToUser = new Map(devicesList.map((d) => [d.id, d.userId])); + + const validEnvelopes = envelopes + .filter((env) => deviceToUser.has(env.recipientDeviceId)) + .map((env) => ({ + messageId, + recipientDeviceId: env.recipientDeviceId, + recipientUserId: deviceToUser.get(env.recipientDeviceId)!, + ciphertext: env.ciphertext, + })); + + if (validEnvelopes.length > 0) { + await tx.insert(messageEnvelopes).values(validEnvelopes); + } + } + + return insertedMessage; }); - if (!fileRecord || fileRecord.status !== 'ready') { - res.status(400).json({ error: 'File is not ready or does not exist' }); - return; - } - } - - // ── persist ──────────────────────────────────────────────────────────────── - const [message] = await db - .insert(messages) - .values({ - id: messageId, - conversationId, - senderId: userId, - senderDeviceId: deviceId ?? null, - contentType: contentType?.trim().toLowerCase() || 'text', - ciphertext: ciphertext || null, - fileId: fileId || null, - }) - .returning(); - - if (envelopes && envelopes.length > 0) { - const deviceIds = envelopes.map((e) => e.recipientDeviceId); - const devicesList = await db.query.userDevices.findMany({ - where: inArray(userDevices.id, deviceIds), - columns: { id: true, userId: true }, - }); - const deviceToUser = new Map(devicesList.map((d) => [d.id, d.userId])); - - const validEnvelopes = envelopes - .filter((env) => deviceToUser.has(env.recipientDeviceId)) - .map((env) => ({ - messageId, - recipientDeviceId: env.recipientDeviceId, - recipientUserId: deviceToUser.get(env.recipientDeviceId)!, - ciphertext: env.ciphertext, - })); - - if (validEnvelopes.length > 0) { - await db.insert(messageEnvelopes).values(validEnvelopes); - } + } catch (error) { + console.error('Transaction failed for message insert:', error); + res.status(500).json({ error: 'Failed to persist message' }); + return; } // ── broadcast via Socket.IO ──────────────────────────────────────────────── diff --git a/apps/backend/src/socket/messaging.ts b/apps/backend/src/socket/messaging.ts index c6f2f63..ead7830 100644 --- a/apps/backend/src/socket/messaging.ts +++ b/apps/backend/src/socket/messaging.ts @@ -184,22 +184,63 @@ export function registerMessagingHandlers(io: Server, socket: AuthSocket): void fileId = fileRow?.id ?? payloadFileId; } - const [message] = await db - .insert(messages) - .values({ - id: messageId, - conversationId, - senderId: userId, - senderDeviceId: deviceId, - contentType: resolvedContentType, - ciphertext: effectiveCiphertext, - fileId: fileId ?? null, - }) - .returning(); - + let message; let recipientDeviceIds: string[] = []; + try { + message = await db.transaction(async (tx) => { + const [insertedMessage] = await tx + .insert(messages) + .values({ + id: messageId, + conversationId, + senderId: userId, + senderDeviceId: deviceId, + contentType: resolvedContentType, + ciphertext: effectiveCiphertext, + fileId: fileId ?? null, + }) + .returning(); + + if (envelopes && envelopes.length > 0) { + const deviceIds = envelopes.map((e) => e.recipientDeviceId); + const devicesList = await tx.query.userDevices.findMany({ + where: inArray(userDevices.id, deviceIds), + columns: { id: true, userId: true }, + }); + const deviceToUser = new Map(devicesList.map((d) => [d.id, d.userId])); + + const validEnvelopes = envelopes + .filter((env) => deviceToUser.has(env.recipientDeviceId)) + .map((env) => ({ + messageId, + recipientDeviceId: env.recipientDeviceId, + recipientUserId: deviceToUser.get(env.recipientDeviceId)!, + ciphertext: env.ciphertext, + })); - if (envelopes && envelopes.length > 0) { + if (validEnvelopes.length > 0) { + await tx.insert(messageEnvelopes).values(validEnvelopes); + recipientDeviceIds = validEnvelopes.map((e) => e.recipientDeviceId); + } + } + + return insertedMessage; + }); + } catch (error) { + console.error('Transaction failed for message insert:', error); + socket.emit('error', { event: 'send_message', message: 'Failed to persist message' }); + return; + } + + if (!message) { + socket.emit('error', { event: 'send_message', message: 'Failed to persist message' }); + return; + } + + socket.emit('message_ack', { messageId, sequenceNumber: message.sequenceNumber }); + + // Publish to Redis after transaction commit + if (redis && envelopes && envelopes.length > 0) { const deviceIds = envelopes.map((e) => e.recipientDeviceId); const devicesList = await db.query.userDevices.findMany({ where: inArray(userDevices.id, deviceIds), @@ -216,31 +257,16 @@ export function registerMessagingHandlers(io: Server, socket: AuthSocket): void ciphertext: env.ciphertext, })); - if (validEnvelopes.length > 0) { - await db.insert(messageEnvelopes).values(validEnvelopes); - - if (redis && message) { - for (const env of validEnvelopes) { - publishToDevice(redis, env.recipientDeviceId, { - messageId: message.id, - conversationId, - ciphertext: env.ciphertext, - sequenceNumber: message.sequenceNumber, - }).catch(() => {}); - } - } - - recipientDeviceIds = validEnvelopes.map((e) => e.recipientDeviceId); + for (const env of validEnvelopes) { + publishToDevice(redis, env.recipientDeviceId, { + messageId: message.id, + conversationId, + ciphertext: env.ciphertext, + sequenceNumber: message.sequenceNumber, + }).catch(() => {}); } } - if (!message) { - socket.emit('error', { event: 'send_message', message: 'Failed to persist message' }); - return; - } - - socket.emit('message_ack', { messageId, sequenceNumber: message.sequenceNumber }); - await deliverMessage(io, message, conversationId); const members = await db.query.conversationMembers.findMany({ @@ -330,59 +356,54 @@ export function registerMessagingHandlers(io: Server, socket: AuthSocket): void return; } - // Enforce full sibling-device coverage (#188). - const siblingIds = await fetchSiblingDeviceIds(userId, deviceId); - if (siblingIds.length > 0) { - const providedIds = new Set(envelopes?.map((e) => e.recipientDeviceId) ?? []); - const missing = siblingIds.filter((id) => !providedIds.has(id)); - if (missing.length > 0) { - socket.emit('error', { - event: 'device_set_mismatch', - message: `Missing envelopes for ${missing.length} sibling device(s)`, - missingDeviceIds: missing, - }); - return; - } - } - - const [message] = await db - .insert(messages) - .values({ - id: messageId, - conversationId, - senderId: userId, - senderDeviceId: deviceId, - contentType: contentType || original.contentType, - ciphertext: ciphertext || null, - editsMessageId: rootMessageId, - }) - .returning(); - + let message; let recipientDeviceIds: string[] = []; + try { + message = await db.transaction(async (tx) => { + const [insertedMessage] = await tx + .insert(messages) + .values({ + id: messageId, + conversationId, + senderId: userId, + senderDeviceId: deviceId, + contentType: contentType || original.contentType, + ciphertext: ciphertext || null, + editsMessageId: rootMessageId, + }) + .returning(); + + if (envelopes && envelopes.length > 0) { + const deviceIds = envelopes.map((e) => e.recipientDeviceId); + + const devicesList = await tx.query.userDevices.findMany({ + where: inArray(userDevices.id, deviceIds), + columns: { id: true, userId: true }, + }); - if (envelopes && envelopes.length > 0) { - const deviceIds = envelopes.map((e) => e.recipientDeviceId); - - const devicesList = await db.query.userDevices.findMany({ - where: inArray(userDevices.id, deviceIds), - columns: { id: true, userId: true }, - }); + const deviceToUser = new Map(devicesList.map((d) => [d.id, d.userId])); - const deviceToUser = new Map(devicesList.map((d) => [d.id, d.userId])); + const validEnvelopes = envelopes + .filter((env) => deviceToUser.has(env.recipientDeviceId)) + .map((env) => ({ + messageId, + recipientDeviceId: env.recipientDeviceId, + recipientUserId: deviceToUser.get(env.recipientDeviceId)!, + ciphertext: env.ciphertext, + })); - const validEnvelopes = envelopes - .filter((env) => deviceToUser.has(env.recipientDeviceId)) - .map((env) => ({ - messageId, - recipientDeviceId: env.recipientDeviceId, - recipientUserId: deviceToUser.get(env.recipientDeviceId)!, - ciphertext: env.ciphertext, - })); + if (validEnvelopes.length > 0) { + await tx.insert(messageEnvelopes).values(validEnvelopes); + recipientDeviceIds = validEnvelopes.map((e) => e.recipientDeviceId); + } + } - if (validEnvelopes.length > 0) { - await db.insert(messageEnvelopes).values(validEnvelopes); - recipientDeviceIds = validEnvelopes.map((e) => e.recipientDeviceId); - } + return insertedMessage; + }); + } catch (error) { + console.error('Transaction failed for message edit:', error); + socket.emit('error', { event: 'edit_message', message: 'Failed to persist message edit' }); + return; } if (message) { @@ -495,18 +516,30 @@ export function registerMessagingHandlers(io: Server, socket: AuthSocket): void return; } - const [message] = await db - .insert(messages) - .values({ - conversationId, - senderId: userId, - content: content.trim(), - contentType, - fileId, - }) - .returning(); + let message; + try { + message = await db.transaction(async (tx) => { + const [insertedMessage] = await tx + .insert(messages) + .values({ + conversationId, + senderId: userId, + ciphertext: content.trim(), + contentType, + fileId, + }) + .returning(); + + return insertedMessage; + }); + } catch (error) { + console.error('Transaction failed for file message:', error); + socket.emit('error', { event: 'send_file_message', message: 'Failed to persist file message' }); + return; + } - io.to(conversationId).emit('new_message', message); + if (message) { + io.to(conversationId).emit('new_message', message); // Emit a file_message event for file-type content so recipients // know to fetch file bytes via GET /files/:id over HTTP. diff --git a/worklogs/progress/174.md b/worklogs/progress/174.md new file mode 100644 index 0000000..0413de6 --- /dev/null +++ b/worklogs/progress/174.md @@ -0,0 +1,79 @@ +# Progress Log for Issue #174: Remove server-side full-text search; return 410 Gone with migration note + +## Overview +Task: Drop the GIN to_tsvector index on messages.content in migration. Make GET /conversations/:id/search return 410 Gone with error message and documentation link. + +## Current Status: Planning Phase + +## Planned Implementation Steps + +### 1. Analysis Phase +- [ ] Identify the GIN to_tsvector index on messages.content +- [ ] Locate the search endpoint at `GET /conversations/:id/search` +- [ ] Review related cache tests that need updating +- [ ] Document current search functionality usage + +### 2. Database Migration +- [ ] Create migration to drop GIN to_tsvector index on messages.content +- [ ] Verify no other dependencies on this index +- [ ] Test migration rollback capability +- [ ] Document migration in changelog + +### 3. API Endpoint Update +- [ ] Update `GET /conversations/:id/search` endpoint +- [ ] Return HTTP status 410 Gone +- [ ] Include helpful error message in response body +- [ ] Provide documentation link for client-side search migration +- [ ] Update API documentation + +### 4. Testing Updates +- [ ] Update existing search-related tests +- [ ] Create tests for 410 Gone response +- [ ] Update cache tests to expect 410 response +- [ ] Verify all test suites pass + +### 5. Documentation +- [ ] Create migration guide for client-side search +- [ ] Document the rationale for removing server-side search +- [ ] Provide examples of client-side search implementation +- [ ] Update any user-facing documentation + +## Files to Modify +1. Database migration file (new) +2. Search endpoint handler (to be identified) +3. Test files for search functionality +4. API documentation + +## Acceptance Criteria Verification +- [ ] GIN index successfully dropped in migration +- [ ] Search endpoint returns 410 Gone status +- [ ] Response includes clear guidance message +- [ ] Related cache tests updated to expect 410 +- [ ] All tests pass with new behavior + +## Technical Considerations +- Need to communicate breaking change clearly +- Provide migration path for clients +- Ensure backward compatibility not expected (410 is permanent) +- Consider impact on existing client applications + +## Migration Response Structure +```json +{ + "error": "Server-side search removed; search is now client-side over decrypted messages", + "docs": "https://docs.example.com/client-side-search-migration" +} +``` + +## Timeline +- Start: Planning phase +- Migration creation: 1 day +- Endpoint updates: 1 day +- Testing updates: 1 day +- Documentation: 0.5 day + +## Notes +- This is a breaking change requiring client updates +- Search functionality moves to client-side for security/privacy +- Decrypted message search must be handled by clients +- 410 status indicates permanent removal of the endpoint \ No newline at end of file diff --git a/worklogs/progress/180.md b/worklogs/progress/180.md new file mode 100644 index 0000000..5a6b22f --- /dev/null +++ b/worklogs/progress/180.md @@ -0,0 +1,63 @@ +# Progress Log for Issue #180: Migrate message insert paths to ciphertext + envelopes + +## Overview +Task: Update every message insert path (socket/messaging.ts, any REST sender) to validate by content_type, assign sequenceNumber, persist the message row, persist per-recipient-device envelopes or single group ciphertext, all in one transaction, then fan-out. + +## Current Status: Planning Phase + +## Planned Implementation Steps + +### 1. Analysis Phase +- [ ] Review current message insert paths in `socket/messaging.ts` +- [ ] Identify all REST sender endpoints that handle message creation +- [ ] Examine existing database schema for message and envelope tables +- [ ] Understand current transaction patterns + +### 2. Database Transaction Design +- [ ] Design transaction flow for message + envelope persistence +- [ ] Ensure atomic commit of message row and per-recipient-device envelopes +- [ ] Verify group ciphertext handling for group messages + +### 3. Code Migration +- [ ] Update `socket/messaging.ts` message insertion +- [ ] Update REST API endpoints for message sending +- [ ] Implement content_type validation +- [ ] Add sequenceNumber assignment logic +- [ ] Implement transactional persistence +- [ ] Ensure no plaintext storage + +### 4. Testing Strategy +- [ ] Update existing send tests to use ciphertext model +- [ ] Create new tests for transactional behavior +- [ ] Test edge cases: partial failures, rollbacks +- [ ] Verify fan-out only occurs after successful commit + +### 5. Acceptance Criteria Verification +- [ ] Confirm all send paths are transactional +- [ ] Verify message + envelopes committed together before delivery +- [ ] Ensure no code path writes plaintext +- [ ] Validate existing tests updated to ciphertext model + +## Files to Modify +1. `apps/backend/src/socket/messaging.ts` - Primary socket message handling +2. REST message sending endpoints (to be identified) +3. Database transaction utilities +4. Test files for message sending + +## Technical Considerations +- Need to maintain backward compatibility during migration +- Ensure proper error handling for transaction failures +- Consider performance impact of envelope generation +- Validate encryption/decryption flows + +## Timeline +- Start: Planning phase +- Implementation: 2-3 days +- Testing: 1-2 days +- Review: 1 day + +## Notes +- This implementation focuses on eliminating plaintext storage +- All messages must be encrypted before persistence +- Envelopes must be created per recipient device +- Transaction integrity is critical for data consistency \ No newline at end of file