diff --git a/apps/webapp/app/bullmq/connection.ts b/apps/webapp/app/bullmq/connection.ts index f9dc98e27..729d152c0 100644 --- a/apps/webapp/app/bullmq/connection.ts +++ b/apps/webapp/app/bullmq/connection.ts @@ -17,7 +17,7 @@ export function getRedisConnection() { const redisConfig: RedisOptions = { host: process.env.REDIS_HOST, - port: parseInt(process.env.REDIS_PORT as string), + port: parseInt(process.env.REDIS_PORT || '6379', 10), password: process.env.REDIS_PASSWORD, maxRetriesPerRequest: null, // Required for BullMQ enableReadyCheck: false, // Required for BullMQ diff --git a/integrations/gmail/package.json b/integrations/gmail/package.json index bdb80f8a6..1930571b9 100644 --- a/integrations/gmail/package.json +++ b/integrations/gmail/package.json @@ -23,6 +23,7 @@ "build:cli": "rimraf bin && bun build src/index.ts --outfile bin/index.js --target node --minify", "build:frontend": "tsup", "dev:frontend": "tsup --watch", + "test": "bun test", "lint": "eslint --ext js,ts,tsx src/ --fix", "prettier": "prettier --config .prettierrc --write ." }, @@ -31,7 +32,6 @@ "react": "^19.2.4", "tsup": "^8.0.1", "@types/nodemailer": "^6.4.17", - "@types/turndown": "^5.0.5", "@babel/preset-typescript": "^7.26.0", "@rollup/plugin-commonjs": "^28.0.1", "@rollup/plugin-json": "^6.1.0", @@ -81,7 +81,6 @@ "openai": "^4.0.0", "react-query": "^3.39.3", "@redplanethq/sdk": "0.1.18", - "turndown": "^7.2.0", "@redplanethq/ui": "2.1.4" } } diff --git a/integrations/gmail/src/__tests__/schedule.test.ts b/integrations/gmail/src/__tests__/schedule.test.ts new file mode 100644 index 000000000..f96f7730c --- /dev/null +++ b/integrations/gmail/src/__tests__/schedule.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, test } from 'bun:test'; + +import { + formatMetadataText, + getDefaultSyncTime, + getHeader, + GmailEmailMetadata, + toGmailTimestamp, +} from '../schedule-utils'; + +describe('toGmailTimestamp', () => { + test('converts ISO date to Unix timestamp in seconds', () => { + const iso = '2024-01-15T10:00:00.000Z'; + const expected = Math.floor(new Date(iso).getTime() / 1000); + expect(toGmailTimestamp(iso)).toBe(expected); + }); + + test('returns an integer (floor)', () => { + const ts = toGmailTimestamp('2024-06-01T00:00:00.500Z'); + expect(Number.isInteger(ts)).toBe(true); + }); +}); + +describe('getDefaultSyncTime', () => { + test('returns an ISO string approximately 24 hours ago', () => { + const before = Date.now(); + const result = getDefaultSyncTime(); + + const resultMs = new Date(result).getTime(); + const expectedMs = before - 24 * 60 * 60 * 1000; + + // Within 100 ms of expected + expect(Math.abs(resultMs - expectedMs)).toBeLessThan(100); + }); + + test('returns a valid ISO string', () => { + const result = getDefaultSyncTime(); + expect(new Date(result).toISOString()).toBe(result); + }); +}); + +describe('getHeader', () => { + const headers = [ + { name: 'From', value: 'Alice ' }, + { name: 'Subject', value: 'Hello' }, + { name: 'To', value: 'Bob ' }, + ]; + + test('finds a header by exact name', () => { + expect(getHeader(headers, 'From')).toBe('Alice '); + }); + + test('is case-insensitive', () => { + expect(getHeader(headers, 'subject')).toBe('Hello'); + expect(getHeader(headers, 'SUBJECT')).toBe('Hello'); + }); + + test('returns empty string when header not found', () => { + expect(getHeader(headers, 'Cc')).toBe(''); + }); +}); + +describe('formatMetadataText', () => { + const baseMeta: GmailEmailMetadata = { + id: 'msg123', + threadId: 'thread456', + subject: 'Hello World', + from: 'Alice ', + to: 'Bob ', + date: 'Mon, 15 Jan 2024 10:00:00 +0000', + internalDate: 1705312800000, + snippet: 'This is a short preview...', + labelIds: ['INBOX', 'IMPORTANT'], + sizeEstimate: 4096, + webLink: 'https://mail.google.com/mail/u/0/#inbox/msg123', + }; + + test('received email uses envelope icon and "Email from" heading', () => { + const text = formatMetadataText(baseMeta, 'received'); + expect(text).toContain('📧'); + expect(text).toContain('Email from Alice '); + }); + + test('sent email uses outbox icon and "Email to" heading', () => { + const text = formatMetadataText(baseMeta, 'sent'); + expect(text).toContain('📤'); + expect(text).toContain('Email to Bob '); + }); + + test('includes all required metadata fields', () => { + const text = formatMetadataText(baseMeta, 'received'); + expect(text).toContain('**ID:** msg123'); + expect(text).toContain('**Thread ID:** thread456'); + expect(text).toContain('**Subject:** Hello World'); + expect(text).toContain('**From:** Alice '); + expect(text).toContain('**To:** Bob '); + expect(text).toContain('**Date:** Mon, 15 Jan 2024 10:00:00 +0000'); + expect(text).toContain('**Internal Date:** 1705312800000'); + expect(text).toContain('**Snippet:** This is a short preview...'); + expect(text).toContain('**Labels:** INBOX, IMPORTANT'); + expect(text).toContain('**Size:** 4096 bytes'); + expect(text).toContain('**Link:** https://mail.google.com/mail/u/0/#inbox/msg123'); + }); + + test('includes historyId when present', () => { + const meta = { ...baseMeta, historyId: 'hist789' }; + const text = formatMetadataText(meta, 'received'); + expect(text).toContain('**History ID:** hist789'); + }); + + test('omits historyId line when not present', () => { + const text = formatMetadataText(baseMeta, 'received'); + expect(text).not.toContain('History ID'); + }); + + test('includes Cc when present', () => { + const meta = { ...baseMeta, cc: 'Charlie ' }; + const text = formatMetadataText(meta, 'received'); + expect(text).toContain('**Cc:** Charlie '); + }); + + test('omits Cc line when not present', () => { + const text = formatMetadataText(baseMeta, 'received'); + expect(text).not.toContain('**Cc:**'); + }); + + test('includes Bcc when present', () => { + const meta = { ...baseMeta, bcc: 'Dan ' }; + const text = formatMetadataText(meta, 'received'); + expect(text).toContain('**Bcc:** Dan '); + }); + + test('does not contain HTML body tags', () => { + const text = formatMetadataText(baseMeta, 'received'); + // Email addresses use angle brackets which are fine; actual HTML elements should not appear + expect(text).not.toMatch(/<(html|body|div|p|span|br|img|a\s)[^>]*>/i); + }); +}); diff --git a/integrations/gmail/src/schedule-utils.ts b/integrations/gmail/src/schedule-utils.ts new file mode 100644 index 000000000..4ff3bd66a --- /dev/null +++ b/integrations/gmail/src/schedule-utils.ts @@ -0,0 +1,83 @@ +export interface GmailEmailMetadata { + id: string; + threadId: string; + historyId?: string; + subject: string; + from: string; + to: string; + cc?: string; + bcc?: string; + date: string; + internalDate: number; + snippet: string; + labelIds: string[]; + sizeEstimate: number; + webLink: string; +} + +/** + * Gets default sync time (24 hours ago). + */ +export function getDefaultSyncTime(): string { + return new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); +} + +/** + * Convert ISO date to Gmail query format as Unix timestamp in seconds. + * Using timestamp is more precise than YYYY/MM/DD which truncates to day. + */ +export function toGmailTimestamp(isoDate: string): number { + return Math.floor(new Date(isoDate).getTime() / 1000); +} + +/** + * Format email metadata as structured Markdown text (no body content). + */ +export function formatMetadataText( + meta: GmailEmailMetadata, + direction: 'received' | 'sent' +): string { + const icon = direction === 'received' ? '📧' : '📤'; + const label = direction === 'received' ? `Email from ${meta.from}` : `Email to ${meta.to}`; + + const lines = [ + `## ${icon} ${label}`, + '', + `**ID:** ${meta.id}`, + `**Thread ID:** ${meta.threadId}`, + ]; + + if (meta.historyId) { + lines.push(`**History ID:** ${meta.historyId}`); + } + + lines.push( + `**Subject:** ${meta.subject}`, + `**From:** ${meta.from}`, + `**To:** ${meta.to}` + ); + + if (meta.cc) lines.push(`**Cc:** ${meta.cc}`); + if (meta.bcc) lines.push(`**Bcc:** ${meta.bcc}`); + + lines.push( + `**Date:** ${meta.date}`, + `**Internal Date:** ${meta.internalDate}`, + `**Snippet:** ${meta.snippet}`, + `**Labels:** ${meta.labelIds.join(', ')}`, + `**Size:** ${meta.sizeEstimate} bytes`, + `**Link:** ${meta.webLink}` + ); + + return lines.join('\n'); +} + +/** + * Extract a header value from a Gmail message headers array. + */ +export function getHeader( + headers: Array<{ name: string; value: string }>, + name: string +): string { + return headers.find((h) => h.name.toLowerCase() === name.toLowerCase())?.value ?? ''; +} diff --git a/integrations/gmail/src/schedule.ts b/integrations/gmail/src/schedule.ts index 8bd99e180..7f3e3502f 100644 --- a/integrations/gmail/src/schedule.ts +++ b/integrations/gmail/src/schedule.ts @@ -1,5 +1,13 @@ -import { getGmailClient, parseEmailContent, formatEmailSender, GmailConfig } from './utils'; -import TurndownService from 'turndown'; +import { getGmailClient, GmailConfig } from './utils'; +import { + formatMetadataText, + getDefaultSyncTime, + getHeader, + GmailEmailMetadata, + toGmailTimestamp, +} from './schedule-utils'; + +export { formatMetadataText, getDefaultSyncTime, GmailEmailMetadata, toGmailTimestamp }; interface GmailSettings { lastSyncTime?: string; @@ -13,7 +21,7 @@ interface GmailActivityCreateParams { } /** - * Creates an activity message based on Gmail data + * Creates an activity message based on Gmail metadata. */ function createActivityMessage(params: GmailActivityCreateParams) { return { @@ -26,63 +34,17 @@ function createActivityMessage(params: GmailActivityCreateParams) { } /** - * Gets default sync time (24 hours ago) - */ -function getDefaultSyncTime(): string { - return new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); -} - -/** - * Initialize Turndown service for HTML to Markdown conversion - */ -const turndownService = new TurndownService({ - headingStyle: 'atx', - codeBlockStyle: 'fenced', - emDelimiter: '*', -}); - -// Remove style, script, and other unwanted elements -turndownService.remove(['style', 'script', 'noscript', 'iframe', 'object', 'embed']); - -/** - * Clean and convert email content to markdown - */ -function cleanEmailContent(htmlContent: string, textContent: string): string { - // If we have HTML content, convert it to markdown - if (htmlContent) { - const markdown = turndownService.turndown(htmlContent); - return markdown - .replace(/\n\n+/g, '\n\n') // Remove excessive line breaks - .replace(/\s+/g, ' ') // Normalize spaces - .trim(); - } - - // Otherwise use text content and clean it - return textContent.replace(/\r/g, '').replace(/\n\n+/g, '\n\n').replace(/\s+/g, ' ').trim(); -} - -/** - * Convert ISO date to Gmail query format as Unix timestamp in seconds - * Using timestamp is more precise than YYYY/MM/DD which truncates to day - */ -function toGmailTimestamp(isoDate: string): number { - return Math.floor(new Date(isoDate).getTime() / 1000); -} - -/** - * Fetch and process received emails + * Fetch and process received emails — returns metadata-only activities. */ async function processReceivedEmails( gmail: any, - lastSyncTime: string, - emailAddress: string + lastSyncTime: string ): Promise<{ activities: any[]; lastEmailTime: number }> { const activities = []; const afterTimestamp = toGmailTimestamp(lastSyncTime); let lastEmailTime = 0; try { - // Query for important received emails after lastSyncTime const response = await gmail.users.messages.list({ userId: 'me', q: `in:inbox is:important after:${afterTimestamp}`, @@ -93,21 +55,19 @@ async function processReceivedEmails( for (const message of messages) { try { - // Get full message details const fullMessage = await gmail.users.messages.get({ userId: 'me', id: message.id, - format: 'full', + format: 'metadata', + metadataHeaders: ['From', 'To', 'Cc', 'Bcc', 'Subject', 'Date'], }); - const headers = fullMessage.data.payload.headers; - const from = headers.find((h: any) => h.name === 'From')?.value || 'Unknown'; - const subject = headers.find((h: any) => h.name === 'Subject')?.value || '(No subject)'; - const date = headers.find((h: any) => h.name === 'Date')?.value || ''; + const data = fullMessage.data; + const headers: Array<{ name: string; value: string }> = data.payload?.headers ?? []; - const internalDate = parseInt(fullMessage.data.internalDate || '0'); + const internalDate = parseInt(data.internalDate || '0', 10); - // Skip emails that are at or before lastSyncTime (Gmail after: is not precise at second level) + // Skip emails at or before lastSyncTime (Gmail after: is not precise at second level) const lastSyncMs = new Date(lastSyncTime).getTime(); if (internalDate <= lastSyncMs) { continue; @@ -117,39 +77,33 @@ async function processReceivedEmails( lastEmailTime = internalDate; } - const sender = formatEmailSender(from); - const threadId = fullMessage.data.threadId || ''; - const { textContent, htmlContent } = parseEmailContent(fullMessage.data.payload); - - // Clean and convert email content to markdown - const cleanedContent = cleanEmailContent(htmlContent, textContent); - - // Skip if no meaningful content - if (!cleanedContent || cleanedContent.length < 10) { - continue; - } - - // Create Gmail web URL - const sourceURL = `https://mail.google.com/mail/u/0/#inbox/${message.id}`; - - // Format activity text with full email content as markdown - const text = `## 📧 Email from ${sender} - -**From:** ${from} -**Subject:** ${subject} -**Date:** ${date} -**Thread ID:** ${threadId} - -${cleanedContent}`; + const cc = getHeader(headers, 'Cc') || undefined; + const bcc = getHeader(headers, 'Bcc') || undefined; + + const meta: GmailEmailMetadata = { + id: data.id ?? message.id, + threadId: data.threadId ?? '', + ...(data.historyId ? { historyId: data.historyId } : {}), + subject: getHeader(headers, 'Subject') || '(No subject)', + from: getHeader(headers, 'From') || 'Unknown', + to: getHeader(headers, 'To') || '', + ...(cc ? { cc } : {}), + ...(bcc ? { bcc } : {}), + date: getHeader(headers, 'Date'), + internalDate, + snippet: data.snippet ?? '', + labelIds: data.labelIds ?? [], + sizeEstimate: data.sizeEstimate ?? 0, + webLink: `https://mail.google.com/mail/u/0/#inbox/${data.id ?? message.id}`, + }; activities.push( createActivityMessage({ - text, - sourceURL, + text: formatMetadataText(meta, 'received'), + sourceURL: meta.webLink, }) ); } catch (error) { - // Silently ignore errors for individual messages console.error('Error processing received email:', error); } } @@ -161,19 +115,17 @@ ${cleanedContent}`; } /** - * Fetch and process sent emails + * Fetch and process sent emails — returns metadata-only activities. */ async function processSentEmails( gmail: any, - lastSyncTime: string, - emailAddress: string + lastSyncTime: string ): Promise<{ activities: any[]; lastEmailTime: number }> { const activities = []; const afterTimestamp = toGmailTimestamp(lastSyncTime); let lastEmailTime = 0; try { - // Query for sent emails after lastSyncTime const response = await gmail.users.messages.list({ userId: 'me', q: `in:sent after:${afterTimestamp}`, @@ -184,21 +136,19 @@ async function processSentEmails( for (const message of messages) { try { - // Get full message details const fullMessage = await gmail.users.messages.get({ userId: 'me', id: message.id, - format: 'full', + format: 'metadata', + metadataHeaders: ['From', 'To', 'Cc', 'Bcc', 'Subject', 'Date'], }); - const headers = fullMessage.data.payload.headers; - const to = headers.find((h: any) => h.name === 'To')?.value || 'Unknown'; - const subject = headers.find((h: any) => h.name === 'Subject')?.value || '(No subject)'; - const date = headers.find((h: any) => h.name === 'Date')?.value || ''; + const data = fullMessage.data; + const headers: Array<{ name: string; value: string }> = data.payload?.headers ?? []; - const internalDate = parseInt(fullMessage.data.internalDate || '0'); + const internalDate = parseInt(data.internalDate || '0', 10); - // Skip emails that are at or before lastSyncTime + // Skip emails at or before lastSyncTime const lastSyncMs = new Date(lastSyncTime).getTime(); if (internalDate <= lastSyncMs) { continue; @@ -208,39 +158,33 @@ async function processSentEmails( lastEmailTime = internalDate; } - const threadId = fullMessage.data.threadId || message.id; - const { textContent, htmlContent } = parseEmailContent(fullMessage.data.payload); - - // Clean and convert email content to markdown - const cleanedContent = cleanEmailContent(htmlContent, textContent); - - // Skip if no meaningful content - if (!cleanedContent || cleanedContent.length < 10) { - continue; - } - - // Create Gmail web URL - const sourceURL = `https://mail.google.com/mail/u/0/#sent/${message.id}`; - - // Format activity text with full email content as markdown - const text = `## 📤 Sent to ${to} - -**From:** ${emailAddress} -**To:** ${to} -**Subject:** ${subject} -**Date:** ${date} -**Thread ID:** ${threadId} - -${cleanedContent}`; + const cc = getHeader(headers, 'Cc') || undefined; + const bcc = getHeader(headers, 'Bcc') || undefined; + + const meta: GmailEmailMetadata = { + id: data.id ?? message.id, + threadId: data.threadId ?? message.id, + ...(data.historyId ? { historyId: data.historyId } : {}), + subject: getHeader(headers, 'Subject') || '(No subject)', + from: getHeader(headers, 'From') || 'Unknown', + to: getHeader(headers, 'To') || '', + ...(cc ? { cc } : {}), + ...(bcc ? { bcc } : {}), + date: getHeader(headers, 'Date'), + internalDate, + snippet: data.snippet ?? '', + labelIds: data.labelIds ?? [], + sizeEstimate: data.sizeEstimate ?? 0, + webLink: `https://mail.google.com/mail/u/0/#sent/${data.id ?? message.id}`, + }; activities.push( createActivityMessage({ - text, - sourceURL, + text: formatMetadataText(meta, 'sent'), + sourceURL: meta.webLink, }) ); } catch (error) { - // Silently ignore errors for individual messages console.error('Error processing sent email:', error); } } @@ -257,18 +201,14 @@ export const handleSchedule = async ( state?: Record ) => { try { - // Check if we have a valid access token if (!config?.access_token) { return []; } - // Get settings or initialize if not present let settings = (state || {}) as GmailSettings; - // Default to 24 hours ago if no last sync time const lastSyncTime = settings.lastSyncTime || getDefaultSyncTime(); - // Create Gmail client const gmailConfig: GmailConfig = { access_token: config.access_token, refresh_token: config.refresh_token || '', @@ -278,7 +218,6 @@ export const handleSchedule = async ( const gmail = await getGmailClient(gmailConfig); - // Get user profile to get email address if (!settings.emailAddress) { try { const profile = await gmail.users.getProfile({ userId: 'me' }); @@ -288,23 +227,18 @@ export const handleSchedule = async ( } } - // Collect all messages const messages = []; - // Process received emails const { activities: receivedActivities, lastEmailTime: receivedLastTime } = - await processReceivedEmails(gmail, lastSyncTime, settings.emailAddress || 'user'); + await processReceivedEmails(gmail, lastSyncTime); messages.push(...receivedActivities); - // Process sent emails const { activities: sentActivities, lastEmailTime: sentLastTime } = await processSentEmails( gmail, - lastSyncTime, - settings.emailAddress || 'user' + lastSyncTime ); messages.push(...sentActivities); - // Only save state if emails were processed const latestEmailTime = Math.max(receivedLastTime, sentLastTime); if (latestEmailTime > 0) { const newSyncTime = new Date(latestEmailTime + 20000).toISOString();