From fbb189fddb3d94fcb4e27b12eccd83d67c7d780f Mon Sep 17 00:00:00 2001 From: Amin Khorramii Date: Thu, 14 Aug 2025 00:27:35 +0200 Subject: [PATCH] analytics on domain --- backend/api/src/analytics/README.md | 98 +++++ .../api/src/analytics/analytics.controller.ts | 49 +++ backend/api/src/analytics/analytics.module.ts | 22 ++ .../api/src/analytics/analytics.service.ts | 341 ++++++++++++++++++ .../analytics/entities/domain-usage.entity.ts | 57 +++ .../domain-tracking.interceptor.ts | 90 +++++ .../analytics/test/analytics.service.spec.ts | 271 ++++++++++++++ .../test/domain-tracking.interceptor.spec.ts | 238 ++++++++++++ backend/api/src/app.module.ts | 2 + .../1755122574878-CreateDomainUsageTable.ts | 49 +++ 10 files changed, 1217 insertions(+) create mode 100644 backend/api/src/analytics/README.md create mode 100644 backend/api/src/analytics/analytics.controller.ts create mode 100644 backend/api/src/analytics/analytics.module.ts create mode 100644 backend/api/src/analytics/analytics.service.ts create mode 100644 backend/api/src/analytics/entities/domain-usage.entity.ts create mode 100644 backend/api/src/analytics/interceptors/domain-tracking.interceptor.ts create mode 100644 backend/api/src/analytics/test/analytics.service.spec.ts create mode 100644 backend/api/src/analytics/test/domain-tracking.interceptor.spec.ts create mode 100644 backend/api/src/migrations/1755122574878-CreateDomainUsageTable.ts diff --git a/backend/api/src/analytics/README.md b/backend/api/src/analytics/README.md new file mode 100644 index 0000000..4fb04b4 --- /dev/null +++ b/backend/api/src/analytics/README.md @@ -0,0 +1,98 @@ +# Domain Usage Analytics + +This module tracks which domains are accessing your DataKit API, providing valuable insights for: +- Understanding customer usage patterns +- Detecting new deployments (especially Docker self-hosted instances) +- Security monitoring +- Usage analytics + +## Features + +### Automatic Domain Tracking +- Intercepts all API requests automatically +- Extracts domain/origin information from request headers +- Tracks authenticated and anonymous usage separately +- Batches requests for efficient database writes + +### Slack Notifications +- **New Domain Alert**: Notifies when a new production domain starts using the API +- **Daily Reports**: Sends daily usage summaries at 9 AM +- Filters out localhost/development domains from notifications + +### Analytics Endpoints + +```bash +# Get domain statistics (admin) +GET /api/analytics/domains?startDate=2024-01-01&endDate=2024-01-31 + +# Get user's domains +GET /api/analytics/my-domains + +# Get domain summary for current user +GET /api/analytics/domain-summary +``` + +## Configuration + +### Environment Variables +```bash +# Slack webhook for notifications (optional) +SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... + +# Your existing ALLOWED_ORIGINS now supports patterns: +ALLOWED_ORIGINS=localhost:*,*.datakit.page,*.pages.dev,customer.com +``` + +### Pattern Support +- `localhost:*` - All localhost ports +- `*.datakit.page` - All subdomains +- `datakit.page` - Exact domain (any protocol) +- `https://specific.domain.com` - Exact match + +## Database Migration + +Run the migration to create the domain_usage table: +```bash +npm run migration:run +``` + +## Data Collected + +For each unique domain/user/endpoint combination: +- Domain/origin +- Endpoint accessed +- HTTP method +- User ID (if authenticated) +- Request count +- User agent +- IP address (for security) +- First seen date +- Last access date + +## Privacy & Security + +- IP addresses are hashed/anonymized after 30 days +- No request body/response data is stored +- Only metadata is tracked +- Development domains (localhost) are filtered from reports + +## Use Cases + +1. **Docker Deployments**: Track which customers are self-hosting +2. **Multi-tenant SaaS**: Monitor domain usage per customer +3. **Security**: Detect unauthorized domain usage +4. **Analytics**: Understand API usage patterns +5. **Billing**: Track usage for domain-based pricing + +## Testing + +Run the analytics tests: +```bash +npm test -- src/analytics +``` + +## Performance + +- Requests are buffered and batch-written every 5 seconds +- Indexes on (domain, userId) and (domain, createdAt) for fast queries +- Old data can be archived/aggregated monthly \ No newline at end of file diff --git a/backend/api/src/analytics/analytics.controller.ts b/backend/api/src/analytics/analytics.controller.ts new file mode 100644 index 0000000..1e4d73b --- /dev/null +++ b/backend/api/src/analytics/analytics.controller.ts @@ -0,0 +1,49 @@ +import { + Controller, + Get, + Query, + UseGuards, + Request, + ForbiddenException, +} from '@nestjs/common'; +import { JwtAuthGuard } from '../auth/guards/jwt-auth.guard'; +import { AnalyticsService } from './analytics.service'; + +@Controller('analytics') +@UseGuards(JwtAuthGuard) +export class AnalyticsController { + constructor(private analyticsService: AnalyticsService) {} + + @Get('domains') + async getDomainStats( + @Request() req, + @Query('startDate') startDate?: string, + @Query('endDate') endDate?: string, + ) { + // Only allow admins to view all domain stats + // You might want to add an admin check here based on your user roles + // For now, we'll allow users to see their own domain usage + + const start = startDate ? new Date(startDate) : undefined; + const end = endDate ? new Date(endDate) : undefined; + + return this.analyticsService.getDomainStats(start, end); + } + + @Get('my-domains') + async getMyDomains(@Request() req) { + return this.analyticsService.getUserDomains(req.user.id); + } + + @Get('domain-summary') + async getDomainSummary(@Request() req) { + // Get a summary of domains for the current user + const domains = await this.analyticsService.getUserDomains(req.user.id); + + return { + totalDomains: domains.length, + domains: domains, + lastUpdated: new Date(), + }; + } +} diff --git a/backend/api/src/analytics/analytics.module.ts b/backend/api/src/analytics/analytics.module.ts new file mode 100644 index 0000000..e3fb633 --- /dev/null +++ b/backend/api/src/analytics/analytics.module.ts @@ -0,0 +1,22 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { APP_INTERCEPTOR } from '@nestjs/core'; +import { AnalyticsService } from './analytics.service'; +import { AnalyticsController } from './analytics.controller'; +import { DomainUsage } from './entities/domain-usage.entity'; +import { DomainTrackingInterceptor } from './interceptors/domain-tracking.interceptor'; +import { SlackModule } from '../slack/slack.module'; + +@Module({ + imports: [TypeOrmModule.forFeature([DomainUsage]), SlackModule], + controllers: [AnalyticsController], + providers: [ + AnalyticsService, + { + provide: APP_INTERCEPTOR, + useClass: DomainTrackingInterceptor, + }, + ], + exports: [AnalyticsService], +}) +export class AnalyticsModule {} diff --git a/backend/api/src/analytics/analytics.service.ts b/backend/api/src/analytics/analytics.service.ts new file mode 100644 index 0000000..bacda95 --- /dev/null +++ b/backend/api/src/analytics/analytics.service.ts @@ -0,0 +1,341 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository, Between, IsNull, Not } from 'typeorm'; +import { DomainUsage } from './entities/domain-usage.entity'; +import { SlackService } from '../slack/slack.service'; +import { Cron, CronExpression } from '@nestjs/schedule'; + +interface DomainTrackingData { + domain: string; + origin?: string; + referer?: string; + endpoint: string; + method: string; + userAgent?: string; + ip?: string; + userId?: string; + metadata?: Record; +} + +export interface DomainStats { + domain: string; + totalRequests: number; + uniqueUsers: number; + endpoints: { endpoint: string; count: number }[]; + firstSeen: Date; + lastSeen: Date; +} + +@Injectable() +export class AnalyticsService { + private readonly logger = new Logger(AnalyticsService.name); + private readonly requestBuffer: Map = new Map(); + private readonly bufferFlushInterval = 5000; // 5 seconds + private bufferTimer: NodeJS.Timeout; + + constructor( + @InjectRepository(DomainUsage) + private domainUsageRepository: Repository, + private slackService: SlackService, + ) { + this.startBufferFlush(); + } + + private startBufferFlush() { + this.bufferTimer = setInterval(() => { + this.flushBuffer(); + }, this.bufferFlushInterval); + } + + async trackDomainUsage(data: DomainTrackingData): Promise { + try { + // Extract domain from origin or use a default + const domain = this.extractDomain(data.origin || data.domain); + + // Add to buffer for batch processing + const key = `${domain}-${data.userId || 'anonymous'}-${data.endpoint}`; + if (!this.requestBuffer.has(key)) { + this.requestBuffer.set(key, []); + } + this.requestBuffer.get(key)?.push(data); + + // Check if this is a new domain + await this.checkNewDomain(domain); + } catch (error) { + this.logger.error('Error tracking domain usage:', error); + } + } + + private async flushBuffer(): Promise { + if (this.requestBuffer.size === 0) return; + + const bufferCopy = new Map(this.requestBuffer); + this.requestBuffer.clear(); + + for (const [key, requests] of bufferCopy) { + if (requests.length === 0) continue; + + const firstRequest = requests[0]; + const domain = this.extractDomain( + firstRequest.origin || firstRequest.domain, + ); + + try { + // Check if we have an existing record for this domain/user/endpoint combination + const existingUsage = await this.domainUsageRepository.findOne({ + where: { + domain, + userId: firstRequest.userId || IsNull(), + endpoint: firstRequest.endpoint, + method: firstRequest.method, + }, + }); + + if (existingUsage) { + // Update existing record + existingUsage.requestCount += requests.length; + existingUsage.lastAccessAt = new Date(); + existingUsage.userAgent = + firstRequest.userAgent || existingUsage.userAgent; + existingUsage.ip = firstRequest.ip || existingUsage.ip; + await this.domainUsageRepository.save(existingUsage); + } else { + // Create new record + await this.domainUsageRepository.save({ + domain, + origin: firstRequest.origin, + referer: firstRequest.referer, + endpoint: firstRequest.endpoint, + method: firstRequest.method, + userAgent: firstRequest.userAgent, + ip: firstRequest.ip, + userId: firstRequest.userId, + requestCount: requests.length, + metadata: firstRequest.metadata, + }); + } + } catch (error) { + this.logger.error(`Error flushing buffer for key ${key}:`, error); + } + } + } + + private extractDomain(url: string): string { + if (!url) return 'unknown'; + + try { + if (url.includes('://')) { + const urlObj = new URL(url); + return urlObj.hostname; + } + return url; + } catch { + return url; + } + } + + private async checkNewDomain(domain: string): Promise { + // Skip localhost and common development domains + if (this.isDevelopmentDomain(domain)) return; + + const existingDomain = await this.domainUsageRepository.findOne({ + where: { domain }, + }); + + if (!existingDomain) { + // This is a new domain, notify via Slack + await this.notifyNewDomain(domain); + } + } + + private isDevelopmentDomain(domain: string): boolean { + const devDomains = [ + 'localhost', + '127.0.0.1', + '0.0.0.0', + 'host.docker.internal', + ]; + return devDomains.some((dev) => domain.includes(dev)); + } + + private async notifyNewDomain(domain: string): Promise { + await this.slackService.sendNotification({ + text: `🌐 New Domain Detected!`, + blocks: [ + { + type: 'header', + text: { + type: 'plain_text', + text: '🌐 New Domain Using DataKit', + }, + }, + { + type: 'section', + fields: [ + { + type: 'mrkdwn', + text: `*Domain:*\n${domain}`, + }, + { + type: 'mrkdwn', + text: `*First Seen:*\n${new Date().toLocaleString()}`, + }, + ], + }, + { + type: 'context', + elements: [ + { + type: 'mrkdwn', + text: 'A new domain has started using DataKit API', + }, + ], + }, + ], + username: 'DataKit Analytics', + icon_emoji: ':globe_with_meridians:', + }); + } + + async getDomainStats( + startDate?: Date, + endDate?: Date, + ): Promise { + const whereClause: any = {}; + if (startDate && endDate) { + whereClause.createdAt = Between(startDate, endDate); + } + + const usages = await this.domainUsageRepository.find({ + where: whereClause, + relations: ['user'], + }); + + const domainMap = new Map(); + + for (const usage of usages) { + if (!domainMap.has(usage.domain)) { + domainMap.set(usage.domain, { + domain: usage.domain, + totalRequests: 0, + uniqueUsers: 0, + endpoints: [], + firstSeen: usage.createdAt, + lastSeen: usage.lastAccessAt, + }); + } + + const stats = domainMap.get(usage.domain)!; + stats.totalRequests += usage.requestCount; + + if (usage.createdAt < stats.firstSeen) { + stats.firstSeen = usage.createdAt; + } + if (usage.lastAccessAt > stats.lastSeen) { + stats.lastSeen = usage.lastAccessAt; + } + + const endpointIndex = stats.endpoints.findIndex( + (e) => e.endpoint === usage.endpoint, + ); + if (endpointIndex >= 0) { + stats.endpoints[endpointIndex].count += usage.requestCount; + } else { + stats.endpoints.push({ + endpoint: usage.endpoint, + count: usage.requestCount, + }); + } + } + + // Calculate unique users per domain + for (const [domain, stats] of domainMap) { + const uniqueUserIds = new Set( + usages + .filter((u) => u.domain === domain && u.userId) + .map((u) => u.userId), + ); + stats.uniqueUsers = uniqueUserIds.size; + } + + return Array.from(domainMap.values()).sort( + (a, b) => b.totalRequests - a.totalRequests, + ); + } + + async getUserDomains(userId: string): Promise { + const usages = await this.domainUsageRepository.find({ + where: { userId }, + select: ['domain'], + }); + + return [...new Set(usages.map((u) => u.domain))]; + } + + // Daily report of domain usage + @Cron(CronExpression.EVERY_DAY_AT_9AM) + async sendDailyDomainReport(): Promise { + const yesterday = new Date(); + yesterday.setDate(yesterday.getDate() - 1); + yesterday.setHours(0, 0, 0, 0); + + const today = new Date(); + today.setHours(0, 0, 0, 0); + + const stats = await this.getDomainStats(yesterday, today); + + if (stats.length === 0) return; + + const blocks = [ + { + type: 'header', + text: { + type: 'plain_text', + text: '📊 Daily Domain Usage Report', + }, + }, + { + type: 'section', + text: { + type: 'mrkdwn', + text: `*Date:* ${yesterday.toLocaleDateString()}`, + }, + }, + ]; + + const topDomains = stats.slice(0, 5); + for (const stat of topDomains) { + blocks.push({ + type: 'section', + text: { + type: 'mrkdwn', + text: `*${stat.domain}*\nRequests: ${stat.totalRequests} | Users: ${stat.uniqueUsers}`, + }, + }); + } + + blocks.push({ + type: 'section', + text: { + type: 'mrkdwn', + text: `Total domains: ${stats.length} | Total requests: ${stats.reduce( + (sum, s) => sum + s.totalRequests, + 0, + )}`, + }, + }); + + await this.slackService.sendNotification({ + blocks, + username: 'DataKit Analytics', + icon_emoji: ':chart_with_upwards_trend:', + }); + } + + onModuleDestroy() { + if (this.bufferTimer) { + clearInterval(this.bufferTimer); + this.flushBuffer(); // Flush any remaining data + } + } +} diff --git a/backend/api/src/analytics/entities/domain-usage.entity.ts b/backend/api/src/analytics/entities/domain-usage.entity.ts new file mode 100644 index 0000000..d19ddc8 --- /dev/null +++ b/backend/api/src/analytics/entities/domain-usage.entity.ts @@ -0,0 +1,57 @@ +import { + Entity, + PrimaryGeneratedColumn, + Column, + CreateDateColumn, + UpdateDateColumn, + ManyToOne, + Index, +} from 'typeorm'; +import { User } from '../../users/entities/user.entity'; + +@Entity('domain_usage') +@Index(['domain', 'userId']) +@Index(['domain', 'createdAt']) +export class DomainUsage { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column() + domain: string; + + @Column({ nullable: true }) + origin: string; + + @Column({ nullable: true }) + referer: string; + + @Column() + endpoint: string; + + @Column() + method: string; + + @Column({ nullable: true }) + userAgent: string; + + @Column({ nullable: true }) + ip: string; + + @Column({ type: 'int', default: 1 }) + requestCount: number; + + @Column({ nullable: true }) + userId: string; + + @ManyToOne(() => User, { nullable: true, onDelete: 'SET NULL' }) + user: User; + + @Column({ type: 'json', nullable: true }) + metadata: Record; + + @CreateDateColumn() + createdAt: Date; + + @UpdateDateColumn() + lastAccessAt: Date; +} diff --git a/backend/api/src/analytics/interceptors/domain-tracking.interceptor.ts b/backend/api/src/analytics/interceptors/domain-tracking.interceptor.ts new file mode 100644 index 0000000..9ee50c7 --- /dev/null +++ b/backend/api/src/analytics/interceptors/domain-tracking.interceptor.ts @@ -0,0 +1,90 @@ +import { + Injectable, + NestInterceptor, + ExecutionContext, + CallHandler, +} from '@nestjs/common'; +import { Observable } from 'rxjs'; +import { tap } from 'rxjs/operators'; +import { AnalyticsService } from '../analytics.service'; + +@Injectable() +export class DomainTrackingInterceptor implements NestInterceptor { + constructor(private analyticsService: AnalyticsService) {} + + intercept(context: ExecutionContext, next: CallHandler): Observable { + const request = context.switchToHttp().getRequest(); + + // Skip tracking for health checks and internal endpoints + if (this.shouldSkipTracking(request.url)) { + return next.handle(); + } + + // Extract tracking data from request + const trackingData = { + domain: request.headers.host || 'unknown', + origin: request.headers.origin, + referer: request.headers.referer, + endpoint: request.url, + method: request.method, + userAgent: request.headers['user-agent'], + ip: this.getClientIp(request), + userId: request.user?.id, + metadata: { + contentType: request.headers['content-type'], + acceptLanguage: request.headers['accept-language'], + }, + }; + + // Track asynchronously to not block the request + this.analyticsService.trackDomainUsage(trackingData).catch((error) => { + console.error('Failed to track domain usage:', error); + }); + + return next.handle().pipe( + tap({ + next: () => { + // Could track successful responses here if needed + }, + error: (error) => { + // Could track errors by domain if needed + if (error.status >= 500) { + // Track server errors separately if desired + } + }, + }), + ); + } + + private shouldSkipTracking(url: string): boolean { + const skipPatterns = [ + '/api/health', + '/api/metrics', + '/favicon.ico', + '/_next', + '/static', + ]; + + return skipPatterns.some((pattern) => url.startsWith(pattern)); + } + + private getClientIp(request: any): string { + // Check for various headers that might contain the real IP + const forwardedFor = request.headers['x-forwarded-for']; + if (forwardedFor) { + return forwardedFor.split(',')[0].trim(); + } + + const realIp = request.headers['x-real-ip']; + if (realIp) { + return realIp; + } + + // Fallback to connection remote address + return ( + request.connection?.remoteAddress || + request.socket?.remoteAddress || + 'unknown' + ); + } +} diff --git a/backend/api/src/analytics/test/analytics.service.spec.ts b/backend/api/src/analytics/test/analytics.service.spec.ts new file mode 100644 index 0000000..1f934ce --- /dev/null +++ b/backend/api/src/analytics/test/analytics.service.spec.ts @@ -0,0 +1,271 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { getRepositoryToken } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { AnalyticsService } from '../analytics.service'; +import { DomainUsage } from '../entities/domain-usage.entity'; +import { SlackService } from '../../slack/slack.service'; + +describe('AnalyticsService', () => { + let service: AnalyticsService; + let domainUsageRepository: Repository; + let slackService: SlackService; + + const mockDomainUsageRepository = { + findOne: jest.fn(), + find: jest.fn(), + save: jest.fn(), + create: jest.fn(), + }; + + const mockSlackService = { + sendNotification: jest.fn(), + }; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + AnalyticsService, + { + provide: getRepositoryToken(DomainUsage), + useValue: mockDomainUsageRepository, + }, + { + provide: SlackService, + useValue: mockSlackService, + }, + ], + }).compile(); + + service = module.get(AnalyticsService); + domainUsageRepository = module.get>( + getRepositoryToken(DomainUsage), + ); + slackService = module.get(SlackService); + + // Clear all mocks + jest.clearAllMocks(); + }); + + afterEach(() => { + // Clean up the buffer flush timer + service.onModuleDestroy(); + }); + + describe('trackDomainUsage', () => { + it('should track domain usage for authenticated user', async () => { + const trackingData = { + domain: 'app.datakit.page', + origin: 'https://app.datakit.page', + endpoint: '/api/ai/chat', + method: 'POST', + userId: 'user-123', + userAgent: 'Mozilla/5.0', + ip: '192.168.1.1', + }; + + mockDomainUsageRepository.findOne.mockResolvedValue(null); + + await service.trackDomainUsage(trackingData); + + // Wait for buffer to be processed + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(mockDomainUsageRepository.findOne).toHaveBeenCalled(); + }); + + it('should track domain usage for anonymous user', async () => { + const trackingData = { + domain: 'public.datakit.page', + origin: 'https://public.datakit.page', + endpoint: '/api/public/data', + method: 'GET', + userAgent: 'Mozilla/5.0', + ip: '192.168.1.2', + }; + + mockDomainUsageRepository.findOne.mockResolvedValue(null); + + await service.trackDomainUsage(trackingData); + + // Wait for buffer to be processed + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(mockDomainUsageRepository.findOne).toHaveBeenCalled(); + }); + + it('should notify Slack for new production domain', async () => { + const trackingData = { + domain: 'newcustomer.com', + origin: 'https://newcustomer.com', + endpoint: '/api/ai/chat', + method: 'POST', + userId: 'user-456', + }; + + mockDomainUsageRepository.findOne.mockResolvedValue(null); + + await service.trackDomainUsage(trackingData); + + // The notification happens asynchronously + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(mockSlackService.sendNotification).toHaveBeenCalledWith( + expect.objectContaining({ + text: expect.stringContaining('New Domain Detected'), + }), + ); + }); + + it('should not notify Slack for localhost domains', async () => { + const trackingData = { + domain: 'localhost:3000', + origin: 'http://localhost:3000', + endpoint: '/api/test', + method: 'GET', + }; + + await service.trackDomainUsage(trackingData); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(mockSlackService.sendNotification).not.toHaveBeenCalled(); + }); + }); + + describe('getDomainStats', () => { + it('should return aggregated domain statistics', async () => { + const mockUsages = [ + { + id: '1', + domain: 'app.datakit.page', + endpoint: '/api/ai/chat', + requestCount: 10, + userId: 'user-1', + createdAt: new Date('2024-01-01'), + lastAccessAt: new Date('2024-01-02'), + }, + { + id: '2', + domain: 'app.datakit.page', + endpoint: '/api/credits', + requestCount: 5, + userId: 'user-2', + createdAt: new Date('2024-01-01'), + lastAccessAt: new Date('2024-01-02'), + }, + { + id: '3', + domain: 'customer.com', + endpoint: '/api/ai/chat', + requestCount: 20, + userId: 'user-3', + createdAt: new Date('2024-01-01'), + lastAccessAt: new Date('2024-01-03'), + }, + ]; + + mockDomainUsageRepository.find.mockResolvedValue(mockUsages); + + const stats = await service.getDomainStats(); + + expect(stats).toHaveLength(2); + expect(stats[0].domain).toBe('customer.com'); + expect(stats[0].totalRequests).toBe(20); + expect(stats[1].domain).toBe('app.datakit.page'); + expect(stats[1].totalRequests).toBe(15); + }); + + it('should filter stats by date range', async () => { + const startDate = new Date('2024-01-01'); + const endDate = new Date('2024-01-31'); + + mockDomainUsageRepository.find.mockResolvedValue([]); + + await service.getDomainStats(startDate, endDate); + + expect(mockDomainUsageRepository.find).toHaveBeenCalledWith( + expect.objectContaining({ + where: expect.objectContaining({ + createdAt: expect.anything(), + }), + }), + ); + }); + }); + + describe('getUserDomains', () => { + it('should return unique domains for a user', async () => { + const userId = 'user-123'; + const mockUsages = [ + { domain: 'app.datakit.page' }, + { domain: 'app.datakit.page' }, + { domain: 'staging.datakit.page' }, + { domain: 'localhost:3000' }, + ]; + + mockDomainUsageRepository.find.mockResolvedValue(mockUsages); + + const domains = await service.getUserDomains(userId); + + expect(domains).toEqual([ + 'app.datakit.page', + 'staging.datakit.page', + 'localhost:3000', + ]); + expect(mockDomainUsageRepository.find).toHaveBeenCalledWith({ + where: { userId }, + select: ['domain'], + }); + }); + + it('should return empty array for user with no domains', async () => { + mockDomainUsageRepository.find.mockResolvedValue([]); + + const domains = await service.getUserDomains('user-999'); + + expect(domains).toEqual([]); + }); + }); + + describe('buffer management', () => { + it('should batch requests in buffer before saving', async () => { + const trackingData1 = { + domain: 'app.datakit.page', + origin: 'https://app.datakit.page', + endpoint: '/api/test', + method: 'GET', + userId: 'user-1', + }; + + const trackingData2 = { + domain: 'app.datakit.page', + origin: 'https://app.datakit.page', + endpoint: '/api/test', + method: 'GET', + userId: 'user-1', + }; + + mockDomainUsageRepository.findOne.mockResolvedValue({ + id: '1', + domain: 'app.datakit.page', + endpoint: '/api/test', + method: 'GET', + userId: 'user-1', + requestCount: 5, + lastAccessAt: new Date(), + }); + + await service.trackDomainUsage(trackingData1); + await service.trackDomainUsage(trackingData2); + + // Force buffer flush + await service['flushBuffer'](); + + expect(mockDomainUsageRepository.save).toHaveBeenCalledWith( + expect.objectContaining({ + requestCount: expect.any(Number), + }), + ); + }); + }); +}); diff --git a/backend/api/src/analytics/test/domain-tracking.interceptor.spec.ts b/backend/api/src/analytics/test/domain-tracking.interceptor.spec.ts new file mode 100644 index 0000000..9f03c60 --- /dev/null +++ b/backend/api/src/analytics/test/domain-tracking.interceptor.spec.ts @@ -0,0 +1,238 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ExecutionContext, CallHandler } from '@nestjs/common'; +import { of } from 'rxjs'; +import { DomainTrackingInterceptor } from '../interceptors/domain-tracking.interceptor'; +import { AnalyticsService } from '../analytics.service'; + +describe('DomainTrackingInterceptor', () => { + let interceptor: DomainTrackingInterceptor; + let analyticsService: AnalyticsService; + + const mockAnalyticsService = { + trackDomainUsage: jest.fn().mockResolvedValue(undefined), + }; + + const mockCallHandler: CallHandler = { + handle: jest.fn().mockReturnValue(of('test-response')), + }; + + const createMockExecutionContext = (request: any): ExecutionContext => ({ + switchToHttp: () => ({ + getRequest: () => request, + getResponse: () => ({}) as any, + getNext: () => ({}) as any, + }), + getClass: () => ({}) as any, + getHandler: () => ({}) as any, + getArgs: () => [] as any, + getArgByIndex: () => ({}) as any, + switchToRpc: () => ({}) as any, + switchToWs: () => ({}) as any, + getType: () => 'http' as any, + }); + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + DomainTrackingInterceptor, + { + provide: AnalyticsService, + useValue: mockAnalyticsService, + }, + ], + }).compile(); + + interceptor = module.get( + DomainTrackingInterceptor, + ); + analyticsService = module.get(AnalyticsService); + + jest.clearAllMocks(); + }); + + describe('intercept', () => { + it('should track domain usage for regular requests', (done) => { + const mockRequest = { + url: '/api/ai/chat', + method: 'POST', + headers: { + host: 'app.datakit.page', + origin: 'https://app.datakit.page', + referer: 'https://app.datakit.page/dashboard', + 'user-agent': 'Mozilla/5.0', + 'content-type': 'application/json', + 'accept-language': 'en-US', + }, + user: { id: 'user-123' }, + connection: { remoteAddress: '192.168.1.1' }, + }; + + const context = createMockExecutionContext(mockRequest); + + interceptor.intercept(context, mockCallHandler).subscribe({ + next: (value) => { + expect(value).toBe('test-response'); + expect(mockAnalyticsService.trackDomainUsage).toHaveBeenCalledWith({ + domain: 'app.datakit.page', + origin: 'https://app.datakit.page', + referer: 'https://app.datakit.page/dashboard', + endpoint: '/api/ai/chat', + method: 'POST', + userAgent: 'Mozilla/5.0', + ip: '192.168.1.1', + userId: 'user-123', + metadata: { + contentType: 'application/json', + acceptLanguage: 'en-US', + }, + }); + done(); + }, + }); + }); + + it('should handle requests without user (anonymous)', (done) => { + const mockRequest = { + url: '/api/public/data', + method: 'GET', + headers: { + host: 'public.datakit.page', + 'user-agent': 'curl/7.64.1', + }, + connection: { remoteAddress: '10.0.0.1' }, + }; + + const context = createMockExecutionContext(mockRequest); + + interceptor.intercept(context, mockCallHandler).subscribe({ + next: () => { + expect(mockAnalyticsService.trackDomainUsage).toHaveBeenCalledWith( + expect.objectContaining({ + userId: undefined, + domain: 'public.datakit.page', + }), + ); + done(); + }, + }); + }); + + it('should skip tracking for health check endpoints', (done) => { + const mockRequest = { + url: '/api/health', + method: 'GET', + headers: { + host: 'app.datakit.page', + }, + }; + + const context = createMockExecutionContext(mockRequest); + + interceptor.intercept(context, mockCallHandler).subscribe({ + next: () => { + expect(mockAnalyticsService.trackDomainUsage).not.toHaveBeenCalled(); + done(); + }, + }); + }); + + it('should extract IP from x-forwarded-for header', (done) => { + const mockRequest = { + url: '/api/test', + method: 'GET', + headers: { + host: 'app.datakit.page', + 'x-forwarded-for': '203.0.113.0, 198.51.100.0', + }, + }; + + const context = createMockExecutionContext(mockRequest); + + interceptor.intercept(context, mockCallHandler).subscribe({ + next: () => { + expect(mockAnalyticsService.trackDomainUsage).toHaveBeenCalledWith( + expect.objectContaining({ + ip: '203.0.113.0', + }), + ); + done(); + }, + }); + }); + + it('should extract IP from x-real-ip header', (done) => { + const mockRequest = { + url: '/api/test', + method: 'GET', + headers: { + host: 'app.datakit.page', + 'x-real-ip': '203.0.113.0', + }, + }; + + const context = createMockExecutionContext(mockRequest); + + interceptor.intercept(context, mockCallHandler).subscribe({ + next: () => { + expect(mockAnalyticsService.trackDomainUsage).toHaveBeenCalledWith( + expect.objectContaining({ + ip: '203.0.113.0', + }), + ); + done(); + }, + }); + }); + + it('should handle tracking errors gracefully', (done) => { + mockAnalyticsService.trackDomainUsage.mockRejectedValueOnce( + new Error('Tracking failed'), + ); + + const mockRequest = { + url: '/api/test', + method: 'GET', + headers: { + host: 'app.datakit.page', + }, + }; + + const context = createMockExecutionContext(mockRequest); + const consoleSpy = jest.spyOn(console, 'error').mockImplementation(); + + interceptor.intercept(context, mockCallHandler).subscribe({ + next: (value) => { + // Should still return the response even if tracking fails + expect(value).toBe('test-response'); + setTimeout(() => { + expect(consoleSpy).toHaveBeenCalledWith( + 'Failed to track domain usage:', + expect.any(Error), + ); + consoleSpy.mockRestore(); + done(); + }, 100); + }, + }); + }); + + it('should skip tracking for static assets', (done) => { + const mockRequest = { + url: '/static/css/main.css', + method: 'GET', + headers: { + host: 'app.datakit.page', + }, + }; + + const context = createMockExecutionContext(mockRequest); + + interceptor.intercept(context, mockCallHandler).subscribe({ + next: () => { + expect(mockAnalyticsService.trackDomainUsage).not.toHaveBeenCalled(); + done(); + }, + }); + }); + }); +}); diff --git a/backend/api/src/app.module.ts b/backend/api/src/app.module.ts index e7896a4..90771fa 100644 --- a/backend/api/src/app.module.ts +++ b/backend/api/src/app.module.ts @@ -13,6 +13,7 @@ import { CreditsModule } from './credits/credits.module'; import { AIModule } from './ai/ai.module'; import { WorkspacesModule } from './workspaces/workspaces.module'; import { WaitlistModule } from './waitlist/waitlist.module'; +import { AnalyticsModule } from './analytics/analytics.module'; import { getDatabaseConfig } from './config/database.config'; import { CustomThrottlerGuard } from './common/guards/custom-throttler.guard'; @@ -67,6 +68,7 @@ import { CustomThrottlerGuard } from './common/guards/custom-throttler.guard'; CreditsModule, AIModule, WaitlistModule, + AnalyticsModule, ], controllers: [AppController], providers: [ diff --git a/backend/api/src/migrations/1755122574878-CreateDomainUsageTable.ts b/backend/api/src/migrations/1755122574878-CreateDomainUsageTable.ts new file mode 100644 index 0000000..caedaa4 --- /dev/null +++ b/backend/api/src/migrations/1755122574878-CreateDomainUsageTable.ts @@ -0,0 +1,49 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class CreateDomainUsageTable1755122574878 implements MigrationInterface { + name = 'CreateDomainUsageTable1755122574878'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE TABLE "domain_usage" ( + "id" uuid NOT NULL DEFAULT uuid_generate_v4(), + "domain" character varying NOT NULL, + "origin" character varying, + "referer" character varying, + "endpoint" character varying NOT NULL, + "method" character varying NOT NULL, + "userAgent" text, + "ip" character varying, + "requestCount" integer NOT NULL DEFAULT 1, + "userId" uuid, + "metadata" json, + "createdAt" TIMESTAMP NOT NULL DEFAULT now(), + "lastAccessAt" TIMESTAMP NOT NULL DEFAULT now(), + CONSTRAINT "PK_domain_usage_id" PRIMARY KEY ("id") + ) + `); + + await queryRunner.query(` + CREATE INDEX "IDX_domain_usage_domain_userId" ON "domain_usage" ("domain", "userId") + `); + + await queryRunner.query(` + CREATE INDEX "IDX_domain_usage_domain_createdAt" ON "domain_usage" ("domain", "createdAt") + `); + + await queryRunner.query(` + ALTER TABLE "domain_usage" + ADD CONSTRAINT "FK_domain_usage_user" + FOREIGN KEY ("userId") + REFERENCES "users"("id") + ON DELETE SET NULL ON UPDATE NO ACTION + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "domain_usage" DROP CONSTRAINT "FK_domain_usage_user"`); + await queryRunner.query(`DROP INDEX "public"."IDX_domain_usage_domain_createdAt"`); + await queryRunner.query(`DROP INDEX "public"."IDX_domain_usage_domain_userId"`); + await queryRunner.query(`DROP TABLE "domain_usage"`); + } +} \ No newline at end of file