Feature/File_Sharing

backend/api/.env.example

@@ -36,4 +36,17 @@ BCRYPT_SALT_ROUNDS==12
 
 ALLOWED_ORIGINS='pages.dev,datakit.page'
 
-SLACK_WEBHOOK_URL=
+SLACK_WEBHOOK_URL=
+
+# R2 file sharing
+R2_ACCOUNT_ID=your_account_id
+R2_ACCESS_KEY_ID=your_access_key
+R2_SECRET_ACCESS_KEY=your_secret_key
+R2_BUCKET_NAME=datakit-shares
+
+R2_CLOUD_ACCOUNT_ID=
+R2_CLOUD_ACCESS_KEY_ID=
+R2_CLOUD_SECRET_ACCESS_KEY=
+R2_CLOUD_BUCKET_NAME=datakit-cloud-storage
+
+SHARE_BASE_URL=https://share.datakit.page/files

backend/api/package.json

@@ -35,12 +35,14 @@
     "migration:create": "npm run typeorm -- migration:create"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.879.0",
+    "@aws-sdk/s3-request-presigner": "^3.879.0",
     "@nestjs/common": "^10.0.0",
     "@nestjs/config": "^4.0.2",
     "@nestjs/core": "^10.0.0",
     "@nestjs/jwt": "^11.0.0",
     "@nestjs/passport": "^11.0.5",
-    "@nestjs/platform-express": "^10.0.0",
+    "@nestjs/platform-express": "^10.4.20",
     "@nestjs/schedule": "^6.0.0",
     "@nestjs/throttler": "^6.4.0",
     "@nestjs/typeorm": "^11.0.0",
@@ -54,6 +56,8 @@
     "cookie-parser": "^1.4.7",
     "duckdb": "^1.3.2",
     "helmet": "^8.1.0",
+    "multer": "^2.0.2",
+    "nanoid": "^5.1.5",
     "passport": "^0.7.0",
     "passport-jwt": "^4.0.1",
     "passport-local": "^1.0.0",
@@ -69,6 +73,7 @@
     "@types/bcryptjs": "^2.4.6",
     "@types/express": "^5.0.0",
     "@types/jest": "^29.5.2",
+    "@types/multer": "^2.0.0",
     "@types/node": "^20.3.1",
     "@types/passport-jwt": "^4.0.1",
     "@types/passport-local": "^1.0.38",

backend/api/src/app.module.ts

@@ -14,6 +14,8 @@ import { AIModule } from './ai/ai.module';
 import { WorkspacesModule } from './workspaces/workspaces.module';
 import { WaitlistModule } from './waitlist/waitlist.module';
 import { PostgresProxyModule } from './postgres-proxy/postgres-proxy.module';
+import { CloudStorageModule } from './cloud-storage/cloud-storage.module';
+import { ProjectSharingModule } from './project-sharing/project-sharing.module';
 import { getDatabaseConfig } from './config/database.config';
 import { CustomThrottlerGuard } from './common/guards/custom-throttler.guard';
 
@@ -69,6 +71,8 @@ import { CustomThrottlerGuard } from './common/guards/custom-throttler.guard';
     AIModule,
     WaitlistModule,
     PostgresProxyModule,
+    CloudStorageModule,
+    ProjectSharingModule,
   ],
   controllers: [AppController],
   providers: [

backend/api/src/auth/guards/optional-jwt-auth.guard.ts

@@ -0,0 +1,20 @@
+import { Injectable, ExecutionContext } from '@nestjs/common';
+import { AuthGuard } from '@nestjs/passport';
+
+@Injectable()
+export class OptionalJwtAuthGuard extends AuthGuard('jwt') {
+  canActivate(context: ExecutionContext) {
+    // Add your custom authentication logic here
+    // for example, call super.canActivate(context) to run the standard JWT authentication
+    return super.canActivate(context);
+  }
+
+  handleRequest(err: any, user: any) {
+    // Don't throw an error if the user is not authenticated
+    // Just return null/undefined which will be available as req.user
+    if (err || !user) {
+      return null;
+    }
+    return user;
+  }
+}

backend/api/src/cloud-storage/cloud-storage.controller.ts

@@ -0,0 +1,198 @@
+import {
+  Controller,
+  Post,
+  Get,
+  Delete,
+  Patch,
+  Param,
+  Body,
+  UseGuards,
+  UseInterceptors,
+  UploadedFile,
+  Request,
+  HttpCode,
+  HttpStatus,
+} from '@nestjs/common';
+import { FileInterceptor } from '@nestjs/platform-express';
+import { JwtAuthGuard } from '../auth/guards/jwt-auth.guard';
+import { CloudStorageService } from './cloud-storage.service';
+import { CreateCloudProjectDto } from './dto/create-cloud-project.dto';
+import { SaveToCloudDto } from './dto/cloud-storage-response.dto';
+
+@Controller('cloud-storage')
+@UseGuards(JwtAuthGuard)
+export class CloudStorageController {
+  constructor(private readonly cloudStorageService: CloudStorageService) {}
+
+  /**
+   * Create a new cloud project in a workspace
+   */
+  @Post('workspace/:workspaceId/project')
+  async createProject(
+    @Request() req,
+    @Param('workspaceId') workspaceId: string,
+    @Body() dto: CreateCloudProjectDto,
+  ) {
+    const userId = req.user.id;
+    return this.cloudStorageService.createProject(userId, workspaceId, dto);
+  }
+
+  /**
+   * Get workspace's cloud projects
+   */
+  @Get('workspace/:workspaceId/projects')
+  async getWorkspaceProjects(
+    @Request() req,
+    @Param('workspaceId') workspaceId: string,
+  ) {
+    const userId = req.user.id;
+    return this.cloudStorageService.getWorkspaceProjects(userId, workspaceId);
+  }
+
+  /**
+   * Get all projects user has access to
+   */
+  @Get('projects')
+  async getUserAccessibleProjects(@Request() req) {
+    const userId = req.user.id;
+    return this.cloudStorageService.getUserAccessibleProjects(userId);
+  }
+
+  /**
+   * Upload file to cloud project
+   */
+  @Post('upload')
+  @UseInterceptors(
+    FileInterceptor('file', {
+      limits: {
+        fileSize: 500 * 1024 * 1024, // 500MB max
+      },
+    }),
+  )
+  async uploadToCloud(
+    @Request() req,
+    @UploadedFile() file: Express.Multer.File,
+    @Body() body: any,
+  ) {
+    if (!file) {
+      throw new Error('No file uploaded');
+    }
+
+    const userId = req.user.id;
+
+    // Parse metadata from body
+    const dto: SaveToCloudDto = {
+      projectId: body.projectId, // Changed from workspaceId
+      fileName: body.fileName,
+      metadata: body.metadata ? JSON.parse(body.metadata) : undefined,
+      replaceIfExists: body.replaceIfExists === 'true',
+      keepVersionHistory: body.keepVersionHistory === 'true',
+    };
+
+    return this.cloudStorageService.saveToCloud(userId, file, dto);
+  }
+
+  /**
+   * Get files in a project
+   */
+  @Get('project/:projectId/files')
+  async getProjectFiles(@Request() req, @Param('projectId') projectId: string) {
+    const userId = req.user.id;
+    return this.cloudStorageService.getProjectFiles(userId, projectId);
+  }
+
+  /**
+   * Get all user's cloud files across all projects
+   */
+  @Get('files')
+  async getAllUserFiles(@Request() req) {
+    const userId = req.user.id;
+    const projects =
+      await this.cloudStorageService.getUserAccessibleProjects(userId);
+
+    const allFiles = [];
+    for (const project of projects) {
+      const files = await this.cloudStorageService.getProjectFiles(
+        userId,
+        project.id,
+      );
+      allFiles.push(
+        ...files.map((file) => ({
+          ...file,
+          projectName: project.name,
+          projectId: project.id,
+        })),
+      );
+    }
+
+    return allFiles;
+  }
+
+  /**
+   * Get file access/download URL
+   */
+  @Post('file/:fileId/access')
+  @HttpCode(HttpStatus.OK)
+  async getFileAccess(@Request() req, @Param('fileId') fileId: string) {
+    const userId = req.user.id;
+    return this.cloudStorageService.getFileAccess(userId, fileId);
+  }
+
+  /**
+   * Delete file from cloud
+   */
+  @Delete('file/:fileId')
+  async deleteFile(@Request() req, @Param('fileId') fileId: string) {
+    const userId = req.user.id;
+    await this.cloudStorageService.deleteFile(userId, fileId);
+    return { message: 'File deleted successfully' };
+  }
+
+  /**
+   * Get user's aggregated storage statistics
+   */
+  @Get('stats')
+  async getUserStorageStats(@Request() req) {
+    const userId = req.user.id;
+    return this.cloudStorageService.getUserStorageStats(userId);
+  }
+
+  /**
+   * Get workspace's storage statistics
+   */
+  @Get('workspace/:workspaceId/stats')
+  async getWorkspaceStorageStats(
+    @Request() req,
+    @Param('workspaceId') workspaceId: string,
+  ) {
+    const userId = req.user.id;
+
+    // Check access - method is private, so we'll call getWorkspaceProjects first
+    await this.cloudStorageService.getWorkspaceProjects(userId, workspaceId);
+
+    return this.cloudStorageService.getWorkspaceStorageStats(workspaceId);
+  }
+
+  /**
+   * Update project
+   */
+  @Patch('project/:projectId')
+  async updateProject(
+    @Request() req,
+    @Param('projectId') projectId: string,
+    @Body() dto: { name?: string; description?: string },
+  ) {
+    const userId = req.user.id;
+    return this.cloudStorageService.updateProject(userId, projectId, dto);
+  }
+
+  /**
+   * Delete project (and all its files)
+   */
+  @Delete('project/:projectId')
+  async deleteProject(@Request() req, @Param('projectId') projectId: string) {
+    const userId = req.user.id;
+    await this.cloudStorageService.deleteProject(userId, projectId);
+    return { message: 'Project deleted successfully' };
+  }
+}

backend/api/src/cloud-storage/cloud-storage.module.ts

@@ -0,0 +1,30 @@
+import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
+import { ConfigModule } from '@nestjs/config';
+import { CloudStorageController } from './cloud-storage.controller';
+import { CloudStorageService } from './cloud-storage.service';
+import { R2CloudStorageService } from './r2-cloud-storage.service';
+import { CloudProject } from './entities/cloud-project.entity';
+import { CloudFile } from './entities/cloud-file.entity';
+import { User } from '../users/entities/user.entity';
+import { Workspace } from '../workspaces/entities/workspace.entity';
+import { WorkspaceMember } from '../workspaces/entities/workspace-member.entity';
+import { Subscription } from '../subscriptions/entities/subscription.entity';
+
+@Module({
+  imports: [
+    ConfigModule,
+    TypeOrmModule.forFeature([
+      CloudProject,
+      CloudFile,
+      User,
+      Workspace,
+      WorkspaceMember,
+      Subscription,
+    ]),
+  ],
+  controllers: [CloudStorageController],
+  providers: [CloudStorageService, R2CloudStorageService],
+  exports: [CloudStorageService],
+})
+export class CloudStorageModule {}