From a9c736c5e17f2d8113a172eb9c171dd4e75832c6 Mon Sep 17 00:00:00 2001 From: venu123143 Date: Sat, 21 Feb 2026 23:51:23 +0530 Subject: [PATCH] get api changes --- .gitignore | 3 +- src/config/s3-secondary.config.ts | 115 ++++++++++ src/config/s3.config.ts | 56 ++++- src/controllers/upload.controller.ts | 40 +++- src/middleware/auth.middleware.ts | 2 + src/routes/upload.routes.ts | 5 + src/scripts/migrate-storage.ts | 312 +++++++++++++++++++++++++++ src/validation/upload.validation.ts | 30 ++- 8 files changed, 553 insertions(+), 10 deletions(-) create mode 100644 src/config/s3-secondary.config.ts create mode 100644 src/scripts/migrate-storage.ts diff --git a/.gitignore b/.gitignore index 83206bb..11d848d 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,8 @@ pids *.pid *.seed *.pid.lock - +*.md +*.txt # Directory for instrumented libs generated by jscoverage/JSCover lib-cov diff --git a/src/config/s3-secondary.config.ts b/src/config/s3-secondary.config.ts new file mode 100644 index 0000000..d4d431a --- /dev/null +++ b/src/config/s3-secondary.config.ts @@ -0,0 +1,115 @@ +import { + S3Client, + PutObjectCommand, + GetObjectCommand, + DeleteObjectCommand, + DeleteObjectsCommand, + ListObjectsV2Command, + type PutObjectCommandInput, + HeadObjectCommand, +} from "@aws-sdk/client-s3"; +import { FolderNameEnum } from "@/config/s3.config"; + +// export enum SecondaryFolderNameEnum { +// FILES = "files", +// VIDEOS = "videos", +// IMAGES = "images", +// DOCUMENTS = "documents", +// RECORDINGS = "recordings" +// } + +export class SecondaryS3Service { + private s3Client: S3Client; + private bucketName: string; + + constructor() { + this.bucketName = process.env.SECONDARY_S3_BUCKET_NAME!; + + this.s3Client = new S3Client({ + endpoint: process.env.SECONDARY_S3_ENDPOINT!, + region: process.env.SECONDARY_S3_REGION || 'auto', + credentials: { + accessKeyId: process.env.SECONDARY_S3_TOKEN_ID!, + secretAccessKey: process.env.SECONDARY_S3_SECRET_KEY!, + }, + forcePathStyle: true, + requestHandler: { + requestTimeout: 300000, // 5 minutes timeout + } as any, + }); + } + + public async listFiles(folder?: FolderNameEnum, maxKeys: number = 1000, continuationToken?: string) { + const command = new ListObjectsV2Command({ + Bucket: this.bucketName, + Prefix: folder ? `${folder}/` : undefined, + MaxKeys: maxKeys, + ContinuationToken: continuationToken, + }); + + const response = await this.s3Client.send(command); + return { + files: response.Contents?.map(obj => ({ + key: obj.Key!, + size: obj.Size!, + lastModified: obj.LastModified!, + etag: obj.ETag!, + })) || [], + isTruncated: response.IsTruncated || false, + nextContinuationToken: response.NextContinuationToken, + }; + } + + public async getFile(key: string): Promise { + const command = new GetObjectCommand({ + Bucket: this.bucketName, + Key: key, + }); + + const response = await this.s3Client.send(command); + + if (!response.Body) { + throw new Error(`No body returned for key: ${key}`); + } + + // Convert stream to buffer + const chunks: Uint8Array[] = []; + for await (const chunk of response.Body as any) { + chunks.push(chunk); + } + return Buffer.concat(chunks); + } + + public async getFileStream(key: string) { + const command = new GetObjectCommand({ + Bucket: this.bucketName, + Key: key, + }); + + const response = await this.s3Client.send(command); + + if (!response.Body) { + throw new Error(`No body returned for key: ${key}`); + } + + return response.Body; + } + + public async getMetadata(key: string) { + const command = new HeadObjectCommand({ + Bucket: this.bucketName, + Key: key, + }); + const response = await this.s3Client.send(command); + return { + contentType: response.ContentType, + contentLength: response.ContentLength, + lastModified: response.LastModified, + metadata: response.Metadata, + etag: response.ETag, + }; + } +} + +export default new SecondaryS3Service(); + diff --git a/src/config/s3.config.ts b/src/config/s3.config.ts index 6b303b0..57e2e9b 100644 --- a/src/config/s3.config.ts +++ b/src/config/s3.config.ts @@ -53,7 +53,7 @@ export class S3Service { } public buildCDNUrl(key: string): string { - return `https://${config.CLOUDFLARE.CDN_DOMAIN}/${key}`; + return `https://fileflow.fsn1.your-objectstorage.com/${key}`; } /** @@ -78,7 +78,7 @@ export class S3Service { public async uploadFile(file: IFile): Promise { // Sanitize original name for HTTP header compatibility const sanitizedOriginalName = this.sanitizeMetadataValue(file.originalName); - + const params: PutObjectCommandInput = { Bucket: config.S3.BUCKET_NAME, Key: `${FolderNameEnum.FILES}/${file.filename}`, @@ -106,7 +106,7 @@ export class S3Service { const path = `${folder}/${key}`; // Sanitize original name for HTTP header compatibility const sanitizedOriginalName = this.sanitizeMetadataValue(fileName); - + const params = { Bucket: config.S3.BUCKET_NAME, Key: path, @@ -124,6 +124,35 @@ export class S3Service { return this.buildCDNUrl(path); } + public async uploadStream( + key: string, + stream: any, + fileName: string, + mimeType: string, + size: number, + folder: FolderNameEnum = FolderNameEnum.FILES + ): Promise { + const path = `${folder}/${key}`; + const sanitizedOriginalName = this.sanitizeMetadataValue(fileName); + + const params = { + Bucket: config.S3.BUCKET_NAME, + Key: path, + Body: stream, + ContentType: mimeType, + ContentLength: size, // Required for streaming + Metadata: { + originalName: sanitizedOriginalName, + uploadedAt: new Date().toISOString(), + size: size.toString(), + }, + CacheControl: 'public, max-age=31536000', + }; + + await this.s3Client.send(new PutObjectCommand(params)); + return this.buildCDNUrl(path); + } + public async getFileUrl(key: string, expiresIn: number = 3600): Promise { const command = new GetObjectCommand({ Bucket: config.S3.BUCKET_NAME, @@ -182,13 +211,32 @@ export class S3Service { size: obj.Size!, lastModified: obj.LastModified!, etag: obj.ETag!, - cdnUrl: this.buildCDNUrl(obj.Key!) + storageClass: obj.StorageClass, + owner: obj.Owner ? { + displayName: obj.Owner.DisplayName, + id: obj.Owner.ID + } : undefined, + cdnUrl: this.buildCDNUrl(obj.Key!), + // Include all other S3 properties + ...(obj as any) })) || [], isTruncated: response.IsTruncated || false, nextContinuationToken: response.NextContinuationToken, + keyCount: response.KeyCount, + maxKeys: response.MaxKeys, + prefix: response.Prefix, + delimiter: response.Delimiter, + encodingType: response.EncodingType, + commonPrefixes: response.CommonPrefixes, }; } + public async getAllFiles(folder?: string, maxKeys: number = 100, continuationToken?: string) { + // Convert string folder name to FolderNameEnum if provided + const folderEnum = folder ? (folder as FolderNameEnum) : undefined; + return await this.listFiles(folderEnum, maxKeys, continuationToken); + } + public async initiateMultipartUpload(fileName: string, mimeType: string): Promise<{ uploadId: string | undefined; key: string }> { const safeName = slugify(fileName, { replacement: "_", // replace invalid chars with underscore diff --git a/src/controllers/upload.controller.ts b/src/controllers/upload.controller.ts index 63cf67b..6051dca 100644 --- a/src/controllers/upload.controller.ts +++ b/src/controllers/upload.controller.ts @@ -12,7 +12,7 @@ const uploadFile = async (c: Context) => { if (!user?.id) { return res.FailureResponse(c, 400, { message: "User not found" }) } - try { + try { const formData = await c.req.formData() const files = formData.getAll("files") as File[] @@ -33,7 +33,7 @@ const uploadFile = async (c: Context) => { data: { results }, related_user_id: user.id, }) - + // Track upload analytics for each file files.forEach((file, index) => { const result = results[index]; @@ -177,7 +177,7 @@ const completeUpload = async (c: Context) => { file_type: metadata.contentType || '', storage_path: key } - + if (user?.id) { const fileName = key.split('/').pop() || 'Unknown file' addToNotificationQueue({ @@ -195,7 +195,7 @@ const completeUpload = async (c: Context) => { return res.SuccessResponse(c, 200, { message: "Upload completed successfully", data: result - }) + }) } catch (error: any) { if (user?.id) { const fileName = key.split('/').pop() || 'Unknown file' @@ -287,7 +287,39 @@ const getPartsByUploadKey = async (c: Context) => { } } +const getAllFiles = async (c: Context) => { + try { + const validatedQuery = c.get('validatedQuery') as { + folder?: string; + maxKeys?: number; + continuationToken?: string + }; + + const { folder, maxKeys = 100, continuationToken } = validatedQuery || {}; + + const result = await s3Service.getAllFiles(folder, maxKeys, continuationToken); + return res.SuccessResponse(c, 200, { + message: "All files retrieved successfully", + data: { + files: result.files, + pagination: { + hasMore: result.isTruncated, + nextContinuationToken: result.nextContinuationToken || null, + maxKeys: maxKeys, + currentCount: result.files.length + } + }, + }) + } catch (error: any) { + return res.FailureResponse(c, 500, { + message: "Failed to get all files", + error: error.message, + }) + } +} + export default { + getAllFiles, getPartsByUploadKey, initiateUpload, uploadFile, diff --git a/src/middleware/auth.middleware.ts b/src/middleware/auth.middleware.ts index 66b75fd..e7352b8 100644 --- a/src/middleware/auth.middleware.ts +++ b/src/middleware/auth.middleware.ts @@ -7,6 +7,7 @@ import redisConn from "@/config/redis.config"; import redisConstants from "@/global/redis-constants"; import { getValidPinSession } from "@/core/session"; import crypto from "crypto"; +import { UserRole } from "@/models/User.model"; @@ -150,6 +151,7 @@ const pinSessionMiddleware: MiddlewareHandler = async (c: Context, next: Next) = } }; + export default { authMiddleware, pinSessionMiddleware, diff --git a/src/routes/upload.routes.ts b/src/routes/upload.routes.ts index 248fe19..e75144c 100644 --- a/src/routes/upload.routes.ts +++ b/src/routes/upload.routes.ts @@ -64,6 +64,11 @@ export class UploadRouter { validateParams(uploadValidation.fileNameValidation), uploadController.deleteFile ) + this.router.get('/file/get-all-files', + Middleware.authMiddleware, + validateQuery(uploadValidation.getAllFilesValidation), + uploadController.getAllFiles + ) } public getRouter() { diff --git a/src/scripts/migrate-storage.ts b/src/scripts/migrate-storage.ts new file mode 100644 index 0000000..592f344 --- /dev/null +++ b/src/scripts/migrate-storage.ts @@ -0,0 +1,312 @@ +import 'dotenv/config'; +import secondaryS3Service from '@/config/s3-secondary.config'; +import primaryS3Service, { FolderNameEnum } from '@/config/s3.config'; + +import fs from 'fs'; +import path from 'path'; +import os from 'os'; + +interface MigrationResult { + successful: string[]; + failed: Array<{ key: string; error: string }>; + skipped: string[]; + totalFiles: number; + totalSize: number; + startTime: Date; + endTime?: Date; +} + +class StorageMigration { + private result: MigrationResult = { + successful: [], + failed: [], + skipped: [], + totalFiles: 0, + totalSize: 0, + startTime: new Date(), + }; + private skipExisting: boolean; + + constructor(skipExisting: boolean = false) { + this.skipExisting = skipExisting; + } + + /** + * Migrate files from secondary storage to primary storage + * @param sourceFolder - Folder to migrate from secondary storage + * @param targetFolder - Target folder in primary storage (defaults to same as source) + */ + async migrateFolder(sourceFolder: FolderNameEnum, targetFolder: FolderNameEnum = FolderNameEnum.IMAGES) { + console.log(`\nšŸš€ Starting migration from secondary storage (${sourceFolder}) to primary storage (${targetFolder})`); + console.log(`ā° Started at: ${this.result.startTime.toISOString()}`); + console.log(`šŸ’¾ Available Memory: ${this.formatBytes(os.freemem())} / ${this.formatBytes(os.totalmem())}`); + console.log(`šŸ“‹ Skip existing files: ${this.skipExisting ? 'Yes' : 'No'}\n`); + + let continuationToken: string | undefined; + let batchNumber = 1; + + do { + console.log(`šŸ“¦ Processing batch ${batchNumber}...`); + // Process 100 files at a time + const listResult = await secondaryS3Service.listFiles(sourceFolder, 100, continuationToken); + + this.result.totalFiles += listResult.files.length; + + // Process files one at a time for large video files to avoid memory and timeout issues + const CONCURRENT_UPLOADS = 1; + for (let i = 0; i < listResult.files.length; i += CONCURRENT_UPLOADS) { + const batch = listResult.files.slice(i, i + CONCURRENT_UPLOADS); + await Promise.all( + batch.map(file => this.migrateFile(file.key, targetFolder, file.size)) + ); + } + + continuationToken = listResult.nextContinuationToken; + batchNumber++; + + console.log(`āœ… Batch ${batchNumber - 1} completed. Progress: ${this.result.successful.length}/${this.result.totalFiles} successful\n`); + + } while (continuationToken); + + this.result.endTime = new Date(); + this.printSummary(); + this.saveResultsToFile(); + } + + /** + * Check if file already exists in primary storage with retry logic + */ + private async fileExists(fileName: string, targetFolder: FolderNameEnum): Promise { + const key = `${targetFolder}/${fileName}`; + const maxRetries = 3; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await primaryS3Service.getMetadata(key); + return true; + } catch (error: any) { + // If it's a 404/not found error, file doesn't exist + if (error?.name === 'NotFound' || error?.$metadata?.httpStatusCode === 404) { + return false; + } + + // For other errors (rate limit, network), retry with backoff + if (attempt < maxRetries) { + const delay = Math.pow(2, attempt) * 1000; // Exponential backoff: 2s, 4s, 8s + console.log(` āš ļø Retry ${attempt}/${maxRetries} checking existence after ${delay}ms...`); + await new Promise(resolve => setTimeout(resolve, delay)); + } else { + // After all retries failed, assume file doesn't exist (safer to re-upload than skip) + console.log(` āš ļø Could not verify existence, will attempt migration`); + return false; + } + } + } + + return false; + } + + /** + * Migrate a single file using streaming to avoid memory issues + */ + private async migrateFile(key: string, targetFolder: FolderNameEnum, fileSize: number): Promise { + const fileName = key.split('/').pop() || key; + + try { + // Check if we should skip existing files + if (this.skipExisting) { + const exists = await this.fileExists(fileName, targetFolder); + if (exists) { + this.result.skipped.push(key); + console.log(` ā­ļø Skipped (already exists): ${key}`); + return; + } + } + + console.log(` šŸ“„ Migrating: ${key} (${this.formatBytes(fileSize)})`); + + // Check available memory before processing + const freeMem = os.freemem(); + const MIN_MEMORY = 500 * 1024 * 1024; // 500 MB minimum + + if (freeMem < MIN_MEMORY) { + throw new Error(`Insufficient memory: ${this.formatBytes(freeMem)} available`); + } + + // Add delay between files to avoid rate limiting (100ms) + await new Promise(resolve => setTimeout(resolve, 100)); + + // Get file metadata from secondary storage with retry + let metadata: Awaited> | undefined; + for (let attempt = 1; attempt <= 3; attempt++) { + try { + metadata = await secondaryS3Service.getMetadata(key); + break; + } catch (error) { + if (attempt === 3) throw error; + const delay = attempt * 2000; + console.log(` āš ļø Retry ${attempt}/3 getting metadata after ${delay}ms...`); + await new Promise(resolve => setTimeout(resolve, delay)); + } + } + + if (!metadata) { + throw new Error('Failed to get metadata after 3 attempts'); + } + + // For videos, use buffer method with timeout + console.log(` ā¬‡ļø Downloading from R2...`); + + // Increase timeout for very large files (10 minutes for files > 400MB) + const downloadTimeout = fileSize > 400 * 1024 * 1024 ? 600000 : 300000; + const fileBuffer = await Promise.race([ + secondaryS3Service.getFile(key), + new Promise((_, reject) => + setTimeout(() => reject(new Error(`Download timeout after ${downloadTimeout/1000/60} minutes`)), downloadTimeout) + ) + ]); + + console.log(` ā¬†ļø Uploading to Hetzner...`); + + // Retry upload with exponential backoff + let uploadSuccess = false; + for (let attempt = 1; attempt <= 3; attempt++) { + try { + await Promise.race([ + primaryS3Service.uploadBuffer( + fileName, + fileBuffer, + fileName, + metadata.contentType || 'application/octet-stream', + targetFolder + ), + new Promise((_, reject) => + setTimeout(() => reject(new Error(`Upload timeout after ${downloadTimeout/1000/60} minutes`)), downloadTimeout) + ) + ]); + uploadSuccess = true; + break; + } catch (error) { + if (attempt === 3) throw error; + const delay = Math.pow(2, attempt) * 2000; + console.log(` āš ļø Upload retry ${attempt}/3 after ${delay}ms...`); + await new Promise(resolve => setTimeout(resolve, delay)); + } + } + + if (!uploadSuccess) { + throw new Error('Upload failed after 3 attempts'); + } + + this.result.totalSize += fileSize; + this.result.successful.push(key); + console.log(` āœ… Success: ${key}`); + + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + this.result.failed.push({ key, error: errorMessage }); + console.error(` āŒ Failed: ${key} - ${errorMessage}`); + + // Force garbage collection hint for large file failures + if (global.gc) { + global.gc(); + } + } + } + + /** + * Print migration summary + */ + private printSummary() { + const duration = this.result.endTime && this.result.startTime + ? (this.result.endTime.getTime() - this.result.startTime.getTime()) / 1000 + : 0; + + console.log('\n' + '='.repeat(80)); + console.log('šŸ“Š MIGRATION SUMMARY'); + console.log('='.repeat(80)); + console.log(`Total Files Processed: ${this.result.totalFiles}`); + console.log(`āœ… Successful: ${this.result.successful.length}`); + console.log(`ā­ļø Skipped (already exist): ${this.result.skipped.length}`); + console.log(`āŒ Failed: ${this.result.failed.length}`); + console.log(`šŸ’¾ Total Data Migrated: ${this.formatBytes(this.result.totalSize)}`); + console.log(`ā±ļø Duration: ${duration.toFixed(2)} seconds`); + console.log(`šŸ“… Started: ${this.result.startTime.toISOString()}`); + console.log(`šŸ“… Ended: ${this.result.endTime?.toISOString()}`); + console.log('='.repeat(80)); + + if (this.result.failed.length > 0) { + console.log('\nāŒ FAILED FILES:'); + console.log('-'.repeat(80)); + this.result.failed.forEach(({ key, error }) => { + console.log(` ā€¢ ${key}`); + console.log(` Error: ${error}`); + }); + } + } + + /** + * Save migration results to a JSON file + */ + private saveResultsToFile() { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const fileName = `migration-report-${timestamp}.json`; + const filePath = path.join(process.cwd(), 'logs', fileName); + + // Create logs directory if it doesn't exist + const logsDir = path.join(process.cwd(), 'logs'); + if (!fs.existsSync(logsDir)) { + fs.mkdirSync(logsDir, { recursive: true }); + } + + fs.writeFileSync(filePath, JSON.stringify(this.result, null, 2)); + console.log(`\nšŸ“„ Detailed report saved to: ${filePath}`); + } + + /** + * Format bytes to human-readable format + */ + private formatBytes(bytes: number): string { + if (bytes === 0) return '0 Bytes'; + const k = 1024; + const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return Math.round((bytes / Math.pow(k, i)) * 100) / 100 + ' ' + sizes[i]; + } +} + +// Main execution +async function main() { + try { + console.log('šŸ”§ Environment Check:'); + console.log(' Secondary S3 Endpoint:', process.env.SECONDARY_S3_ENDPOINT); + console.log(' Secondary S3 Bucket:', process.env.SECONDARY_S3_BUCKET_NAME); + console.log(' Primary S3 Endpoint:', process.env.S3_ENDPOINT); + console.log(' Primary S3 Bucket:', process.env.S3_BUCKET_NAME); + + if (!process.env.SECONDARY_S3_TOKEN_ID || + !process.env.SECONDARY_S3_SECRET_KEY || + !process.env.SECONDARY_S3_ENDPOINT || + !process.env.SECONDARY_S3_BUCKET_NAME) { + throw new Error('Missing required secondary S3 environment variables'); + } + + // Set skipExisting to true to resume from where it crashed + const SKIP_EXISTING_FILES = true; + const migration = new StorageMigration(SKIP_EXISTING_FILES); + + // Migrate videos folder from secondary (R2) to primary (Hetzner) + await migration.migrateFolder(FolderNameEnum.VIDEOS, FolderNameEnum.VIDEOS); + + console.log('\nāœØ Migration completed successfully!'); + process.exit(0); + + } catch (error) { + console.error('\nšŸ’„ Migration failed:', error); + process.exit(1); + } +} + +// Run the migration +main(); + diff --git a/src/validation/upload.validation.ts b/src/validation/upload.validation.ts index 8e46fdd..cd52c0a 100644 --- a/src/validation/upload.validation.ts +++ b/src/validation/upload.validation.ts @@ -143,6 +143,33 @@ const uploadIdValidation = Joi.object({ }) }); +// Validation for getAllFiles endpoint (query parameters) +const getAllFilesValidation = Joi.object({ + folder: Joi.string() + .valid('files', 'videos', 'images', 'documents') + .optional() + .messages({ + 'any.only': 'Folder must be one of: files, videos, images, documents' + }), + maxKeys: Joi.number() + .integer() + .min(1) + .max(1000) + .default(100) + .optional() + .messages({ + 'number.base': 'maxKeys must be a number', + 'number.integer': 'maxKeys must be a whole number', + 'number.min': 'maxKeys must be at least 1', + 'number.max': 'maxKeys cannot exceed 1000' + }), + continuationToken: Joi.string() + .optional() + .messages({ + 'string.base': 'continuationToken must be a string' + }) +}); + export default { initiateUploadValidation, uploadChunkValidation, @@ -150,5 +177,6 @@ export default { abortUploadValidation, getPartsValidation, fileNameValidation, - uploadIdValidation + uploadIdValidation, + getAllFilesValidation };