diff --git a/site/app/api/admin/cleanup/route.ts b/site/app/api/admin/cleanup/route.ts index dff5676..3482df6 100644 --- a/site/app/api/admin/cleanup/route.ts +++ b/site/app/api/admin/cleanup/route.ts @@ -73,50 +73,31 @@ export async function GET() { // POST /api/admin/cleanup - Perform cleanup operations export async function POST(request: NextRequest) { try { - const body = await request.json() + const body = await request.json().catch(() => ({})) const { - cleanupFailedPosts = false, + cleanupFailedPosts = true, cleanupOldPosts = false, cleanupCompletedJobs = false, - cleanupMedia = false, + cleanupMedia = true, olderThanDays = 30 } = body + // Use the new cleanup service for automatic cleanup + const { cleanupService } = await import('@/lib/cleanup') + const cleanupResults = await cleanupService.runAllCleanup() + const results = { - failedPostsDeleted: 0, + failedPostsDeleted: cleanupResults.failedPosts, oldPostsDeleted: 0, completedJobsDeleted: 0, - mediaFilesDeleted: 0, - mediaSizeFreed: 0 + mediaFilesDeleted: cleanupResults.orphanedMedia, + mediaSizeFreed: 0, + cookieFilesDeleted: cleanupResults.cookieFiles, + tempFilesDeleted: cleanupResults.tempFiles, + stuckPostsReset: cleanupResults.stuckPosts, } - // Cleanup failed posts - if (cleanupFailedPosts) { - const failedPosts = await prisma.post.findMany({ - where: { status: 'FAILED' }, - select: { id: true, assets: { select: { pathOrKey: true } } } - }) - - for (const post of failedPosts) { - // Delete associated assets - for (const asset of post.assets) { - try { - const assetPath = path.join(process.cwd(), 'media', asset.pathOrKey) - await fs.unlink(assetPath) - results.mediaFilesDeleted++ - } catch (error) { - logger.warn(`Could not delete asset ${asset.pathOrKey}:`, error) - } - } - } - - const deleteResult = await prisma.post.deleteMany({ - where: { status: 'FAILED' } - }) - results.failedPostsDeleted = deleteResult.count - } - - // Cleanup old posts + // Additional cleanup if requested if (cleanupOldPosts) { const cutoffDate = new Date(Date.now() - olderThanDays * 24 * 60 * 60 * 1000) @@ -165,36 +146,6 @@ export async function POST(request: NextRequest) { results.completedJobsDeleted = deleteResult.count } - // Cleanup orphaned media files - if (cleanupMedia) { - try { - const mediaPath = path.join(process.cwd(), 'media') - const files = await fs.readdir(mediaPath, { withFileTypes: true }) - - // Get all asset paths from database - const assets = await prisma.asset.findMany({ - select: { pathOrKey: true } - }) - const assetPaths = new Set(assets.map(asset => asset.pathOrKey)) - - for (const file of files) { - if (file.isFile() && !assetPaths.has(file.name)) { - try { - const filePath = path.join(mediaPath, file.name) - const stats = await fs.stat(filePath) - results.mediaSizeFreed += stats.size - await fs.unlink(filePath) - results.mediaFilesDeleted++ - } catch (error) { - logger.warn(`Could not delete orphaned file ${file.name}:`, error) - } - } - } - } catch (error) { - logger.warn('Could not cleanup orphaned media files:', error) - } - } - logger.info({ results }, 'Database cleanup completed') return NextResponse.json({ diff --git a/site/app/api/health/route.ts b/site/app/api/health/route.ts new file mode 100644 index 0000000..59e1d1a --- /dev/null +++ b/site/app/api/health/route.ts @@ -0,0 +1,193 @@ +import { NextResponse } from 'next/server' +import { prisma } from '@/lib/db/prisma' +import { createLogger } from '@/lib/logger' + +const logger = createLogger('api/health') + +// Configuration +const MEMORY_WARNING_MB = parseInt(process.env.MEMORY_WARNING_MB || '600') // 75% of 800M limit +const MEMORY_CRITICAL_MB = parseInt(process.env.MEMORY_CRITICAL_MB || '750') // ~94% of 800M limit + +// Track last successful processing time +let lastProcessingTime: Date | null = null +let errorCount = 0 +const errorWindow = 60 * 60 * 1000 // 1 hour in milliseconds +const errorTimestamps: number[] = [] + +export async function GET() { + try { + const startTime = Date.now() + + // Check database connection + let databaseStatus = 'unknown' + let databaseError: string | null = null + try { + await prisma.$queryRaw`SELECT 1` + databaseStatus = 'connected' + } catch (error) { + databaseStatus = 'disconnected' + databaseError = error instanceof Error ? error.message : 'Unknown error' + logger.error('Database connection failed', error) + } + + // Get memory usage + const memoryUsage = process.memoryUsage() + const memoryUsageMB = { + rss: Math.round(memoryUsage.rss / 1024 / 1024), + heapTotal: Math.round(memoryUsage.heapTotal / 1024 / 1024), + heapUsed: Math.round(memoryUsage.heapUsed / 1024 / 1024), + external: Math.round(memoryUsage.external / 1024 / 1024), + } + + // Get background job status + let backgroundJobStatus = 'unknown' + let postCounts: any = {} + try { + const stats = await prisma.post.groupBy({ + by: ['status'], + _count: { + id: true, + }, + }) + + postCounts = stats.reduce((acc, stat) => { + acc[stat.status] = stat._count.id + return acc + }, {} as Record) + + // Get last processed post time + const lastProcessed = await prisma.post.findFirst({ + where: { + processedAt: { + not: null, + }, + }, + orderBy: { + processedAt: 'desc', + }, + select: { + processedAt: true, + }, + }) + + if (lastProcessed?.processedAt) { + lastProcessingTime = lastProcessed.processedAt + } + + backgroundJobStatus = 'running' + } catch (error) { + backgroundJobStatus = 'error' + logger.error('Failed to get background job status', error) + } + + // Clean up old error timestamps + const now = Date.now() + while (errorTimestamps.length > 0 && now - errorTimestamps[0] > errorWindow) { + errorTimestamps.shift() + } + errorCount = errorTimestamps.length + + // Calculate uptime + const uptime = process.uptime() + const uptimeFormatted = formatUptime(uptime) + + // Determine overall health status + const isHealthy = databaseStatus === 'connected' && memoryUsageMB.heapUsed < MEMORY_CRITICAL_MB + const status = isHealthy ? 'online' : 'degraded' + + const responseTime = Date.now() - startTime + + const healthData = { + status, + timestamp: new Date().toISOString(), + uptime: uptimeFormatted, + uptimeSeconds: Math.floor(uptime), + responseTimeMs: responseTime, + memory: { + usage: memoryUsageMB, + limit: { + warning: MEMORY_WARNING_MB, + critical: MEMORY_CRITICAL_MB, + }, + status: getMemoryStatus(memoryUsageMB.heapUsed), + }, + database: { + status: databaseStatus, + error: databaseError, + }, + backgroundJobs: { + status: backgroundJobStatus, + lastSuccessfulProcessing: lastProcessingTime?.toISOString() || null, + lastSuccessfulProcessingAgo: lastProcessingTime + ? formatTimeSince(lastProcessingTime) + : 'never', + postCounts, + }, + errors: { + countLastHour: errorCount, + status: errorCount < 10 ? 'low' : errorCount < 50 ? 'medium' : 'high', + }, + } + + // Return appropriate status code based on health + const statusCode = isHealthy ? 200 : 503 + + return NextResponse.json( + { + success: true, + data: healthData, + }, + { status: statusCode } + ) + } catch (error) { + logger.error('Health check failed', error) + + // Track error + errorTimestamps.push(Date.now()) + + return NextResponse.json( + { + success: false, + error: 'Health check failed', + status: 'offline', + timestamp: new Date().toISOString(), + }, + { status: 503 } + ) + } +} + +// Helper function to format uptime +function formatUptime(seconds: number): string { + const days = Math.floor(seconds / (24 * 60 * 60)) + seconds %= 24 * 60 * 60 + const hours = Math.floor(seconds / (60 * 60)) + seconds %= 60 * 60 + const minutes = Math.floor(seconds / 60) + seconds = Math.floor(seconds % 60) + + const parts = [] + if (days > 0) parts.push(`${days}d`) + if (hours > 0) parts.push(`${hours}h`) + if (minutes > 0) parts.push(`${minutes}m`) + if (seconds > 0 || parts.length === 0) parts.push(`${seconds}s`) + + return parts.join(' ') +} + +// Helper function to get memory status +function getMemoryStatus(heapUsedMB: number): 'healthy' | 'warning' | 'critical' { + if (heapUsedMB < MEMORY_WARNING_MB) return 'healthy' + if (heapUsedMB < MEMORY_CRITICAL_MB) return 'warning' + return 'critical' +} + +// Helper function to format time since +function formatTimeSince(date: Date): string { + const seconds = Math.floor((Date.now() - date.getTime()) / 1000) + + if (seconds < 60) return `${seconds}s ago` + if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago` + if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago` + return `${Math.floor(seconds / 86400)}d ago` +} diff --git a/site/app/api/posts/process/route.ts b/site/app/api/posts/process/route.ts index 05cade6..f7b85c5 100644 --- a/site/app/api/posts/process/route.ts +++ b/site/app/api/posts/process/route.ts @@ -11,6 +11,9 @@ export async function POST(request: NextRequest) { const body = await request.json() const { batchSize = 5, intervalMinutes = 30 } = body + // Limit batch size to prevent memory issues (max 10) + const safeBatchSize = Math.min(10, Math.max(1, batchSize)) + // Get NEW posts (regardless of processedAt - some might be partially processed) const newPosts = await prisma.post.findMany({ where: { @@ -19,7 +22,7 @@ export async function POST(request: NextRequest) { orderBy: { createdUtc: 'asc', }, - take: batchSize, + take: safeBatchSize, }) if (newPosts.length === 0) { diff --git a/site/app/api/posts/route.ts b/site/app/api/posts/route.ts index f8dbf16..319fc32 100644 --- a/site/app/api/posts/route.ts +++ b/site/app/api/posts/route.ts @@ -10,8 +10,10 @@ const logger = createLogger('api/posts') export async function GET(request: NextRequest) { try { const searchParams = request.nextUrl.searchParams - const page = parseInt(searchParams.get('page') || '1') - const pageSize = parseInt(searchParams.get('pageSize') || '20') + const page = Math.max(1, parseInt(searchParams.get('page') || '1')) + const requestedPageSize = parseInt(searchParams.get('pageSize') || '20') + // Limit page size to prevent performance issues (max 100) + const pageSize = Math.min(100, Math.max(1, requestedPageSize)) const status = searchParams.get('status') || 'NEW' const subreddit = searchParams.get('subreddit') const includeNsfw = searchParams.get('includeNsfw') === 'true' diff --git a/site/background-scheduler.js b/site/background-scheduler.js index 4c3c9f5..3379409 100644 --- a/site/background-scheduler.js +++ b/site/background-scheduler.js @@ -5,8 +5,31 @@ const prisma = new PrismaClient(); console.log('🚀 Starting background job scheduler...'); -// Auto-processing cron job (runs every 30 seconds to process NEW posts) -cron.schedule('*/30 * * * * *', async () => { +// Configuration +const API_BASE_URL = process.env.API_BASE_URL || 'http://localhost:3002'; + +// Auto-processing cron job (runs every 2 minutes to process NEW posts) +// Reduced from 30 seconds to prevent CPU spikes and missed executions +let isProcessing = false; +const MAX_CONCURRENT_POSTS = 5; +const PROCESSING_TIMEOUT = 5 * 60 * 1000; // 5 minutes + +cron.schedule('*/2 * * * *', async () => { + // Prevent concurrent executions + if (isProcessing) { + console.log('⏭️ Auto-processing already running, skipping...'); + return; + } + + isProcessing = true; + const startTime = Date.now(); + + // Set timeout to prevent stuck processing + const timeoutId = setTimeout(() => { + console.log('⚠️ Auto-processing timeout reached, forcing reset'); + isProcessing = false; + }, PROCESSING_TIMEOUT); + try { console.log('⏰ Auto-processing cron job running...'); @@ -26,7 +49,7 @@ cron.schedule('*/30 * * * * *', async () => { const config = { enabled: false, delaySeconds: 10, - batchSize: 1 + batchSize: 1 // Default batch size }; settings.forEach(setting => { @@ -38,7 +61,7 @@ cron.schedule('*/30 * * * * *', async () => { config.delaySeconds = parseInt(setting.value); break; case 'auto_processing_batch_size': - config.batchSize = parseInt(setting.value); + config.batchSize = Math.min(MAX_CONCURRENT_POSTS, parseInt(setting.value) || 1); break; } }); @@ -72,7 +95,7 @@ cron.schedule('*/30 * * * * *', async () => { console.log(`📝 Processing: ${post.title.substring(0, 50)}...`); // Call the process posts API endpoint - const response = await fetch(`http://localhost:3002/api/posts/process`, { + const response = await fetch(`${API_BASE_URL}/api/posts/process`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ postId: post.id }) @@ -94,8 +117,15 @@ cron.schedule('*/30 * * * * *', async () => { console.log(`❌ Error processing post: ${error.message}`); } } + + const duration = Date.now() - startTime; + console.log(`✅ Auto-processing completed in ${duration}ms`); } catch (error) { console.log(`❌ Auto-processing error: ${error.message}`); + console.error(error); + } finally { + clearTimeout(timeoutId); + isProcessing = false; } }); @@ -145,7 +175,7 @@ cron.schedule('* * * * *', async () => { console.log('🔄 Checking for posts to auto-publish...'); // Call the auto-publish API endpoint - const response = await fetch(`http://localhost:3002/api/auto-publish`, { + const response = await fetch(`${API_BASE_URL}/api/auto-publish`, { method: 'POST', headers: { 'Content-Type': 'application/json' } }); @@ -209,11 +239,35 @@ cron.schedule('*/30 * * * *', async () => { } }); +// Cleanup cron job (runs every hour) +cron.schedule('0 * * * *', async () => { + try { + console.log('⏰ Cleanup cron job running...'); + + // Call the cleanup API endpoint + const response = await fetch(`${API_BASE_URL}/api/admin/cleanup`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' } + }); + + const result = await response.json(); + + if (result.success) { + console.log('✅ Cleanup completed:', result.data); + } else { + console.log('❌ Cleanup failed:', result.error); + } + } catch (error) { + console.log(`❌ Cleanup error: ${error.message}`); + } +}); + console.log('✅ Background job scheduler started successfully!'); console.log('📊 Cron jobs scheduled:'); -console.log(' - Auto-processing: Every 30 seconds'); +console.log(' - Auto-processing: Every 2 minutes'); console.log(' - Auto-publish: Every minute'); console.log(' - Auto-fetch: Every 30 minutes'); +console.log(' - Cleanup: Every hour'); // Keep the process running process.on('SIGINT', async () => { diff --git a/site/ecosystem.config.js b/site/ecosystem.config.js index feda8db..6a28179 100644 --- a/site/ecosystem.config.js +++ b/site/ecosystem.config.js @@ -8,14 +8,26 @@ module.exports = { instances: 1, autorestart: true, watch: false, - max_memory_restart: '1G', + max_memory_restart: '800M', + max_restarts: 10, + min_uptime: '10s', + restart_delay: 4000, env: { NODE_ENV: 'production', PORT: 3002 }, error_file: '/home/ubuntu/apps/yapgrid/logs/web-error.log', out_file: '/home/ubuntu/apps/yapgrid/logs/web-out.log', - time: true + merge_logs: true, + time: true, + log_date_format: 'YYYY-MM-DD HH:mm:ss Z', + // Log rotation + log_file: '/home/ubuntu/apps/yapgrid/logs/web-combined.log', + max_size: '50M', + retain: 7, // Keep last 7 log files + compress: true, + // Performance monitoring + exp_backoff_restart_delay: 100 } ] } diff --git a/site/lib/cleanup.ts b/site/lib/cleanup.ts new file mode 100644 index 0000000..4caee42 --- /dev/null +++ b/site/lib/cleanup.ts @@ -0,0 +1,236 @@ +import { prisma } from '@/lib/db/prisma' +import { promises as fs } from 'fs' +import path from 'path' +import { createLogger } from '@/lib/logger' +import { config } from '@/lib/config' + +const logger = createLogger('cleanup') + +export class CleanupService { + private mediaDir: string + private tempDir: string + + constructor() { + this.mediaDir = config.MEDIA_DIR || './media' + this.tempDir = path.join(process.cwd(), 'temp') + } + + /** + * Remove failed download attempts older than specified hours + */ + async removeOldFailedPosts(olderThanHours = 1): Promise { + try { + const cutoffDate = new Date() + cutoffDate.setHours(cutoffDate.getHours() - olderThanHours) + + const result = await prisma.post.deleteMany({ + where: { + status: 'FAILED', + processedAt: { + lt: cutoffDate, + }, + }, + }) + + if (result.count > 0) { + logger.info('Removed old failed posts', { count: result.count, olderThanHours }) + } + + return result.count + } catch (error) { + logger.error('Failed to remove old failed posts', error) + return 0 + } + } + + /** + * Clean up temporary cookie files + */ + async cleanupTempCookieFiles(): Promise { + try { + const files = await fs.readdir(this.tempDir) + let cleanedCount = 0 + + for (const file of files) { + if (file.startsWith('cookies_') && file.endsWith('.txt')) { + const filePath = path.join(this.tempDir, file) + try { + await fs.unlink(filePath) + cleanedCount++ + logger.debug('Deleted temp cookie file', { file }) + } catch (error) { + logger.debug('Could not delete cookie file', { file }) + } + } + } + + if (cleanedCount > 0) { + logger.info('Cleaned up cookie files', { count: cleanedCount }) + } + + return cleanedCount + } catch (error) { + logger.error('Failed to cleanup cookie files', error) + return 0 + } + } + + /** + * Clean up all temporary files older than specified hours + */ + async cleanupAllTempFiles(olderThanHours = 1): Promise { + try { + const files = await fs.readdir(this.tempDir) + const now = Date.now() + const maxAge = olderThanHours * 60 * 60 * 1000 + let cleanedCount = 0 + + for (const file of files) { + const filePath = path.join(this.tempDir, file) + try { + const stats = await fs.stat(filePath) + if (now - stats.mtime.getTime() > maxAge) { + await fs.unlink(filePath) + cleanedCount++ + logger.debug('Deleted old temp file', { file }) + } + } catch (error) { + // Skip files that can't be accessed + logger.debug('Could not process temp file', { file }) + } + } + + if (cleanedCount > 0) { + logger.info('Cleaned up temp files', { count: cleanedCount }) + } + + return cleanedCount + } catch (error) { + logger.error('Failed to cleanup temp files', error) + return 0 + } + } + + /** + * Remove orphaned media files that don't have corresponding database records + */ + async removeOrphanedMediaFiles(): Promise { + try { + // Get all media files + const files = await fs.readdir(this.mediaDir) + let orphanedCount = 0 + + for (const file of files) { + // Extract post ID from filename (format: {postId}.{ext}) + const postIdMatch = file.match(/^([^.]+)\./) + if (!postIdMatch) continue + + const postId = postIdMatch[1] + + // Check if asset exists in database + const asset = await prisma.asset.findFirst({ + where: { + pathOrKey: { + contains: file, + }, + }, + }) + + if (!asset) { + const filePath = path.join(this.mediaDir, file) + try { + await fs.unlink(filePath) + orphanedCount++ + logger.debug('Deleted orphaned media file', { file }) + } catch (error) { + logger.debug('Could not delete orphaned file', { file }) + } + } + } + + if (orphanedCount > 0) { + logger.info('Removed orphaned media files', { count: orphanedCount }) + } + + return orphanedCount + } catch (error) { + logger.error('Failed to remove orphaned media files', error) + return 0 + } + } + + /** + * Reset stuck posts that have been in "processing" state for too long + */ + async resetStuckPosts(stuckThresholdMinutes = 30): Promise { + try { + const cutoffDate = new Date() + cutoffDate.setMinutes(cutoffDate.getMinutes() - stuckThresholdMinutes) + + // Find posts that are stuck in DOWNLOADING state + const stuckPosts = await prisma.post.findMany({ + where: { + status: 'DOWNLOADING', + processedAt: { + lt: cutoffDate, + }, + }, + }) + + if (stuckPosts.length === 0) { + return 0 + } + + // Reset them to NEW status so they can be retried + const result = await prisma.post.updateMany({ + where: { + id: { + in: stuckPosts.map((p) => p.id), + }, + }, + data: { + status: 'NEW', + processedAt: null, + error: 'Reset from stuck DOWNLOADING state', + }, + }) + + if (result.count > 0) { + logger.info('Reset stuck posts', { count: result.count, stuckThresholdMinutes }) + } + + return result.count + } catch (error) { + logger.error('Failed to reset stuck posts', error) + return 0 + } + } + + /** + * Run all cleanup operations + */ + async runAllCleanup(): Promise<{ + failedPosts: number + cookieFiles: number + tempFiles: number + orphanedMedia: number + stuckPosts: number + }> { + logger.info('Running all cleanup operations...') + + const results = { + failedPosts: await this.removeOldFailedPosts(1), + cookieFiles: await this.cleanupTempCookieFiles(), + tempFiles: await this.cleanupAllTempFiles(1), + orphanedMedia: await this.removeOrphanedMediaFiles(), + stuckPosts: await this.resetStuckPosts(30), + } + + logger.info('Cleanup operations completed', results) + + return results + } +} + +// Singleton instance +export const cleanupService = new CleanupService() diff --git a/site/lib/media/downloader.ts b/site/lib/media/downloader.ts new file mode 100644 index 0000000..78d862d --- /dev/null +++ b/site/lib/media/downloader.ts @@ -0,0 +1,380 @@ +import { spawn } from 'child_process' +import { promises as fs } from 'fs' +import path from 'path' +import { createLogger } from '@/lib/logger' +import { config } from '@/lib/config' + +const logger = createLogger('media-downloader') + +export interface MediaInfo { + filename: string + url: string + type: 'video' | 'image' | 'gif' + width?: number + height?: number + duration?: number + size?: number +} + +export class MediaDownloader { + private tempDir: string + private mediaDir: string + private retryDelays = [2000, 5000, 10000] // 2s, 5s, 10s + + constructor() { + this.tempDir = path.join(process.cwd(), 'temp') + this.mediaDir = config.MEDIA_DIR || './media' + this.ensureDirectories() + } + + private async ensureDirectories() { + try { + await fs.mkdir(this.tempDir, { recursive: true }) + await fs.mkdir(this.mediaDir, { recursive: true }) + } catch (error) { + logger.error('Failed to create directories', error) + } + } + + /** + * Download media from post URL with retry logic + */ + async downloadMedia(post: any, postId: string): Promise { + const url = post.url || post.permalink + + // Check if this is a text-only post + if (this.isTextOnlyPost(post)) { + logger.debug('Skipping text-only post', { postId }) + return null + } + + // Try downloading with retries + for (let attempt = 0; attempt < this.retryDelays.length; attempt++) { + try { + if (attempt > 0) { + logger.info(`Retry attempt ${attempt + 1} for post`, { postId }) + await this.delay(this.retryDelays[attempt - 1]) + } + + // Handle different media types + if (url.includes('v.redd.it') || url.includes('/comments/')) { + return await this.downloadRedditVideo(url, postId) + } else if (url.includes('i.redd.it')) { + return await this.downloadRedditImage(url, postId) + } else if (url.includes('/gallery/')) { + return await this.downloadRedditGallery(url, postId) + } else { + // Try generic download + return await this.downloadGenericMedia(url, postId) + } + } catch (error) { + const isLastAttempt = attempt === this.retryDelays.length - 1 + + if (this.is403Error(error)) { + logger.debug('HTTP 403 error - Reddit blocked request', { postId, url, attempt }) + } else if (this.isJsonParseError(error)) { + logger.debug('Failed to parse JSON from yt-dlp', { postId, url, attempt }) + } else { + logger.warn('Download attempt failed', { postId, url, attempt, error: error instanceof Error ? error.message : 'Unknown error' }) + } + + if (isLastAttempt) { + logger.error('All download attempts failed', { postId, url, totalAttempts: this.retryDelays.length }) + return null + } + } + } + + return null + } + + /** + * Download Reddit video using yt-dlp with better configuration + */ + private async downloadRedditVideo(url: string, postId: string): Promise { + const outputFilename = `${postId}.mp4` + const tempPath = path.join(this.tempDir, outputFilename) + const finalPath = path.join(this.mediaDir, outputFilename) + + // Use the URL as-is - yt-dlp handles v.redd.it URLs with proper extractor args + const downloadUrl = url + + const cookieFile = path.join(this.tempDir, `cookies_${postId}.txt`) + let cookieFileCreated = false + + try { + // Create empty cookie file if needed + if (config.REDDIT_SESSION_COOKIE) { + await fs.writeFile(cookieFile, config.REDDIT_SESSION_COOKIE) + cookieFileCreated = true + } + + const ytdlpArgs = [ + downloadUrl, + '-o', tempPath, + '--format', 'bestvideo[height<=720]+bestaudio/best[height<=720]/best', + '--merge-output-format', 'mp4', + '--no-playlist', + '--no-warnings', + '--quiet', + '--print-json', + '--user-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + '--extractor-args', 'reddit:player_only=False', + '--max-filesize', `${config.MAX_FILESIZE_MB}M`, + ] + + // Add cookie file if it was created + if (cookieFileCreated) { + ytdlpArgs.push('--cookies', cookieFile) + } + + logger.info('Starting video download with yt-dlp', { postId, url: downloadUrl }) + + const result = await this.executeCommand('yt-dlp', ytdlpArgs) + const metadata = JSON.parse(result) + + // Check duration limit + if (metadata.duration && metadata.duration > config.MAX_DURATION_SECONDS) { + throw new Error(`Video duration (${metadata.duration}s) exceeds maximum (${config.MAX_DURATION_SECONDS}s)`) + } + + // Move to final location + await fs.rename(tempPath, finalPath) + + // Get file stats + const stats = await fs.stat(finalPath) + + logger.info('Video downloaded successfully', { postId, filename: outputFilename, size: stats.size }) + + return { + filename: outputFilename, + url: `/api/media/${encodeURIComponent(outputFilename)}`, + type: 'video', + width: metadata.width || 1920, + height: metadata.height || 1080, + duration: metadata.duration || 0, + size: stats.size, + } + } catch (error) { + // Don't attempt fallback for 403 errors (blocked by Reddit) + // For other errors, the retry logic in downloadMedia will handle it + throw error + } finally { + // Clean up cookie file + if (cookieFileCreated) { + try { + await fs.unlink(cookieFile) + } catch (error) { + logger.debug('Failed to clean up cookie file', { cookieFile }) + } + } + + // Clean up temp file on error + try { + await fs.access(tempPath) + await fs.unlink(tempPath) + } catch { + // File doesn't exist, that's fine + } + } + } + + /** + * Download Reddit image + */ + private async downloadRedditImage(url: string, postId: string): Promise { + const ext = path.extname(url) || '.jpg' + const outputFilename = `${postId}${ext}` + const finalPath = path.join(this.mediaDir, outputFilename) + + try { + // Use yt-dlp for image download as well + const ytdlpArgs = [ + url, + '-o', finalPath, + '--no-playlist', + '--quiet', + ] + + await this.executeCommand('yt-dlp', ytdlpArgs) + + const stats = await fs.stat(finalPath) + + logger.info('Image downloaded successfully', { postId, filename: outputFilename }) + + return { + filename: outputFilename, + url: `/api/media/${encodeURIComponent(outputFilename)}`, + type: 'image', + size: stats.size, + } + } catch (error) { + logger.warn('Failed to download image', { postId, url, error: error instanceof Error ? error.message : 'Unknown' }) + throw error + } + } + + /** + * Download Reddit gallery (first image only) + * Gallery posts are not currently supported and will be skipped + */ + private async downloadRedditGallery(url: string, postId: string): Promise { + logger.info('Gallery posts are not currently supported - this post will be skipped', { postId, url }) + // Galleries are skipped gracefully - they won't be marked as failed + throw new Error('Gallery posts are not currently supported - this post will be skipped') + } + + /** + * Download generic media + */ + private async downloadGenericMedia(url: string, postId: string): Promise { + logger.debug('Generic media download', { postId, url }) + + try { + const ext = path.extname(url) || '.mp4' + const outputFilename = `${postId}${ext}` + const finalPath = path.join(this.mediaDir, outputFilename) + + const ytdlpArgs = [ + url, + '-o', finalPath, + '--no-playlist', + '--quiet', + '--print-json', + ] + + const result = await this.executeCommand('yt-dlp', ytdlpArgs) + const metadata = JSON.parse(result) + + const stats = await fs.stat(finalPath) + + // Detect media type based on file extension + const videoExts = ['.mp4', '.webm', '.mov', '.avi', '.mkv', '.flv', '.wmv'] + const imageExts = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp'] + const extLower = ext.toLowerCase() + + const mediaType = videoExts.includes(extLower) ? 'video' : imageExts.includes(extLower) ? 'image' : 'video' + + logger.info('Generic media downloaded', { postId, filename: outputFilename, type: mediaType }) + + return { + filename: outputFilename, + url: `/api/media/${encodeURIComponent(outputFilename)}`, + type: mediaType, + width: metadata.width, + height: metadata.height, + duration: metadata.duration, + size: stats.size, + } + } catch (error) { + logger.debug('Generic media download failed', { postId, url }) + throw error + } + } + + /** + * Check if post is text-only + */ + private isTextOnlyPost(post: any): boolean { + const url = post.url || '' + return ( + url.includes('/comments/') && + !url.includes('i.redd.it') && + !url.includes('v.redd.it') && + !url.includes('/gallery/') && + !post.preview + ) + } + + /** + * Execute command with promise wrapper + */ + private executeCommand(command: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + const proc = spawn(command, args) + let stdout = '' + let stderr = '' + + proc.stdout.on('data', (data) => { + stdout += data.toString() + }) + + proc.stderr.on('data', (data) => { + stderr += data.toString() + }) + + proc.on('close', (code) => { + if (code === 0) { + resolve(stdout.trim()) + } else { + const error = new Error(`${command} exited with code ${code}: ${stderr}`) + ;(error as any).stderr = stderr + ;(error as any).code = code + reject(error) + } + }) + + proc.on('error', (error) => { + reject(error) + }) + }) + } + + /** + * Check if error is a 403 error + */ + private is403Error(error: any): boolean { + if (!error) return false + const errorStr = error.message || error.stderr || error.toString() + return errorStr.includes('HTTP Error 403') || errorStr.includes('Blocked') + } + + /** + * Check if error is a JSON parse error + */ + private isJsonParseError(error: any): boolean { + if (!error) return false + const errorStr = error.message || error.toString() + return errorStr.includes('Failed to parse JSON') || errorStr.includes('Unexpected token') + } + + /** + * Delay helper + */ + private delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) + } + + /** + * Clean up old temp files + */ + async cleanupTempFiles(olderThanHours = 1): Promise { + try { + const files = await fs.readdir(this.tempDir) + const now = Date.now() + const maxAge = olderThanHours * 60 * 60 * 1000 + + let cleanedCount = 0 + for (const file of files) { + const filePath = path.join(this.tempDir, file) + try { + const stats = await fs.stat(filePath) + if (now - stats.mtime.getTime() > maxAge) { + await fs.unlink(filePath) + cleanedCount++ + logger.debug('Deleted old temp file', { file }) + } + } catch (error) { + // Skip files that can't be accessed + logger.debug('Could not process temp file', { file }) + } + } + + if (cleanedCount > 0) { + logger.info('Cleaned up temp files', { count: cleanedCount }) + } + } catch (error) { + logger.error('Failed to cleanup temp files', error) + } + } +}