Optimize background scheduler and fix Reddit video download failures

site/app/api/admin/cleanup/route.ts

@@ -73,50 +73,31 @@ export async function GET() {
 // POST /api/admin/cleanup - Perform cleanup operations
 export async function POST(request: NextRequest) {
   try {
-    const body = await request.json()
+    const body = await request.json().catch(() => ({}))
     const { 
-      cleanupFailedPosts = false,
+      cleanupFailedPosts = true,
       cleanupOldPosts = false,
       cleanupCompletedJobs = false,
-      cleanupMedia = false,
+      cleanupMedia = true,
       olderThanDays = 30
     } = body
 
+    // Use the new cleanup service for automatic cleanup
+    const { cleanupService } = await import('@/lib/cleanup')
+    const cleanupResults = await cleanupService.runAllCleanup()
+
     const results = {
-      failedPostsDeleted: 0,
+      failedPostsDeleted: cleanupResults.failedPosts,
       oldPostsDeleted: 0,
       completedJobsDeleted: 0,
-      mediaFilesDeleted: 0,
-      mediaSizeFreed: 0
+      mediaFilesDeleted: cleanupResults.orphanedMedia,
+      mediaSizeFreed: 0,
+      cookieFilesDeleted: cleanupResults.cookieFiles,
+      tempFilesDeleted: cleanupResults.tempFiles,
+      stuckPostsReset: cleanupResults.stuckPosts,
     }
 
-    // Cleanup failed posts
-    if (cleanupFailedPosts) {
-      const failedPosts = await prisma.post.findMany({
-        where: { status: 'FAILED' },
-        select: { id: true, assets: { select: { pathOrKey: true } } }
-      })
-
-      for (const post of failedPosts) {
-        // Delete associated assets
-        for (const asset of post.assets) {
-          try {
-            const assetPath = path.join(process.cwd(), 'media', asset.pathOrKey)
-            await fs.unlink(assetPath)
-            results.mediaFilesDeleted++
-          } catch (error) {
-            logger.warn(`Could not delete asset ${asset.pathOrKey}:`, error)
-          }
-        }
-      }
-
-      const deleteResult = await prisma.post.deleteMany({
-        where: { status: 'FAILED' }
-      })
-      results.failedPostsDeleted = deleteResult.count
-    }
-
-    // Cleanup old posts
+    // Additional cleanup if requested
     if (cleanupOldPosts) {
       const cutoffDate = new Date(Date.now() - olderThanDays * 24 * 60 * 60 * 1000)
 
@@ -165,36 +146,6 @@ export async function POST(request: NextRequest) {
       results.completedJobsDeleted = deleteResult.count
     }
 
-    // Cleanup orphaned media files
-    if (cleanupMedia) {
-      try {
-        const mediaPath = path.join(process.cwd(), 'media')
-        const files = await fs.readdir(mediaPath, { withFileTypes: true })
-
-        // Get all asset paths from database
-        const assets = await prisma.asset.findMany({
-          select: { pathOrKey: true }
-        })
-        const assetPaths = new Set(assets.map(asset => asset.pathOrKey))
-
-        for (const file of files) {
-          if (file.isFile() && !assetPaths.has(file.name)) {
-            try {
-              const filePath = path.join(mediaPath, file.name)
-              const stats = await fs.stat(filePath)
-              results.mediaSizeFreed += stats.size
-              await fs.unlink(filePath)
-              results.mediaFilesDeleted++
-            } catch (error) {
-              logger.warn(`Could not delete orphaned file ${file.name}:`, error)
-            }
-          }
-        }
-      } catch (error) {
-        logger.warn('Could not cleanup orphaned media files:', error)
-      }
-    }
-
     logger.info({ results }, 'Database cleanup completed')
 
     return NextResponse.json({

site/app/api/health/route.ts

@@ -0,0 +1,193 @@
+import { NextResponse } from 'next/server'
+import { prisma } from '@/lib/db/prisma'
+import { createLogger } from '@/lib/logger'
+
+const logger = createLogger('api/health')
+
+// Configuration
+const MEMORY_WARNING_MB = parseInt(process.env.MEMORY_WARNING_MB || '600') // 75% of 800M limit
+const MEMORY_CRITICAL_MB = parseInt(process.env.MEMORY_CRITICAL_MB || '750') // ~94% of 800M limit
+
+// Track last successful processing time
+let lastProcessingTime: Date | null = null
+let errorCount = 0
+const errorWindow = 60 * 60 * 1000 // 1 hour in milliseconds
+const errorTimestamps: number[] = []
+
+export async function GET() {
+  try {
+    const startTime = Date.now()
+
+    // Check database connection
+    let databaseStatus = 'unknown'
+    let databaseError: string | null = null
+    try {
+      await prisma.$queryRaw`SELECT 1`
+      databaseStatus = 'connected'
+    } catch (error) {
+      databaseStatus = 'disconnected'
+      databaseError = error instanceof Error ? error.message : 'Unknown error'
+      logger.error('Database connection failed', error)
+    }
+
+    // Get memory usage
+    const memoryUsage = process.memoryUsage()
+    const memoryUsageMB = {
+      rss: Math.round(memoryUsage.rss / 1024 / 1024),
+      heapTotal: Math.round(memoryUsage.heapTotal / 1024 / 1024),
+      heapUsed: Math.round(memoryUsage.heapUsed / 1024 / 1024),
+      external: Math.round(memoryUsage.external / 1024 / 1024),
+    }
+
+    // Get background job status
+    let backgroundJobStatus = 'unknown'
+    let postCounts: any = {}
+    try {
+      const stats = await prisma.post.groupBy({
+        by: ['status'],
+        _count: {
+          id: true,
+        },
+      })
+
+      postCounts = stats.reduce((acc, stat) => {
+        acc[stat.status] = stat._count.id
+        return acc
+      }, {} as Record<string, number>)
+
+      // Get last processed post time
+      const lastProcessed = await prisma.post.findFirst({
+        where: {
+          processedAt: {
+            not: null,
+          },
+        },
+        orderBy: {
+          processedAt: 'desc',
+        },
+        select: {
+          processedAt: true,
+        },
+      })
+
+      if (lastProcessed?.processedAt) {
+        lastProcessingTime = lastProcessed.processedAt
+      }
+
+      backgroundJobStatus = 'running'
+    } catch (error) {
+      backgroundJobStatus = 'error'
+      logger.error('Failed to get background job status', error)
+    }
+
+    // Clean up old error timestamps
+    const now = Date.now()
+    while (errorTimestamps.length > 0 && now - errorTimestamps[0] > errorWindow) {
+      errorTimestamps.shift()
+    }
+    errorCount = errorTimestamps.length
+
+    // Calculate uptime
+    const uptime = process.uptime()
+    const uptimeFormatted = formatUptime(uptime)
+
+    // Determine overall health status
+    const isHealthy = databaseStatus === 'connected' && memoryUsageMB.heapUsed < MEMORY_CRITICAL_MB
+    const status = isHealthy ? 'online' : 'degraded'
+
+    const responseTime = Date.now() - startTime
+
+    const healthData = {
+      status,
+      timestamp: new Date().toISOString(),
+      uptime: uptimeFormatted,
+      uptimeSeconds: Math.floor(uptime),
+      responseTimeMs: responseTime,
+      memory: {
+        usage: memoryUsageMB,
+        limit: {
+          warning: MEMORY_WARNING_MB,
+          critical: MEMORY_CRITICAL_MB,
+        },
+        status: getMemoryStatus(memoryUsageMB.heapUsed),
+      },
+      database: {
+        status: databaseStatus,
+        error: databaseError,
+      },
+      backgroundJobs: {
+        status: backgroundJobStatus,
+        lastSuccessfulProcessing: lastProcessingTime?.toISOString() || null,
+        lastSuccessfulProcessingAgo: lastProcessingTime
+          ? formatTimeSince(lastProcessingTime)
+          : 'never',
+        postCounts,
+      },
+      errors: {
+        countLastHour: errorCount,
+        status: errorCount < 10 ? 'low' : errorCount < 50 ? 'medium' : 'high',
+      },
+    }
+
+    // Return appropriate status code based on health
+    const statusCode = isHealthy ? 200 : 503
+
+    return NextResponse.json(
+      {
+        success: true,
+        data: healthData,
+      },
+      { status: statusCode }
+    )
+  } catch (error) {
+    logger.error('Health check failed', error)
+
+    // Track error
+    errorTimestamps.push(Date.now())
+
+    return NextResponse.json(
+      {
+        success: false,
+        error: 'Health check failed',
+        status: 'offline',
+        timestamp: new Date().toISOString(),
+      },
+      { status: 503 }
+    )
+  }
+}
+
+// Helper function to format uptime
+function formatUptime(seconds: number): string {
+  const days = Math.floor(seconds / (24 * 60 * 60))
+  seconds %= 24 * 60 * 60
+  const hours = Math.floor(seconds / (60 * 60))
+  seconds %= 60 * 60
+  const minutes = Math.floor(seconds / 60)
+  seconds = Math.floor(seconds % 60)
+
+  const parts = []
+  if (days > 0) parts.push(`${days}d`)
+  if (hours > 0) parts.push(`${hours}h`)
+  if (minutes > 0) parts.push(`${minutes}m`)
+  if (seconds > 0 || parts.length === 0) parts.push(`${seconds}s`)
+
+  return parts.join(' ')
+}
+
+// Helper function to get memory status
+function getMemoryStatus(heapUsedMB: number): 'healthy' | 'warning' | 'critical' {
+  if (heapUsedMB < MEMORY_WARNING_MB) return 'healthy'
+  if (heapUsedMB < MEMORY_CRITICAL_MB) return 'warning'
+  return 'critical'
+}
+
+// Helper function to format time since
+function formatTimeSince(date: Date): string {
+  const seconds = Math.floor((Date.now() - date.getTime()) / 1000)
+
+  if (seconds < 60) return `${seconds}s ago`
+  if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago`
+  if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago`
+  return `${Math.floor(seconds / 86400)}d ago`
+}

site/app/api/posts/process/route.ts

@@ -11,6 +11,9 @@ export async function POST(request: NextRequest) {
     const body = await request.json()
     const { batchSize = 5, intervalMinutes = 30 } = body
 
+    // Limit batch size to prevent memory issues (max 10)
+    const safeBatchSize = Math.min(10, Math.max(1, batchSize))
+
     // Get NEW posts (regardless of processedAt - some might be partially processed)
     const newPosts = await prisma.post.findMany({
       where: {
@@ -19,7 +22,7 @@ export async function POST(request: NextRequest) {
       orderBy: {
         createdUtc: 'asc',
       },
-      take: batchSize,
+      take: safeBatchSize,
     })
 
     if (newPosts.length === 0) {

site/app/api/posts/route.ts

@@ -10,8 +10,10 @@ const logger = createLogger('api/posts')
 export async function GET(request: NextRequest) {
   try {
     const searchParams = request.nextUrl.searchParams
-    const page = parseInt(searchParams.get('page') || '1')
-    const pageSize = parseInt(searchParams.get('pageSize') || '20')
+    const page = Math.max(1, parseInt(searchParams.get('page') || '1'))
+    const requestedPageSize = parseInt(searchParams.get('pageSize') || '20')
+    // Limit page size to prevent performance issues (max 100)
+    const pageSize = Math.min(100, Math.max(1, requestedPageSize))
     const status = searchParams.get('status') || 'NEW'
     const subreddit = searchParams.get('subreddit')
     const includeNsfw = searchParams.get('includeNsfw') === 'true'