Configure background scheduler in PM2 and optimize cron frequencies

SCHEDULER_SETUP.md

@@ -0,0 +1,102 @@
+# Background Scheduler Setup
+
+## Overview
+The background scheduler runs three cron jobs:
+- **Auto-processing**: Processes NEW posts to READY status (every 2 minutes)
+- **Auto-publish**: Publishes READY posts to homepage (every 1 minute)
+- **Auto-fetch**: Fetches new posts from Reddit campaigns (every 30 minutes)
+
+## Initial Setup
+
+### 1. Initialize Database Settings
+Run this script once to create the required settings in the database:
+
+```bash
+cd /home/ubuntu/apps/yapgrid/site
+node scripts/init-job-settings.js
+```
+
+### 2. Deploy with PM2
+The `ecosystem.config.js` now includes both the Next.js app and the background scheduler.
+
+```bash
+cd /home/ubuntu/apps/yapgrid/site
+pm2 delete all
+pm2 start ecosystem.config.js
+pm2 save
+```
+
+### 3. Verify Jobs are Running
+Check PM2 status:
+
+```bash
+pm2 status
+```
+
+You should see:
+- yapgrid-nextjs (online)
+- background-scheduler (online)
+
+Check logs:
+
+```bash
+pm2 logs background-scheduler
+```
+
+## Job Control
+
+All jobs are controlled via the admin panel at `https://yapgrid.com/admin/jobs`
+
+### Enable/Disable Jobs
+Jobs are enabled by default. To disable a job:
+
+1. Go to admin panel
+2. Click "Stop" on the desired job
+3. This updates the database setting (e.g., `auto_processing_enabled = 'false'`)
+
+### Manual Trigger
+You can manually trigger any job from the admin panel by clicking the "Start" button when the job is stopped, or by using the API endpoints.
+
+## Troubleshooting
+
+### Jobs Not Running
+Check if settings are enabled:
+
+```bash
+cd /home/ubuntu/apps/yapgrid/site
+node -e "const {PrismaClient}=require('@prisma/client');const p=new PrismaClient();p.setting.findMany({where:{key:{in:['auto_processing_enabled','auto_posting_enabled','auto_ingest_enabled']}}}).then(console.log).finally(()=>p.\$disconnect())"
+```
+
+### Scheduler Not Starting
+Check logs:
+
+```bash
+pm2 logs background-scheduler --lines 100
+```
+
+Restart scheduler:
+
+```bash
+pm2 restart background-scheduler
+```
+
+### High CPU Usage
+If auto-processing is causing high CPU usage, reduce the batch size:
+
+```sql
+UPDATE Setting SET value = '5' WHERE key = 'auto_processing_batch_size';
+```
+
+## Configuration
+
+Job settings are stored in the `Setting` table:
+
+| Key | Default | Description |
+|-----|---------|-------------|
+| auto_processing_enabled | true | Enable/disable auto-processing |
+| auto_processing_delay_seconds | 10 | Delay between processing posts |
+| auto_processing_batch_size | 10 | Number of posts to process per run |
+| auto_posting_enabled | true | Enable/disable auto-publishing |
+| auto_posting_interval_minutes | 30 | Interval for publishing posts |
+| auto_posting_batch_size | 5 | Number of posts to publish per run |
+| auto_ingest_enabled | true | Enable/disable auto-fetching |

site/background-scheduler.js

@@ -5,8 +5,8 @@ const prisma = new PrismaClient();
 
 console.log('🚀 Starting background job scheduler...');
 
-// Auto-processing cron job (runs every 30 seconds to process NEW posts)
-cron.schedule('*/30 * * * * *', async () => {
+// Auto-processing cron job (runs every 2 minutes to process NEW posts)
+cron.schedule('*/2 * * * *', async () => {
   try {
     console.log('⏰ Auto-processing cron job running...');
 
@@ -211,7 +211,7 @@ cron.schedule('*/30 * * * *', async () => {
 
 console.log('✅ Background job scheduler started successfully!');
 console.log('📊 Cron jobs scheduled:');
-console.log('  - Auto-processing: Every 30 seconds');
+console.log('  - Auto-processing: Every 2 minutes');
 console.log('  - Auto-publish: Every minute');
 console.log('  - Auto-fetch: Every 30 minutes');
 

site/ecosystem.config.js

@@ -16,6 +16,22 @@ module.exports = {
       error_file: '/home/ubuntu/apps/yapgrid/logs/web-error.log',
       out_file: '/home/ubuntu/apps/yapgrid/logs/web-out.log',
       time: true
+    },
+    {
+      name: 'background-scheduler',
+      script: 'background-scheduler.js',
+      cwd: '/home/ubuntu/apps/yapgrid/site',
+      instances: 1,
+      autorestart: true,
+      watch: false,
+      max_memory_restart: '500M',
+      env: {
+        NODE_ENV: 'production',
+        PORT: 3002
+      },
+      error_file: '/home/ubuntu/apps/yapgrid/logs/scheduler-error.log',
+      out_file: '/home/ubuntu/apps/yapgrid/logs/scheduler-out.log',
+      time: true
     }
   ]
 }

site/scripts/init-job-settings.js

@@ -0,0 +1,84 @@
+const { PrismaClient } = require('@prisma/client');
+const prisma = new PrismaClient();
+
+async function initializeJobSettings() {
+  console.log('🔧 Initializing job settings...');
+
+  try {
+    // Auto-processing settings
+    await prisma.setting.upsert({
+      where: { key: 'auto_processing_enabled' },
+      update: {},
+      create: { key: 'auto_processing_enabled', value: 'true' }
+    });
+
+    await prisma.setting.upsert({
+      where: { key: 'auto_processing_delay_seconds' },
+      update: {},
+      create: { key: 'auto_processing_delay_seconds', value: '10' }
+    });
+
+    await prisma.setting.upsert({
+      where: { key: 'auto_processing_batch_size' },
+      update: {},
+      create: { key: 'auto_processing_batch_size', value: '10' }
+    });
+
+    // Auto-posting settings
+    await prisma.setting.upsert({
+      where: { key: 'auto_posting_enabled' },
+      update: {},
+      create: { key: 'auto_posting_enabled', value: 'true' }
+    });
+
+    await prisma.setting.upsert({
+      where: { key: 'auto_posting_interval_minutes' },
+      update: {},
+      create: { key: 'auto_posting_interval_minutes', value: '30' }
+    });
+
+    await prisma.setting.upsert({
+      where: { key: 'auto_posting_batch_size' },
+      update: {},
+      create: { key: 'auto_posting_batch_size', value: '5' }
+    });
+
+    // Auto-ingest settings
+    await prisma.setting.upsert({
+      where: { key: 'auto_ingest_enabled' },
+      update: {},
+      create: { key: 'auto_ingest_enabled', value: 'true' }
+    });
+
+    console.log('✅ Job settings initialized successfully!');
+
+    // Display current settings
+    const allSettings = await prisma.setting.findMany({
+      where: {
+        key: {
+          in: [
+            'auto_processing_enabled',
+            'auto_processing_delay_seconds',
+            'auto_processing_batch_size',
+            'auto_posting_enabled',
+            'auto_posting_interval_minutes',
+            'auto_posting_batch_size',
+            'auto_ingest_enabled'
+          ]
+        }
+      }
+    });
+
+    console.log('\n📊 Current Settings:');
+    allSettings.forEach(setting => {
+      console.log(`  ${setting.key}: ${setting.value}`);
+    });
+
+  } catch (error) {
+    console.error('❌ Error initializing settings:', error);
+  } finally {
+    await prisma.$disconnect();
+  }
+}
+
+initializeJobSettings();