Merge dev to main

.env.example

@@ -79,3 +79,7 @@ VITE_REVERB_APP_KEY="${REVERB_APP_KEY}"
 VITE_REVERB_HOST=localhost
 VITE_REVERB_PORT="${REVERB_PORT}"
 VITE_REVERB_SCHEME="${REVERB_SCHEME}"
+
+ANONYMIZATION_UPLOAD_RETENTION_DAYS=30
+ANONYMIZATION_STAGING_RETENTION_DAYS=7
+ANONYMIZATION_PACKAGE_OWNER=ANON_DATA

.github/workflows/build-and-deploy.yml

@@ -150,3 +150,5 @@ jobs:
         run: |
           oc rollout restart deployment/klamm-app
           oc rollout restart deployment/klamm-queue-worker
+          oc rollout restart deployment/klamm-queue-anonymization-worker
+          oc rollout restart deployment/klamm-reverb-worker

Dockerfile

@@ -16,7 +16,9 @@ RUN apt-get update && apt-get install -y \
     postgresql-client \
     nodejs \
     npm \
-    && docker-php-ext-install pdo_mysql pdo_pgsql pgsql mbstring exif pcntl bcmath gd intl zip \
+    && docker-php-ext-install pdo_mysql pdo_pgsql pgsql mbstring exif pcntl bcmath gd intl zip opcache \
+    && pecl install redis \
+    && docker-php-ext-enable redis opcache \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # Enable Apache mod_rewrite
@@ -70,6 +72,9 @@ COPY 000-default.conf /etc/apache2/sites-available/000-default.conf
 # Copy PHP upload config
 COPY docker/php/uploads.ini /usr/local/etc/php/conf.d/uploads.ini
 
+# Copy PHP OPcache config for production performance
+COPY docker/php/opcache.ini /usr/local/etc/php/conf.d/opcache.ini
+
 # Generate APP_KEY
 RUN echo "APP_KEY=" > .env
 RUN php artisan key:generate

Dockerfile.worker

@@ -16,9 +16,9 @@ RUN apt-get update && apt-get install -y \
     postgresql-client \
     nodejs \
     npm \
-    && docker-php-ext-install pdo_mysql pdo_pgsql pgsql mbstring exif pcntl bcmath gd intl zip \
+    && docker-php-ext-install pdo_mysql pdo_pgsql pgsql mbstring exif pcntl bcmath gd intl zip opcache \
     && pecl install redis \
-    && docker-php-ext-enable redis \
+    && docker-php-ext-enable redis opcache \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # Set working directory
@@ -30,6 +30,9 @@ COPY --from=composer:latest /usr/bin/composer /usr/bin/composer
 # Copy existing application directory contents
 COPY . /var/www
 
+# Copy PHP OPcache config for production performance
+COPY docker/php/opcache.ini /usr/local/etc/php/conf.d/opcache.ini
+
 # Install Node.js dependencies if package.json exists
 RUN if [ -f package.json ]; then npm install; fi
 

app/Console/Commands/PurgeAnonymizationUploads.php

@@ -5,20 +5,26 @@
 use App\Models\Anonymizer\AnonymousUpload;
 use Carbon\CarbonImmutable;
 use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Storage;
 use Throwable;
 
 class PurgeAnonymizationUploads extends Command
 {
-    protected $signature = 'anonymization:purge-uploads {--dry-run : Report what would be deleted without deleting files} {--limit=500 : Max records to process per run}';
+    protected $signature = 'anonymization:purge-uploads {--dry-run : Report what would be deleted without deleting files} {--limit=500 : Max records to process per run} {--staging-limit=1000 : Max uploads to scan for staging pruning per run} {--staging-upload-chunk=200 : Upload IDs per staging delete pass} {--row-chunk=5000 : Staging row IDs deleted per statement}';
 
-    protected $description = 'Deletes stored anonymization CSV uploads after their retention period has elapsed.';
+    protected $description = 'Deletes retained anonymization upload files and prunes stale staging rows.';
 
     public function handle(): int
     {
         $now = CarbonImmutable::now();
         $dryRun = (bool) $this->option('dry-run');
         $limit = (int) $this->option('limit');
+        $stagingLimit = max(1, (int) $this->option('staging-limit'));
+        $stagingUploadChunk = max(1, (int) $this->option('staging-upload-chunk'));
+        $rowChunk = max(100, (int) $this->option('row-chunk'));
+        $stagingRetentionDays = max(1, (int) config('anonymizer.staging_retention_days', 7));
+        $stagingCutoff = $now->subDays($stagingRetentionDays);
 
         $query = AnonymousUpload::query()
             ->whereNotNull('path')
@@ -30,57 +36,106 @@ public function handle(): int
 
         $uploads = $query->get();
 
-        if ($uploads->isEmpty()) {
-            $this->info('No expired uploads to purge.');
-            return self::SUCCESS;
-        }
-
         $purged = 0;
         $missing = 0;
         $failed = 0;
 
-        foreach ($uploads as $upload) {
-            $disk = $upload->file_disk ?: config('filesystems.default', 'local');
-            $path = $upload->path;
-
-            if (! $path) {
-                continue;
-            }
-
-            $storage = Storage::disk($disk);
-
-            try {
-                $exists = $storage->exists($path);
+        if ($uploads->isNotEmpty()) {
+            foreach ($uploads as $upload) {
+                $disk = $upload->file_disk ?: config('filesystems.default', 'local');
+                $path = $upload->path;
 
-                if ($dryRun) {
-                    $this->line(sprintf('[dry-run] %s:%s (%s)', $disk, $path, $exists ? 'exists' : 'missing'));
+                if (! $path) {
                     continue;
                 }
 
-                if ($exists) {
-                    $storage->delete($path);
-                    $purged++;
-                } else {
-                    $missing++;
+                $storage = Storage::disk($disk);
+
+                try {
+                    $exists = $storage->exists($path);
+
+                    if ($dryRun) {
+                        $this->line(sprintf('[dry-run] %s:%s (%s)', $disk, $path, $exists ? 'exists' : 'missing'));
+                        continue;
+                    }
+
+                    if ($exists) {
+                        $storage->delete($path);
+                        $purged++;
+                    } else {
+                        $missing++;
+                    }
+
+                    $upload->forceFill([
+                        'file_deleted_at' => $now,
+                        'file_deleted_reason' => 'retention',
+                    ])->save();
+                } catch (Throwable $exception) {
+                    $failed++;
+                    report($exception);
+                    $this->error(sprintf('Failed to purge upload #%d (%s:%s): %s', $upload->id, $disk, $path, $exception->getMessage()));
                 }
+            }
+        }
 
-                $upload->forceFill([
-                    'file_deleted_at' => $now,
-                    'file_deleted_reason' => 'retention',
-                ])->save();
-            } catch (Throwable $exception) {
-                $failed++;
-                report($exception);
-                $this->error(sprintf('Failed to purge upload #%d (%s:%s): %s', $upload->id, $disk, $path, $exception->getMessage()));
+        $stagingUploadIds = AnonymousUpload::query()
+            ->whereIn('status', ['completed', 'failed'])
+            ->where('updated_at', '<=', $stagingCutoff)
+            ->orderBy('id')
+            ->limit($stagingLimit)
+            ->pluck('id')
+            ->all();
+
+        $stagingRows = 0;
+        $stagingUploads = count($stagingUploadIds);
+
+        if ($stagingUploadIds !== []) {
+            if ($dryRun) {
+                $stagingRows = (int) DB::table('anonymous_siebel_stagings')
+                    ->whereIn('upload_id', $stagingUploadIds)
+                    ->count();
+            } else {
+                foreach (array_chunk($stagingUploadIds, $stagingUploadChunk) as $idChunk) {
+                    do {
+                        $rowIds = DB::table('anonymous_siebel_stagings')
+                            ->whereIn('upload_id', $idChunk)
+                            ->orderBy('id')
+                            ->limit($rowChunk)
+                            ->pluck('id')
+                            ->all();
+
+                        if ($rowIds === []) {
+                            break;
+                        }
+
+                        $stagingRows += (int) DB::table('anonymous_siebel_stagings')
+                            ->whereIn('id', $rowIds)
+                            ->delete();
+                    } while (count($rowIds) === $rowChunk);
+                }
             }
         }
 
         if ($dryRun) {
-            $this->info(sprintf('Dry run complete. Candidates: %d', $uploads->count()));
+            $this->info(sprintf(
+                'Dry run complete. File candidates: %d. Staging candidates: %d uploads, %d rows (older than %d days).',
+                $uploads->count(),
+                $stagingUploads,
+                $stagingRows,
+                $stagingRetentionDays
+            ));
             return self::SUCCESS;
         }
 
-        $this->info(sprintf('Purged: %d, missing: %d, failed: %d', $purged, $missing, $failed));
+        $this->info(sprintf(
+            'Purged files: %d, missing files: %d, failed file purges: %d, pruned staging uploads: %d, pruned staging rows: %d (older than %d days)',
+            $purged,
+            $missing,
+            $failed,
+            $stagingUploads,
+            $stagingRows,
+            $stagingRetentionDays
+        ));
 
         return $failed > 0 ? self::FAILURE : self::SUCCESS;
     }

app/Constants/Fodig/Anonymizer/SiebelMetadata.php

@@ -25,6 +25,31 @@ final class SiebelMetadata
         'RELATED_COLUMNS',
     ];
 
+    public const TEMP_HEADER_COLUMNS = [
+        'DB_INSTANCE',
+        'OWNER',
+        'QUALFIELD',
+        'COLUMN_ID',
+        'TABLE_NAME',
+        'COLUMN_NAME',
+        'ANON_RULE',
+        'ANON_NOTE',
+        'PR_KEY',
+        'REF_TAB_NAME',
+        'NUM_DISTINCT',
+        'NUM_NOT_NULL',
+        'NUM_NULLS',
+        'NUM_ROWS',
+        'DATA_TYPE',
+        'DATA_LENGTH',
+        'DATA_PRECISION',
+        'DATA_SCALE',
+        'COMMENTS',
+        'SBL_USER_NAME',
+        'SBL_DESC_TEXT',
+        'NULLABLE',
+    ];
+
     public const IMPORT_DIRECTORY = 'anonymous-siebel/imports';
 
     // Defaults applied when transforming Siebel-column CSVs without explicit scope

app/Filament/Components/FormDeploymentsManager.php

@@ -27,17 +27,18 @@ public static function schema()
             ->schema([
                 Grid::make(3)
                     ->schema([
-                        // Test Environment
-                        Section::make('Test Environment')
-                            ->icon('heroicon-o-beaker')
+                        // Dev Environment
+                        Section::make('Development Environment')
+                            ->icon('heroicon-o-cog')
                             ->compact()
                             ->schema([
-                                Placeholder::make('test_deployment_info')
+                                Placeholder::make('dev_deployment_info')
                                     ->label('')
                                     ->content(function ($record) {
-                                        if (!$record) return 'No deployment details yet';
+                                        if (!$record)
+                                            return 'No deployment details yet';
 
-                                        $deployment = FormDeployment::getDeploymentForFormAndEnvironment($record->id, 'test');
+                                        $deployment = FormDeployment::getDeploymentForFormAndEnvironment($record->id, 'dev');
                                         if (!$deployment) {
                                             return 'No deployment details yet';
                                         }
@@ -46,7 +47,7 @@ public static function schema()
                                     })
                                     ->extraAttributes(['class' => 'text-sm']),
                                 Actions::make([
-                                    Action::make('deploy_to_test')
+                                    Action::make('deploy_to_dev')
                                         ->label('Deploy')
                                         ->icon('heroicon-o-rocket-launch')
                                         ->color('warning')
@@ -56,7 +57,8 @@ public static function schema()
                                             Select::make('form_version_id')
                                                 ->label('Form Version')
                                                 ->options(function ($record) {
-                                                    if (!$record) return [];
+                                                    if (!$record)
+                                                        return [];
                                                     return $record->formVersions()
                                                         ->whereIn('status', ['approved', 'published'])
                                                         ->get()
@@ -73,26 +75,27 @@ public static function schema()
                                         ->action(function (array $data, $record) {
                                             FormDeployment::deployToEnvironment(
                                                 $data['form_version_id'],
-                                                'test',
+                                                'dev',
                                                 Carbon::parse($data['deployed_at'])
                                             );
                                         })
-                                        ->successNotificationTitle('Successfully deployed to Test environment'),
+                                        ->successNotificationTitle('Successfully deployed to Development environment'),
                                 ])
                                     ->alignment(Alignment::Left),
                             ]),
 
-                        // Dev Environment
-                        Section::make('Development Environment')
-                            ->icon('heroicon-o-cog')
+                        // Test Environment
+                        Section::make('Test Environment')
+                            ->icon('heroicon-o-beaker')
                             ->compact()
                             ->schema([
-                                Placeholder::make('dev_deployment_info')
+                                Placeholder::make('test_deployment_info')
                                     ->label('')
                                     ->content(function ($record) {
-                                        if (!$record) return 'No deployment details yet';
+                                        if (!$record)
+                                            return 'No deployment details yet';
 
-                                        $deployment = FormDeployment::getDeploymentForFormAndEnvironment($record->id, 'dev');
+                                        $deployment = FormDeployment::getDeploymentForFormAndEnvironment($record->id, 'test');
                                         if (!$deployment) {
                                             return 'No deployment details yet';
                                         }
@@ -101,7 +104,7 @@ public static function schema()
                                     })
                                     ->extraAttributes(['class' => 'text-sm']),
                                 Actions::make([
-                                    Action::make('deploy_to_dev')
+                                    Action::make('deploy_to_test')
                                         ->label('Deploy')
                                         ->icon('heroicon-o-rocket-launch')
                                         ->color('info')
@@ -111,7 +114,8 @@ public static function schema()
                                             Select::make('form_version_id')
                                                 ->label('Form Version')
                                                 ->options(function ($record) {
-                                                    if (!$record) return [];
+                                                    if (!$record)
+                                                        return [];
                                                     return $record->formVersions()
                                                         ->whereIn('status', ['approved', 'published'])
                                                         ->get()
@@ -128,11 +132,11 @@ public static function schema()
                                         ->action(function (array $data, $record) {
                                             FormDeployment::deployToEnvironment(
                                                 $data['form_version_id'],
-                                                'dev',
+                                                'test',
                                                 Carbon::parse($data['deployed_at'])
                                             );
                                         })
-                                        ->successNotificationTitle('Successfully deployed to Development environment'),
+                                        ->successNotificationTitle('Successfully deployed to Test environment'),
                                 ])
                                     ->alignment(Alignment::Left),
                             ]),
@@ -145,7 +149,8 @@ public static function schema()
                                 Placeholder::make('prod_deployment_info')
                                     ->label('')
                                     ->content(function ($record) {
-                                        if (!$record) return 'No deployment details yet';
+                                        if (!$record)
+                                            return 'No deployment details yet';
 
                                         $deployment = FormDeployment::getDeploymentForFormAndEnvironment($record->id, 'prod');
                                         if (!$deployment) {
@@ -166,7 +171,8 @@ public static function schema()
                                             Select::make('form_version_id')
                                                 ->label('Form Version')
                                                 ->options(function ($record) {
-                                                    if (!$record) return [];
+                                                    if (!$record)
+                                                        return [];
                                                     return $record->formVersions()
                                                         ->whereIn('status', ['approved', 'published'])
                                                         ->get()