ENG-3924: Add info-level execution logs for paused privacy request status transitions

changelog/8251-info-execution-logs-for-status-transitions.yaml

@@ -0,0 +1,4 @@
+type: Changed
+description: Added info-level execution logs for paused privacy requests to surface why a request was paused (webhook halt or batch email send)
+pr: 8251
+labels: []

clients/admin-ui/src/features/privacy-requests/types.ts

@@ -25,6 +25,7 @@ export enum ExecutionLogStatus {
   RETRYING = "retrying",
   SKIPPED = "skipped",
   POLLING = "polling",
+  INFO = "info",
   // Audit log statuses for pre-approval webhooks
   APPROVED = "approved",
   DENIED = "denied",
@@ -43,6 +44,7 @@ export const ExecutionLogStatusLabels: Record<ExecutionLogStatus, string> = {
   [ExecutionLogStatus.RETRYING]: "Retrying",
   [ExecutionLogStatus.SKIPPED]: "Skipped",
   [ExecutionLogStatus.POLLING]: "Awaiting polling",
+  [ExecutionLogStatus.INFO]: "Info",
   [ExecutionLogStatus.APPROVED]: "Approved",
   [ExecutionLogStatus.DENIED]: "Denied",
   [ExecutionLogStatus.PRE_APPROVAL_WEBHOOK_TRIGGERED]: "Webhooks triggered",
@@ -63,6 +65,7 @@ export const ExecutionLogStatusColors: Record<
   [ExecutionLogStatus.PAUSED]: undefined,
   [ExecutionLogStatus.RETRYING]: undefined,
   [ExecutionLogStatus.POLLING]: CUSTOM_TAG_COLOR.WARNING,
+  [ExecutionLogStatus.INFO]: CUSTOM_TAG_COLOR.INFO,
   [ExecutionLogStatus.APPROVED]: CUSTOM_TAG_COLOR.SUCCESS,
   [ExecutionLogStatus.DENIED]: CUSTOM_TAG_COLOR.WARNING,
   [ExecutionLogStatus.PRE_APPROVAL_WEBHOOK_TRIGGERED]: CUSTOM_TAG_COLOR.INFO,

src/fides/api/models/privacy_request/privacy_request.py

@@ -1651,6 +1651,28 @@ def add_error_execution_log(
             },
         )
 
+    def add_info_execution_log(
+        self,
+        db: Session,
+        connection_key: Optional[str],
+        dataset_name: Optional[str],
+        collection_name: Optional[str],
+        message: str,
+        action_type: ActionType,
+    ) -> ExecutionLog:
+        return ExecutionLog.create(
+            db=db,
+            data={
+                "privacy_request_id": self.id,
+                "connection_key": connection_key,
+                "dataset_name": dataset_name,
+                "collection_name": collection_name,
+                "status": ExecutionLogStatus.info,
+                "message": message,
+                "action_type": action_type,
+            },
+        )
+
 
 class PrivacyRequestError(Base):
     """The DB ORM model to track PrivacyRequests error message status."""

src/fides/api/models/worker_task.py

@@ -18,6 +18,7 @@ class ExecutionLogStatus(enum.Enum):
     retrying = "retrying"
     skipped = "skipped"
     polling = "polling"
+    info = "info"
 
     @classmethod
     def in_progress_statuses(cls) -> Set["ExecutionLogStatus"]:

src/fides/api/service/privacy_request/email_batch_service.py

@@ -159,6 +159,14 @@ def requeue_privacy_requests_after_email_send(
         )
         privacy_request.status = PrivacyRequestStatus.paused
         privacy_request.save(db=db)
+        privacy_request.add_info_execution_log(
+            db,
+            connection_key=None,
+            dataset_name=None,
+            collection_name=None,
+            message="Request paused after batch email send, resuming from post-webhooks step",
+            action_type=privacy_request.policy.get_action_type(),  # type: ignore[arg-type]
+        )
 
         queue_privacy_request(
             privacy_request_id=privacy_request.id,

src/fides/api/service/privacy_request/request_runner_service.py

@@ -783,6 +783,14 @@ def run_privacy_request(
 
             except PrivacyRequestPaused as exc:
                 privacy_request.pause_processing(session)
+                privacy_request.add_info_execution_log(
+                    session,
+                    connection_key=None,
+                    dataset_name=None,
+                    collection_name=None,
+                    message=f"Request paused by webhook halt instruction: {exc}",
+                    action_type=privacy_request.policy.get_action_type(),  # type: ignore[arg-type]
+                )
                 _log_warning(exc, CONFIG.dev_mode)
                 return
 
@@ -1376,6 +1384,14 @@ def run_webhooks_and_report_status(
                 webhook.key,
             )
             privacy_request.pause_processing(db)
+            privacy_request.add_info_execution_log(
+                db,
+                connection_key=webhook.key,
+                dataset_name=None,
+                collection_name=None,
+                message=f"Request paused by webhook: {webhook.key}",
+                action_type=privacy_request.policy.get_action_type(),  # type: ignore[arg-type]
+            )
             initiate_paused_privacy_request_followup(privacy_request)
             return False
         except ClientUnsuccessfulException as exc:

tests/fides/ops/models/privacy_request/test_execution_logs.py

@@ -144,3 +144,14 @@ def test_execution_log_large_data(db, execution_log_data):
     assert retrieved_execution_log.message == large_message
     assert retrieved_execution_log.fields_affected == large_fields_affected
     execution_log.delete(db)
+
+
+def test_execution_log_info_status(db, execution_log_data):
+    """Test that execution logs can be created with the info status."""
+    execution_log_data["status"] = "info"
+    execution_log = ExecutionLog.create(db, data=execution_log_data)
+
+    retrieved = db.query(ExecutionLog).filter_by(privacy_request_id="test_id").first()
+    assert retrieved is not None
+    assert retrieved.status == ExecutionLogStatus.info
+    execution_log.delete(db)

tests/fides/ops/service/privacy_request/test_request_runner_service.py

@@ -3012,3 +3012,31 @@ def test_consent_runner_soft_time_limit(
             if "soft time limit" in (log.message or "").lower()
         ]
         assert len(timeout_logs) == 1
+
+
+class TestInfoExecutionLogs:
+    """Tests that info-level execution logs are created when requests are paused."""
+
+    @mock.patch(
+        "fides.api.models.privacy_request.PrivacyRequest.trigger_policy_webhook"
+    )
+    def test_webhook_halt_creates_info_log(
+        self,
+        mock_trigger_policy_webhook,
+        db,
+        privacy_request,
+        policy_pre_execution_webhooks,
+    ):
+        """When a webhook returns halt=true, an info execution log explains why the request paused."""
+        mock_trigger_policy_webhook.side_effect = PrivacyRequestPaused(
+            "Request received to halt"
+        )
+
+        proceed = run_webhooks_and_report_status(db, privacy_request, PolicyPreWebhook)
+        assert not proceed
+        assert privacy_request.status == PrivacyRequestStatus.paused
+
+        info_logs = privacy_request.execution_logs.filter_by(status="info").all()
+        assert len(info_logs) == 1
+        assert "paused by webhook" in info_logs[0].message.lower()
+        assert info_logs[0].connection_key == policy_pre_execution_webhooks[0].key