From f3d411e1898c987186bd6830f4c5ef6531bca8cb Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Wed, 3 Jun 2026 19:27:09 +0200 Subject: [PATCH 01/21] =?UTF-8?q?=E2=9C=A8(integrations)=20add=20outgoing?= =?UTF-8?q?=20webhooks=20for=20every=20incoming=20email?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/webhooks.md | 461 +++ env.d/development/backend.defaults | 3 + src/backend/core/admin.py | 30 +- src/backend/core/api/serializers.py | 137 +- src/backend/core/api/viewsets/channel.py | 170 +- src/backend/core/factories.py | 29 +- src/backend/core/mda/autoreply.py | 170 +- src/backend/core/mda/dispatch_webhooks.py | 781 +++++ src/backend/core/mda/inbound_create.py | 5 +- src/backend/core/mda/inbound_pipeline.py | 675 +++++ src/backend/core/mda/inbound_tasks.py | 439 ++- src/backend/core/models.py | 104 +- src/backend/core/services/ssrf.py | 40 +- src/backend/core/services/thread_events.py | 6 +- .../core/channel/regenerated_api_key.html | 8 +- .../tests/api/test_channel_scope_level.py | 8 +- src/backend/core/tests/api/test_channels.py | 132 + .../core/tests/mda/test_dispatch_webhooks.py | 2525 +++++++++++++++++ .../core/tests/mda/test_inbound_auth.py | 31 +- .../tests/mda/test_inbound_spoofed_sender.py | 4 +- .../core/tests/mda/test_spam_processing.py | 80 +- src/backend/messages/settings.py | 9 +- .../integrations-data-grid.tsx | 4 + .../modal-compose-integration/index.tsx | 20 +- .../webhook-integration-form.tsx | 380 +++ 25 files changed, 5775 insertions(+), 476 deletions(-) create mode 100644 docs/webhooks.md create mode 100644 src/backend/core/mda/dispatch_webhooks.py create mode 100644 src/backend/core/mda/inbound_pipeline.py create mode 100644 src/backend/core/tests/mda/test_dispatch_webhooks.py create mode 100644 src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx diff --git a/docs/webhooks.md b/docs/webhooks.md new file mode 100644 index 000000000..96e688be2 --- /dev/null +++ b/docs/webhooks.md @@ -0,0 +1,461 @@ +# Outbound Webhooks + +For every inbound message Messages can `POST` a notification to an HTTP +endpoint of your choosing. This page documents the on-the-wire format so +receivers can be implemented against a stable contract. + +Webhooks are **outbound** — Messages calls out to your endpoint. Anything +**inbound** (third parties calling into Messages) goes through other +channel types (`api_key`, `widget`, `mta`, etc.) and is not the subject +of this document. + +## When does it fire? + +For each inbound message accepted by the MTA the delivery pipeline goes +through two webhook phases: + +1. **`before_spam`** — fires right after the message has been parsed, + before any spam check. `is_spam` is not yet known. +2. **`after_spam`** — fires after the spam verdict but before the + `Message` row is created. `is_spam` is known. + +Each webhook channel picks **one** phase via `settings.phase` +(default `after_spam`). + +A channel may also be **blocking** (`settings.blocking: true`). A +blocking webhook gets to shape delivery: it can drop the message, ask +to be retried later, or return a small JSON body that overrides the +spam verdict and/or attaches labels to the resulting thread (see +[Response contract](#response-contract) below). Non-blocking webhooks +are fire-and-forget — failures are logged and the pipeline continues +unchanged. + +## Channel scopes + +A webhook channel can be configured at three scopes: + +| `scope_level` | Fires on | How to create | +| ------------- | --------------------------------------- | -------------------------------------------- | +| `mailbox` | Messages delivered to that mailbox | Mailbox admin via the **Integrations** modal | +| `maildomain` | Messages delivered to any mailbox of the domain | Maildomain admin via API / admin | +| `global` | Every message on the instance | Superuser via the Django admin or CLI | + +A given inbound message fans out to every matching channel. +`global` is intentionally not creatable through the public REST API — +it's a sensitive instance-wide hook. + +## Configuration + +A webhook channel stores its configuration in `Channel.settings` +(a JSON dict): + +```json +{ + "url": "https://example.com/inbox-hook", + "events": ["message.received"], + "phase": "after_spam", + "format": "eml", + "blocking": false, + "auth_method": "jwt" +} +``` + +| Key | Type | Default | Description | +| ------------- | -------- | -------------- | --------------------------------------------------------------------------- | +| `url` | string | **required** | `http://` or `https://` endpoint. Validated by the SSRF guard at each call. | +| `events` | string[] | **required** | Currently only `message.received` is implemented. | +| `phase` | string | `after_spam` | `before_spam` or `after_spam`. | +| `format` | string | `eml` | `eml`, `jmap`, or `jmap_without_body` (see [Payload formats](#payload-formats)). | +| `blocking` | bool | `false` | If true, the webhook response determines delivery (see [Response contract](#response-contract) below). | +| `auth_method` | string | **required** | `jwt` or `api_key` (see [Authentication](#authentication)). | + +The serializer validates every change to `settings`, on create **and** +on settings-only PATCH — there is no path that lets a malformed value +slip onto an existing channel. + +## HTTP request shape + +Every call is: + +* `POST` to `settings.url`. +* `User-Agent: Messages-Webhook/1.0`. +* 30-second timeout. +* HTTP `3xx` is **not** followed — receivers must respond on the URL + configured, not after a redirect. +* The destination hostname/IP must pass the shared SSRF check (no + loopback, link-local, private, multicast, reserved, or cloud metadata + addresses; no IP literals). + +### Authentication + +Every webhook channel has **one root secret**, minted server-side, +returned exactly once at create time and rotatable via +`POST /channels/{id}/regenerate-secret/`. The `auth_method` +setting picks how that root is presented on each POST. The root itself +never travels on the wire. + +| `auth_method` | Headers sent | Wire value | Receiver verifies | +| ------------- | ----------------------------------------------------------------------------------------------------------- | --------------------------------------------------------- | ------------------------------------------------------------------------------ | +| `jwt` | `X-StMsg-Webhook-Timestamp`, `X-StMsg-Webhook-Signature: v1=`, `Authorization: Bearer ` | HMAC sig + JWT, both **keyed** by the root | HMAC-SHA256 of `f"{timestamp}.{body}"` with the root, **or** the HS256 JWT. | +| `api_key` | `X-StMsg-Api-Key: ` | `whk_` + `HMAC-SHA256(root, "messages.webhook.api_key.v1").hex()` | Constant-time compare of the header against the receiver's stored copy. | + +A channel sends **only** the headers for its configured method — the +unused presentation never rides on the wire, so it can't leak through +receiver-side proxies or debug panes. The API-key value is a +**one-way derivation** of the root, so a receiver-side leak of the API +key reveals nothing about the root: HMAC/JWT verification on other +receivers stays unforgeable. + +#### Picking a method + +- `jwt` — best when the receiver controls a server (n8n, your own + Lambda, a Flask/Express app, Cloudflare Worker). Body integrity is + proven by the HMAC; JWT lets receivers verify with stock libraries. +- `api_key` — for low-code receivers that can only check a header + (Zapier "API key in header" trigger, IFTTT, a Zap webhook step). + +#### Switching methods on an existing channel + +PATCH the channel's `settings.auth_method`. The root secret is **not** +rotated — only the wire presentation changes — but the receiver was +given the old method's credential at creation. To get the new method's +credential, call `POST /channels/{id}/regenerate-secret/`: the +response returns either `webhook_secret` (jwt) or `webhook_api_key` +(api_key), matching the channel's current method. Rotation invalidates +the previous credential, so update the receiver before the next inbound +message lands. + +### Envelope headers (always set, regardless of format) + +| Header | Value | +| --------------------- | ---------------------------------------------------------------- | +| `Content-Type` | `message/rfc822` for `eml`, `application/json` for both JMAP variants | +| `X-StMsg-Event` | `message.received` | +| `X-StMsg-Phase` | `before_spam` or `after_spam` | +| `X-StMsg-Channel-Id` | UUID of the firing webhook Channel | +| `X-StMsg-Mailbox` | Destination mailbox address | +| `X-StMsg-Recipient` | Envelope `RCPT TO` (usually the same as `X-StMsg-Mailbox`) | +| `X-StMsg-Is-Spam` | `true`, `false`, or `unknown` (`unknown` in the `before_spam` phase) | +| `X-StMsg-Message-Id` | Original `Message-ID` header value (angle-bracketed), if any | + +### Response contract + +The classification below applies to **blocking** webhooks. Non-blocking +webhooks treat every outcome as success — their bodies are ignored. + +| Outcome | Decision | What happens | +| ------------------------------------- | ---------- | ------------------------------------------------------------------------------- | +| HTTP `2xx`, empty / non-JSON body | CONTINUE | Delivery proceeds normally. | +| HTTP `2xx` + JSON action body | see below | Body parsed for `action` / `is_spam` / `labels`. | +| HTTP `4xx` | DROP | Receiver definitively rejected the message; `InboundMessage` deleted. | +| HTTP `408`, `429`, `5xx` | RETRY | Transient — `InboundMessage` kept; the 5-min sweep re-fires the webhook. | +| Connection error, timeout, DNS, etc. | RETRY | Transient. | +| SSRF rejection | DROP | Config error on our side — retrying won't help. | +| Missing signing secret (misconfig) | DROP | The dispatcher fails closed rather than POST an unsigned request. | + +`RETRY` is bounded: an `InboundMessage` held in retry for more than +**7 days** is dropped with a loud `ERROR` log. This prevents a +permanently-broken receiver from pinning a row forever. (When you fix +the receiver within 7 days, the next sweep delivers normally.) + +#### JSON action body + +When a blocking webhook returns `HTTP 2xx` with `Content-Type: +application/json`, the body MAY contain the following keys. All are +optional; unknown keys are ignored. + +```json +{ + "action": "drop", + "is_spam": true, + "labels": ["b3c9c1c3-1f4a-4d4a-9b2d-9c5a2a7c0a01"], + "assign_to": ["alice@example.org"], + "mark_starred": true, + "mark_read": true, + "mark_trashed": false, + "mark_archived": true, + "skip_autoreply": true, + "add_event": [ + {"type": "im", "content": "AI summary: budget Q4 update"} + ], + "reply_draft": {"template": "b3c9c1c3-1f4a-4d4a-9b2d-9c5a2a7c0a01"} +} +``` + +| Key | Type | Meaning | +| ---------------- | -------------- | ---------------------------------------------------------------------------------------------------------------- | +| `action` | `"drop"` / `"retry"` | `"drop"` drops the message at this phase; `"retry"` re-queues the inbound task (bounded by the 7-day retry budget). Any other value (or omission) is treated as accept. Case-insensitive. | +| `is_spam` | bool | Override the spam verdict. Acts as a full antispam: in the `before_spam` phase this **skips rspamd**. | +| `labels` | string[] | UUIDs of `Label` rows in the destination mailbox to attach to the thread once it is created. | +| `assign_to` | string[] | OIDC emails of users to assign to the resulting thread (one `ThreadEvent ASSIGN` per webhook, channel-attributed). | +| `mark_starred` | bool (true only) | Star the resulting thread for the destination mailbox. | +| `mark_read` | bool (true only) | Mark the resulting thread as read for the destination mailbox. | +| `mark_trashed` | bool (true only) | Land the message with `is_trashed=true`. (Distinct from `action: "drop"` — the row stays, just hidden.) | +| `mark_archived` | bool (true only) | Land the message with `is_archived=true`. | +| `skip_autoreply` | bool (true only) | Suppress the standard autoreply for this message (in addition to the `is_spam=true` suppression). | +| `add_event` | object[] | Persist one `ThreadEvent` per entry, attributed to this webhook channel. See [Events](#add_event-events). | +| `reply_draft` | object | `{template: ""}` — materialise a draft reply for the user to refine + send. See [Reply drafts](#reply_draft-drafts). | + +Notes: + +* `action: "drop"` always wins. Setting `action: "drop"` together with + `labels` or `assign_to` still drops — the thread is never created, + so neither side effect is applied. +* `is_spam` discriminates between **explicit false (ham)** and **no + opinion**: returning `{}` leaves the dispatcher's verdict (typically + rspamd) untouched, while returning `{"is_spam": false}` forces ham. +* `labels` only makes sense for **mailbox-scoped** channels: labels are + per-mailbox. For domain- or global-scoped channels the UUIDs are + validated against the receiving mailbox; unknown UUIDs are logged and + skipped, not raised — a misbehaving webhook must not stall delivery. +* `assign_to` resolves each email to a User row with + `email__iexact`. The resolution is **strict but quiet**: emails that + resolve to zero users, to multiple users (ambiguous — `User.email` + isn't unique, see `MAILBOX_ROLES_CAN_BE_ASSIGNED`), or to a user + whose mailbox role isn't one of `EDITOR` / `SENDER` / `ADMIN` are + logged and skipped. **No auto-create**: a webhook receiver cannot + mint a User row. Each blocking webhook that contributes assignees + produces its own `ThreadEvent` with `channel` set to that webhook's + channel, so the audit timeline keeps per-receiver attribution. The + resulting `ThreadEvent.author` is `null` (the receiver is not a + user); the existing partial UniqueConstraint on `UserEvent(user, + thread) WHERE type=assign` makes duplicate asks idempotent. +* Bool flags (`mark_starred` / `mark_read` / `mark_trashed` / `mark_archived` / + `skip_autoreply`) use **`true`-only semantics**: a receiver opting in + with `true` flips the flag; `false`, missing, or non-bool values + are "no opinion". The multi-webhook merge is therefore a simple OR + — a later receiver can't silently veto an earlier receiver's + directive. `mark_trashed` / `mark_archived` set the corresponding + field on the `Message` row at creation time; `mark_starred` / `mark_read` + set `starred_at` / `read_at` on the destination `ThreadAccess` (no- + op when already set, so re-firing doesn't reset them). + +#### `add_event` events + +`add_event` is a list of structured events to persist on the resulting +thread. Each entry becomes one `ThreadEvent`, attributed to the +firing webhook via the `channel` FK; `author` is `null`. + +Supported types: + +| `type` | Required fields | Effect | +| ------ | ------------------ | ----------------------------------------------------------------------------------- | +| `"im"` | `content` (string) | Persists as an internal-message ThreadEvent — the same surface humans post into. | + +Unknown types are silently skipped at the classifier — the contract +stays forward-compatible so receivers can begin emitting new types +(e.g. `"iframe"`) before the server learns them, with no churn for +the receivers that already work. + +#### `reply_draft` drafts + +`reply_draft: {"template": ""}` materialises a **draft reply** +to the incoming message, pre-filled from a `MessageTemplate`. The +draft is threaded under the inbound message, ``Re:``-prefixed, and +addressed to the original sender — the user reviews and refines it +in the UI, then sends with a click. **We do not auto-send.** + +Implementation reuses the autoreply pipeline (sender contact, subject +prefix, message + recipient creation, signature resolution); the only +difference is the body lands in `draft_blob` (the rich-text editor's +JSON shape, from the template's `raw_body`), not in `blob`. The +editor round-trip is therefore identical to a hand-composed draft. + +Validation: + +* The template must be `type=message` and `is_active=true`, scoped to + the **destination mailbox or its maildomain**. Templates from other + mailboxes / domains are silently skipped — a webhook receiver + cannot draft from arbitrary templates. +* Templates from outside the destination scope are silently skipped + (logged, not raised). +* If the inbound message has no sender we can reply to, the draft is + skipped (same rule the autoreply path uses). + +Each blocking webhook that asks produces **one draft** attributed to +its own channel (`Message.channel` FK preserved for audit). If two +webhooks each ask, the user sees two drafts — they pick which one to +send, or delete both. + +#### Multi-webhook merge + +When several blocking webhooks fire on the same phase, their outcomes +merge deterministically: + +* **decision**: most severe wins (`DROP` > `RETRY` > `CONTINUE`). The + dispatcher short-circuits the fan-out as soon as any webhook drops. +* **is_spam**: last decisive value wins (DB iteration order). +* **labels**: set union across all webhooks. +* **assign_to**: each webhook's list lands as its own ThreadEvent + (channel attribution preserved). A user assigned by an earlier + webhook is absorbed by the partial UniqueConstraint when a later + webhook re-asks — no duplicate UserEvent, the first ask is the + canonical attribution. +* **mark_starred / mark_read / mark_trashed / mark_archived / skip_autoreply**: + OR-merged — any `true` wins. +* **add_event**: each entry lands as its own ThreadEvent, in the + order webhooks fired. No deduplication. +* **reply_draft**: each blocking webhook that asks produces one draft + Message, attributed to its own channel. No deduplication — multiple + receivers each asking yield multiple drafts. + +## Payload formats + +The three formats are mutually exclusive — pick one per channel. The +envelope headers above are identical across formats. + +### `eml` (default) + +The request body is the **raw RFC-822 message bytes**, exactly as the +MTA received them. + +```http +POST /inbox-hook HTTP/1.1 +Content-Type: message/rfc822 +X-StMsg-Event: message.received +X-StMsg-Phase: after_spam +X-StMsg-Channel-Id: 05f1f991-c2e9-4fa7-8a78-98c3aa904c7c +X-StMsg-Mailbox: alice@example.com +X-StMsg-Recipient: alice@example.com +X-StMsg-Is-Spam: false +X-StMsg-Message-Id: + +From: Bob +To: alice@example.com +Subject: Hi +Message-ID: +Content-Type: text/plain; charset=utf-8 + +Hello, Alice! +``` + +This is the simplest format. Any email library can parse it +(`email.message_from_bytes` in Python, JavaMail's `MimeMessage`, +mailparser in Node, etc.). + +### `jmap` + +The request body is a **strictly JMAP-compliant `Email` object** per +[RFC 8621 §4.1][rfc8621] serialised as JSON. The body is the object +itself — there is **no surrounding envelope** in the JSON; envelope +metadata lives in the headers above. + +```json +{ + "messageId": ["abc123@example.org"], + "inReplyTo": [], + "references": [], + "from": [{"email": "bob@example.org", "name": "Bob"}], + "to": [{"email": "alice@example.com", "name": ""}], + "cc": null, + "bcc": null, + "sender": null, + "replyTo": null, + "subject": "Hi", + "sentAt": "2026-01-01T12:00:00Z", + "receivedAt": "2026-06-01T08:43:21Z", + "headers": [ + {"name": "from", "value": "Bob "}, + {"name": "to", "value": "alice@example.com"}, + {"name": "subject", "value": "Hi"} + ], + "bodyValues": { + "1": {"value": "Hello, Alice!", "isEncodingProblem": false, "isTruncated": false} + }, + "textBody": [ + {"partId": "1", "blobId": null, "size": 13, "name": null, "type": "text/plain", "charset": "utf-8", "disposition": null, "cid": null, "language": null, "location": null} + ], + "htmlBody": [], + "attachments": [], + "hasAttachment": false, + "preview": null +} +``` + +#### Fields omitted on purpose + +JMAP defines a few `Email` properties that only make sense once the +message is **stored** in a JMAP server. The webhook fires *before* the +`Message` row exists (and may abort delivery in the blocking case), so +these are intentionally absent: + +* `id`, `blobId`, `threadId`, `mailboxIds`, `keywords`. + +Attachment **bytes** are also intentionally omitted: JMAP keeps +attachment content behind a `blobId` and a separate fetch, which has no +analogue in a fire-and-forget webhook. The `attachments[]` entries +still describe each attachment's `type`, `size`, `name`, `disposition` +and `cid`. If you need the raw bytes pick `format: "eml"` instead. + +#### Date formatting + +`sentAt` and `receivedAt` are JMAP `UTCDate` strings: ISO-8601 in UTC +with an explicit `Z` suffix, e.g. `2026-01-01T12:00:00Z` (not +`+00:00`). This matches RFC 8621 §1.4. + +### `jmap_without_body` + +Same JMAP `Email` shape as `jmap`, but the body content and attachments +are dropped: + +* `textBody`, `htmlBody`, `bodyValues`, `attachments` are **omitted**. +* `hasAttachment` is preserved as a single boolean so receivers can + still tell whether the original message had attachments. +* All envelope fields (`from`, `to`, `subject`, `messageId`, `headers`, + `sentAt`, `receivedAt`, …) are included. + +Use this format when you only need the "a message arrived" signal plus +addressing metadata — for example to forward to a chat channel — and +don't want the body content to leave the instance over the wire. + +## Example receiver + +A minimal Python receiver that accepts both formats: + +```python +import email +import json +from flask import Flask, request + +app = Flask(__name__) + +@app.post("/inbox-hook") +def inbox_hook(): + content_type = request.headers.get("Content-Type", "") + if content_type.startswith("message/rfc822"): + msg = email.message_from_bytes(request.get_data()) + print("EML subject:", msg["subject"]) + elif content_type.startswith("application/json"): + body = request.get_json() + print("JMAP subject:", body["subject"]) + # Body content may not be there in jmap_without_body mode. + body_values = body.get("bodyValues") or {} + for part_id, value in body_values.items(): + print(f" part {part_id}: {value['value'][:80]}") + else: + return "unsupported", 415 + + # Echo envelope metadata for logging. + print("phase:", request.headers["X-StMsg-Phase"]) + print("is_spam:", request.headers["X-StMsg-Is-Spam"]) + print("mailbox:", request.headers["X-StMsg-Mailbox"]) + return "", 200 +``` + +## Security notes + +* The endpoint URL is **caller-controlled** (a mailbox admin sets it), + so every call goes through the shared `SSRFSafeSession`: + * Only `http://` and `https://` URLs are accepted. + * IP literals are rejected — a domain name is required. + * Hostnames resolving to loopback, link-local, private, multicast, + reserved, or cloud-metadata addresses are rejected. + * The validated IP is **pinned** for the actual connection, defeating + DNS-rebinding (TOCTOU). For HTTPS the TLS certificate is verified + against the original hostname. +* Blocking webhooks are silent for the original sender — the inbound + SMTP transaction has already been accepted. A blocking-drop is + visible only through logs and the pipeline's `dropped_by_webhook` + return value. + +[rfc8621]: https://www.rfc-editor.org/rfc/rfc8621#section-4.1 diff --git a/env.d/development/backend.defaults b/env.d/development/backend.defaults index 3c98b7d8c..77d0d6fb1 100644 --- a/env.d/development/backend.defaults +++ b/env.d/development/backend.defaults @@ -106,6 +106,9 @@ AI_MODEL= FEATURE_AI_SUMMARY=False FEATURE_AI_AUTOLABELS=False +# Integrations enabled in the integrations modal (mailbox admins) +FEATURE_MAILBOX_ADMIN_CHANNELS=api_key,widget,webhook + # Third-party services # Drive - https://github.com/suitenumerique/drive DRIVE_BASE_URL= diff --git a/src/backend/core/admin.py b/src/backend/core/admin.py index 960b63ede..d9355556d 100644 --- a/src/backend/core/admin.py +++ b/src/backend/core/admin.py @@ -573,7 +573,7 @@ def get_urls(self): urls = super().get_urls() custom_urls = [ path( - "/regenerate-api-key/", + "/regenerate-secret/", self.admin_site.admin_view(self.regenerate_api_key_view), name="core_channel_regenerate_api_key", ), @@ -581,13 +581,16 @@ def get_urls(self): return custom_urls + urls def regenerate_api_key_view(self, request, object_id): - """Regenerate the api_key secret on an api_key channel. - - Delegates the actual rotation to ``Channel.rotate_api_key`` (single - source of truth, shared with the DRF create + regenerate flows). The - plaintext is rendered ONCE in the response body and never stored in - cookies, session, or the messages framework — closing the window - where a credential could leak through signed-cookie message storage. + """Regenerate the secret on an api_key or webhook channel. + + Both channel types authenticate with a single root secret + (``Channel.rotate_secret`` is the single source of truth, shared + with the DRF create + regenerate flows): for api_key channels it + is the API key itself, for webhook channels it is the HMAC/JWT + signing secret. The plaintext is rendered ONCE in the response + body and never stored in cookies, session, or the messages + framework — closing the window where a credential could leak + through signed-cookie message storage. """ # pylint: disable=import-outside-toplevel from core.enums import ChannelTypes @@ -599,20 +602,21 @@ def regenerate_api_key_view(self, request, object_id): if channel is None: messages.error(request, "Channel not found.") return redirect("..") - if channel.type != ChannelTypes.API_KEY: + if channel.type not in (ChannelTypes.API_KEY, ChannelTypes.WEBHOOK): messages.error( - request, "Only api_key channels can have their secret regenerated." + request, + "Only api_key and webhook channels can have their secret regenerated.", ) return redirect("..") - plaintext = channel.rotate_api_key() + plaintext = channel.rotate_secret() context = { **self.admin_site.each_context(request), "opts": self.model._meta, # noqa: SLF001 "original": channel, - "title": "New api_key generated", - "api_key": plaintext, + "title": "New secret generated", + "secret": plaintext, } return TemplateResponse( request, "admin/core/channel/regenerated_api_key.html", context diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index a9f5754df..f3eb092e0 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -4,6 +4,7 @@ import hashlib import json import uuid +from urllib.parse import urlparse from django.conf import settings from django.core.exceptions import ValidationError as DjangoValidationError @@ -15,6 +16,11 @@ from rest_framework.exceptions import PermissionDenied from core import enums, models +from core.mda.dispatch_webhooks import ( + PHASE_AFTER_SPAM, + VALID_FORMATS, + VALID_PHASES, +) from core.mda.inline_images import extract_inline_images_html from core.services.blob_gc import schedule_for_gc from core.services.identity import keycloak as keycloak_service @@ -2027,25 +2033,40 @@ class Meta: # the plaintext ``settings`` JSONField — a DB read would otherwise # surface every user's CalDAV password. enums.ChannelTypes.CALDAV: ["username", "password"], + # The outgoing webhook secret is generated server-side and + # written straight to ``encrypted_settings["secret"]``; callers + # must never smuggle their own value in via ``settings``. The + # same bytes back both auth methods (JWT/HMAC and API key) — + # see ``Channel.rotate_secret``. + enums.ChannelTypes.WEBHOOK: ["secret"], } + # Channel types that mint a plaintext secret on create. The + # viewset surfaces the minted value via the right response key + # (api_key for api_key channels, webhook_secret / webhook_api_key + # for webhook channels per ``settings.auth_method``). + _ROTATABLE_TYPES = frozenset( + {enums.ChannelTypes.API_KEY, enums.ChannelTypes.WEBHOOK} + ) + def create(self, validated_data): - # For api_key channels, mint the secret on a transient instance so - # the resulting ``encrypted_settings`` rides through the normal - # ``super().create()`` save path. The plaintext is stashed on the - # saved row for ChannelViewSet.create() to surface exactly once, - # mirroring the ``_generated_password`` pattern at channel.py:68-74. - generated_api_key = None - if validated_data.get("type") == enums.ChannelTypes.API_KEY: + # Mint the per-type secret on a transient instance so the + # resulting ``encrypted_settings`` rides through the normal + # ``super().create()`` save path. The plaintext (when one + # exists at rest — only api_key channels store it indirectly + # via a hash) is stashed on the saved row for + # ``ChannelViewSet.create()`` to surface exactly once. + plaintext = None + if validated_data.get("type") in self._ROTATABLE_TYPES: transient = models.Channel(**validated_data) - generated_api_key = transient.rotate_api_key(save=False) + plaintext = transient.rotate_secret(save=False) validated_data["encrypted_settings"] = transient.encrypted_settings instance = super().create(validated_data) # pylint: disable=protected-access - if generated_api_key is not None: - instance._generated_api_key = generated_api_key # noqa: SLF001 + if plaintext is not None and instance.type == enums.ChannelTypes.API_KEY: + instance._generated_api_key = plaintext # noqa: SLF001 return instance def validate_settings(self, value): @@ -2236,6 +2257,22 @@ def _validate_webhook_settings(self, attrs): raise serializers.ValidationError( {"settings": "webhook settings.url must be http:// or https://"} ) + # Each POST carries the HMAC/JWT signing material in its headers, + # so plaintext http would leak credentials in transit. Require + # https except for a local-dev escape hatch: a loopback host, or + # DEBUG (where operators routinely point at a local receiver). + parsed_url = urlparse(url) + host = (parsed_url.hostname or "").lower() + is_loopback = host in ("localhost", "127.0.0.1", "::1") + if parsed_url.scheme != "https" and not (settings.DEBUG or is_loopback): + raise serializers.ValidationError( + { + "settings": ( + "webhook settings.url must use https:// " + "(http:// is only allowed for localhost)." + ) + } + ) events = settings_data.get("events") if not events or not isinstance(events, list): @@ -2250,6 +2287,45 @@ def _validate_webhook_settings(self, attrs): {"settings": f"Unknown webhook events: {unknown}"} ) + # Optional dispatcher knobs. Validated here so a malformed PATCH + # to ``settings`` can't strand a webhook channel in a state the + # dispatcher silently treats as defaults. + phase = settings_data.get("phase", PHASE_AFTER_SPAM) + if phase not in VALID_PHASES: + raise serializers.ValidationError( + { + "settings": ( + f"webhook settings.phase must be one of: " + f"{', '.join(sorted(VALID_PHASES))}." + ) + } + ) + body_format = settings_data.get("format", "eml") + if body_format not in VALID_FORMATS: + raise serializers.ValidationError( + { + "settings": ( + f"webhook settings.format must be one of: " + f"{', '.join(sorted(VALID_FORMATS))}." + ) + } + ) + if "blocking" in settings_data and not isinstance( + settings_data["blocking"], bool + ): + raise serializers.ValidationError( + {"settings": "webhook settings.blocking must be a boolean."} + ) + if settings_data.get("auth_method") not in ("jwt", "api_key"): + raise serializers.ValidationError( + { + "settings": ( + "webhook settings.auth_method is required and must " + "be 'jwt' or 'api_key'." + ) + } + ) + def validate(self, attrs): """Validate channel data. @@ -2311,6 +2387,47 @@ def validate(self, attrs): return attrs +class ChannelCreateResponseSerializer(ChannelSerializer): + """Schema-only view of the channel-create 201 response. + + ``ChannelViewSet.create`` returns the full ``ChannelSerializer`` + payload plus the freshly-minted plaintext credentials — surfaced + exactly once on creation and never retrievable again. They are + declared here as read-only fields so generated API clients see them + in the OpenAPI schema. This serializer is never used to serialize a + response directly; the view assembles the body by hand. + """ + + api_key = serializers.CharField( + read_only=True, + required=False, + help_text="api_key channels only — the plaintext API key.", + ) + password = serializers.CharField( + read_only=True, + required=False, + help_text="Plaintext password, when the channel type mints one.", + ) + webhook_secret = serializers.CharField( + read_only=True, + required=False, + help_text="webhook channels with auth_method=jwt — the HMAC/JWT signing secret.", + ) + webhook_api_key = serializers.CharField( + read_only=True, + required=False, + help_text="webhook channels with auth_method=api_key — the derived API key.", + ) + + class Meta(ChannelSerializer.Meta): + fields = ChannelSerializer.Meta.fields + [ + "api_key", + "password", + "webhook_secret", + "webhook_api_key", + ] + + class MessageTemplateSerializer(serializers.ModelSerializer): """Serialize message templates for POST/PUT/PATCH operations.""" diff --git a/src/backend/core/api/viewsets/channel.py b/src/backend/core/api/viewsets/channel.py index 20093209a..65c6aa906 100644 --- a/src/backend/core/api/viewsets/channel.py +++ b/src/backend/core/api/viewsets/channel.py @@ -20,6 +20,41 @@ from .. import permissions, serializers +def _attach_credential(data: dict, channel: models.Channel) -> None: + """Add the channel's freshly-minted credential to ``data``. + + Type-specific response key — lets the frontend branch on presence + instead of sniffing prefixes: + + - ``api_key`` channels → ``api_key`` + - ``webhook`` channels (``auth_method='jwt'``) → ``webhook_secret`` + (the raw root from ``encrypted_settings["secret"]``) + - ``webhook`` channels (``auth_method='api_key'``) → + ``webhook_api_key`` (HMAC-derived from the root) + + For api_key channels the plaintext is one-shot (we only store the + hash), so callers must stash it on ``instance._generated_api_key`` + via the serializer's create flow. For webhook channels the raw root + sits in ``encrypted_settings["secret"]`` and ``get_webhook_api_key`` + derives lazily — both readable straight off ``channel``. + """ + if channel.type == ChannelTypes.API_KEY: + plaintext = getattr(channel, "_generated_api_key", None) + if plaintext: + data["api_key"] = plaintext + return + if channel.type == ChannelTypes.WEBHOOK: + auth_method = (channel.settings or {}).get("auth_method") + if auth_method == "jwt": + root = (channel.encrypted_settings or {}).get("secret") + if root: + data["webhook_secret"] = root + elif auth_method == "api_key": + derived = channel.get_webhook_api_key() + if derived: + data["webhook_api_key"] = derived + + @extend_schema( tags=["channels"], description="Manage integration channels for a mailbox" ) @@ -62,17 +97,6 @@ def get_serializer_context(self): context["mailbox"] = self.mailbox return context - @extend_schema( - request=serializers.ChannelSerializer, - responses={ - 201: OpenApiResponse( - response=serializers.ChannelSerializer, - description="Channel created successfully", - ), - 400: OpenApiResponse(description="Invalid input data"), - 403: OpenApiResponse(description="Permission denied"), - }, - ) def get_save_kwargs(self): """Hook for subclasses to inject the scope-level + target FKs. @@ -89,6 +113,21 @@ def get_save_kwargs(self): "user": self.request.user, } + @extend_schema( + request=serializers.ChannelSerializer, + responses={ + 201: OpenApiResponse( + response=serializers.ChannelCreateResponseSerializer, + description=( + "Channel created successfully. The response carries the " + "one-time plaintext credentials (api_key / webhook_secret / " + "webhook_api_key / password) which are never returned again." + ), + ), + 400: OpenApiResponse(description="Invalid input data"), + 403: OpenApiResponse(description="Permission denied"), + }, + ) def create(self, request, *args, **kwargs): """Create a new channel. @@ -106,16 +145,11 @@ def create(self, request, *args, **kwargs): instance = serializer.save(**self.get_save_kwargs()) data = serializer.data - # Surface plaintext secrets exactly once on creation. Each generator - # lives on the instance under `_generated_*` (see ChannelSerializer). - # Subsequent GETs never return any of these. - for attr, response_key in ( - ("_generated_password", "password"), - ("_generated_api_key", "api_key"), - ): - value = getattr(instance, attr, None) - if value: - data[response_key] = value + # Surface plaintext credentials exactly once on creation — + # subsequent GETs never return them. + if password := getattr(instance, "_generated_password", None): + data["password"] = password + _attach_credential(data, instance) return Response(data, status=status.HTTP_201_CREATED) @extend_schema( @@ -157,55 +191,83 @@ def destroy(self, request, *args, **kwargs): responses={ 200: OpenApiResponse( response=inline_serializer( - name="RegeneratedApiKeyResponse", + name="RegeneratedSecretResponse", fields={ - "id": drf_serializers.CharField( - help_text="Channel id (also the X-Channel-Id header value).", - ), + "id": drf_serializers.CharField(help_text="Channel id."), "api_key": drf_serializers.CharField( + required=False, help_text=( - "Freshly generated plaintext api_key. Returned " - "ONCE on regeneration and cannot be retrieved later." + "Present for ``api_key`` channels — the " + "plaintext used in subsequent X-API-Key " + "calls. Returned ONCE." + ), + ), + "webhook_secret": drf_serializers.CharField( + required=False, + help_text=( + "Present for webhook channels with " + "``auth_method='jwt'`` — the freshly " + "minted root receivers use to verify the " + "HMAC sig and JWT." + ), + ), + "webhook_api_key": drf_serializers.CharField( + required=False, + help_text=( + "Present for webhook channels with " + "``auth_method='api_key'`` — the " + "HMAC-derived value sent as " + "X-StMsg-Api-Key. Changes whenever the " + "root rotates." ), ), }, ), description=( - "Returns the freshly generated plaintext api_key. The " - "previous secret is invalidated immediately. The plaintext " - "is shown ONCE and cannot be retrieved later." + "Rotates the channel's secret. Single-active: the " + "previous credential is invalidated immediately. " + "The response carries exactly one of ``api_key`` / " + "``webhook_secret`` / ``webhook_api_key`` matching " + "the channel's type (and, for webhooks, its current " + "``auth_method``)." ), ), - 400: OpenApiResponse(description="Channel is not an api_key channel"), + 400: OpenApiResponse(description="Channel type has no rotatable secret"), 403: OpenApiResponse(description="Permission denied"), 404: OpenApiResponse(description="Channel not found"), }, ) - @action(detail=True, methods=["post"], url_path="regenerate-api-key") - def regenerate_api_key(self, request, *args, **kwargs): - """Regenerate the api_key on this channel. - - Single-active rotation: the new secret REPLACES the old one - immediately, so any client still using the old secret will - start failing on the next call. This is the only rotation - flow exposed via DRF. - - Smooth (dual-active) rotation — appending a new hash without - removing the old one so clients can migrate over a window — is - intentionally a superadmin-only feature available via Django admin. + @action(detail=True, methods=["post"], url_path="regenerate-secret") + def regenerate_secret(self, request, *args, **kwargs): + """Rotate this channel's secret. + + Type-agnostic entry point: ``Channel.rotate_secret`` dispatches + on ``self.type`` and persists the new credential in the + appropriate storage shape (hash for ``api_key``, plaintext for + ``webhook``). Channel types without a rotatable secret raise + and surface as HTTP 400. + + Single-active rotation. Smooth (dual-active) rotation — + appending a new hash without removing the old one so clients + can migrate over a window — is intentionally a superadmin-only + feature available via Django admin. """ instance = self.get_object() - if instance.type != ChannelTypes.API_KEY: - raise ValidationError( - {"type": "Only api_key channels can have their secret regenerated."} - ) - - plaintext = instance.rotate_api_key() - - return Response( - {"id": str(instance.id), "api_key": plaintext}, - status=status.HTTP_200_OK, - ) + try: + plaintext = instance.rotate_secret() + except ValueError as exc: + raise ValidationError({"type": str(exc)}) from exc + + # api_key channels store only the hash; stash the just-minted + # plaintext on the instance so ``_attach_credential`` can find + # it (the field is read-once and never persisted). + if instance.type == ChannelTypes.API_KEY: + # pylint: disable-next=protected-access + instance._generated_api_key = plaintext # noqa: SLF001 + + payload: dict = {"id": str(instance.id)} + _attach_credential(payload, instance) + return Response(payload, status=status.HTTP_200_OK) @extend_schema( diff --git a/src/backend/core/factories.py b/src/backend/core/factories.py index 4cda587ce..d20ebd39f 100644 --- a/src/backend/core/factories.py +++ b/src/backend/core/factories.py @@ -358,6 +358,17 @@ class Meta: name = factory.Sequence(lambda n: f"Test Channel {n}") type = factory.fuzzy.FuzzyChoice(["widget", "mta"]) settings = factory.Dict({"config": {"enabled": True}}) + # Auto-mint a placeholder webhook signing secret for webhook channels + # so tests don't all have to do it themselves — the dispatcher + # fails-closed without one. Non-webhook channels keep an empty + # encrypted_settings (matching production behaviour). + encrypted_settings = factory.LazyAttribute( + lambda o: ( + {"secret": "whsec_factory_test"} + if o.type == "webhook" + else {} + ) + ) mailbox = factory.SubFactory(MailboxFactory) maildomain = None user = None @@ -373,6 +384,20 @@ class Meta: ) ) + @factory.post_generation + def _default_webhook_auth_method( # pylint: disable=unused-argument + self, create, extracted, **kwargs + ): + """Inject ``auth_method='jwt'`` into webhook channel settings if a + test didn't set one. The serializer requires it on real creates; + this just spares every webhook test from spelling it out.""" + if not create or self.type != "webhook": + return + if (self.settings or {}).get("auth_method"): + return + self.settings = {**(self.settings or {}), "auth_method": "jwt"} + self.save(update_fields=["settings"]) + def make_api_key_channel( *, @@ -387,7 +412,7 @@ def make_api_key_channel( """Create an api_key Channel and return ``(channel, plaintext)``. Single source of truth for tests that need a working api_key. Mints - the secret via ``Channel.rotate_api_key`` so the production helper is + the secret via ``Channel.rotate_secret`` so the production helper is exercised on the same code path the DRF/admin flows use, and so the plaintext prefix stays consistent across production and tests. @@ -403,7 +428,7 @@ def make_api_key_channel( user=user, settings={"scopes": list(scopes), **(extra_settings or {})}, ) - plaintext = channel.rotate_api_key(save=False) + plaintext = channel.rotate_secret(save=False) channel.save() return channel, plaintext diff --git a/src/backend/core/mda/autoreply.py b/src/backend/core/mda/autoreply.py index fda9e5594..5cad2597c 100644 --- a/src/backend/core/mda/autoreply.py +++ b/src/backend/core/mda/autoreply.py @@ -175,6 +175,69 @@ def should_send_autoreply( return template +def _create_reply_record_from_template( + template: models.MessageTemplate, + mailbox: models.Mailbox, + inbound_message: models.Message, + *, + is_draft: bool, + channel: Optional[models.Channel] = None, +): + """Build the reply ``Message`` row + ``MessageRecipient`` and resolve + the validated signature. + + Shared by the autoreply path (which then composes + DKIM-signs the + MIME via ``compose_and_sign_mime`` and triggers an async send) and + the webhook-driven ``reply_draft`` path (which stores the template's + editor-format raw body as ``draft_blob`` so the user can refine the + draft inline in the UI before sending). + + Returns ``(message, validated_signature)``. The caller is + responsible for attaching the body blob (sent message → ``blob``, + draft → ``draft_blob``) inside the same transaction. + """ + # 1. Get or create a Contact for the mailbox's own email + mailbox_email = str(mailbox) + mailbox_contact, _ = models.Contact.objects.get_or_create( + email=mailbox_email, + mailbox=mailbox, + defaults={"name": mailbox.contact.name if mailbox.contact else mailbox_email}, + ) + + # 2. Build subject with Re: prefix + subject = reply_subject(inbound_message.subject or "")[:255] + + # 3. Resolve signature: forced domain/mailbox signature takes priority + # over the one attached to the template + validated_signature = mailbox.get_validated_signature( + template.signature.id if template.signature else None + ) + + # 4. Create Message record + message = models.Message.objects.create( + thread=inbound_message.thread, + sender=mailbox_contact, + subject=subject, + parent=inbound_message, + sent_at=None if is_draft else timezone.now(), + is_draft=is_draft, + is_sender=True, + is_trashed=False, + is_spam=False, + signature=validated_signature if is_draft else None, + channel=channel, + ) + + # 5. Create MessageRecipient (must exist before compose_and_sign_mime) + models.MessageRecipient.objects.create( + message=message, + contact=inbound_message.sender, + type=MessageRecipientTypeChoices.TO, + ) + + return message, validated_signature + + def send_autoreply_for_message( template: models.MessageTemplate, mailbox: models.Mailbox, @@ -195,50 +258,17 @@ def send_autoreply_for_message( ) return - thread = inbound_message.thread - - # 1. Get or create a Contact for the mailbox's own email (the autoreply sender) - mailbox_email = str(mailbox) - mailbox_contact, _ = models.Contact.objects.get_or_create( - email=mailbox_email, - mailbox=mailbox, - defaults={"name": mailbox.contact.name if mailbox.contact else mailbox_email}, - ) - - # 2. Build subject with Re: prefix - subject = reply_subject(inbound_message.subject or "")[:255] - - # 3-7: Create records and compose MIME atomically so a failure in - # compose_and_sign_mime does not leave orphan Message/Recipient rows. + # 1-5 + compose: atomic so a failure in compose_and_sign_mime + # cannot leave orphan Message / Recipient rows. with transaction.atomic(): - # 3. Create Message record - message = models.Message.objects.create( - thread=thread, - sender=mailbox_contact, - subject=subject, - parent=inbound_message, - sent_at=timezone.now(), + message, validated_signature = _create_reply_record_from_template( + template, + mailbox, + inbound_message, is_draft=False, - is_sender=True, - is_trashed=False, - is_spam=False, - ) - - # 4. Create MessageRecipient (must exist before compose_and_sign_mime) - models.MessageRecipient.objects.create( - message=message, - contact=inbound_message.sender, - type=MessageRecipientTypeChoices.TO, - ) - - # 5. Resolve signature: forced domain/mailbox signature takes priority - # over the one attached to the autoreply template - validated_signature = mailbox.get_validated_signature( - template.signature.id if template.signature else None ) - # 6. Compose MIME, DKIM sign, and store as blob - # (signature + reply quote embedding is handled by compose_and_sign_mime) + # Compose MIME, DKIM sign, and store as blob auto_reply_headers = [ ("Auto-Submitted", "auto-replied"), ("X-Auto-Response-Suppress", "All"), @@ -257,13 +287,13 @@ def send_autoreply_for_message( ) message.save(update_fields=["mime_id", "blob", "has_attachments"]) - # 7. Trigger async send (outside transaction to avoid sending before commit) + # Trigger async send (outside transaction to avoid sending before commit) send_message_task.delay(str(message.id)) - # 8. Update thread stats — do NOT update read_at here: the autoreply + # Update thread stats — do NOT update read_at here: the autoreply # sender is away, so the thread must stay unread for them to notice # new messages when they return. - thread.update_stats() + inbound_message.thread.update_stats() logger.info( "Autoreply message %s created and queued for sending (mailbox=%s, to=%s)", @@ -273,6 +303,60 @@ def send_autoreply_for_message( ) +def create_draft_reply_from_template( + template: models.MessageTemplate, + mailbox: models.Mailbox, + inbound_message: models.Message, + *, + channel: Optional[models.Channel] = None, +) -> Optional[models.Message]: + """Materialise a draft reply from ``template`` against + ``inbound_message``. Returns the draft Message, or ``None`` if the + inbound has no sender (no one to reply to — same skip as the + autoreply path). + + Stores the template's editor-format ``raw_body`` JSON as + ``draft_blob`` so the recipient mailbox can edit the draft inline + in the rich-text editor before sending — no MIME pre-composition, + no premature DKIM. The send-draft pipeline composes + signs at + send time exactly as it does for hand-composed drafts. + """ + if not inbound_message.sender or not inbound_message.sender.email: + logger.warning( + "Cannot create reply_draft: inbound message %s has no sender email", + inbound_message.id, + ) + return None + + raw_body = template.raw_body or "{}" + raw_body_bytes = raw_body.encode("utf-8") + + with transaction.atomic(): + message, _ = _create_reply_record_from_template( + template, + mailbox, + inbound_message, + is_draft=True, + channel=channel, + ) + message.draft_blob = models.Blob.objects.create_blob( + content=raw_body_bytes, + content_type="application/json", + ) + message.save(update_fields=["draft_blob"]) + + inbound_message.thread.update_stats() + + logger.info( + "Draft reply %s created from template %s (mailbox=%s, channel=%s)", + message.id, + template.id, + mailbox.id, + channel.id if channel else None, + ) + return message + + def try_send_autoreply( mailbox: models.Mailbox, parsed_email: JmapEmail, diff --git a/src/backend/core/mda/dispatch_webhooks.py b/src/backend/core/mda/dispatch_webhooks.py new file mode 100644 index 000000000..2ef3a4af0 --- /dev/null +++ b/src/backend/core/mda/dispatch_webhooks.py @@ -0,0 +1,781 @@ +"""User-configured outbound webhooks, modelled as pipeline ``Step``s. + +For each delivered message the inbound pipeline (see +``inbound_pipeline.py``) iterates every webhook-type Channel that +matches the destination mailbox (``scope_level=MAILBOX``), its domain +(``scope_level=MAILDOMAIN``), or any global channel +(``scope_level=GLOBAL``). The webhook fires either ``before_spam`` or +``after_spam`` according to ``settings.phase``; a ``blocking`` webhook +can abort delivery (DROP), request retry (RETRY), override the spam +verdict, or attach labels via its JSON response body. + +This file is webhook-specific: HTTP plumbing, signing (HMAC + JWT or +API key), JMAP body building, SSRF-safe POST, response classification. +The pipeline-side glue is ``UserWebhookStep`` + ``webhook_steps_for_mailbox``. + +The HTTP client is the shared ``SSRFSafeSession`` — webhook URLs are +attacker-controllable, so the same hostname/IP rejection rules used by +the image proxy and IMAP importer apply here too. + +Two body formats are supported (see ``docs/webhooks.md``): + - ``format="eml"`` (default): raw RFC-822 bytes, ``Content-Type: + message/rfc822``. Webhook envelope metadata lives in ``X-StMsg-*`` + headers. + - ``format="jmap"``: JMAP-compliant ``Email`` object (RFC 8621 §4.1) + serialised as a single JSON document with ``Content-Type: + application/json``. The body is a strictly compliant Email object — + same envelope metadata in ``X-StMsg-*`` headers. +""" + +# pylint: disable=broad-exception-caught + +from __future__ import annotations + +import hashlib +import hmac +import json +import logging +import secrets +import time +import uuid as uuid_module +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Set, Tuple + +from django.db.models import Q + +import jwt +from jmap_email import JmapEmail, first_msgid + +from core import enums, models +from core.mda.inbound_pipeline import Decision, InboundContext, Step +from core.services.ssrf import SSRFSafeSession, SSRFValidationError + +logger = logging.getLogger(__name__) + +WEBHOOK_TIMEOUT = 30 # seconds + +# Hard cap on the receiver response body we parse for the action JSON. +# The contract body is tiny (action / is_spam / labels = a few hundred +# bytes at most). A bigger response is almost certainly an HTML error +# page from a misconfigured proxy — parse what we have, ignore the +# rest, never let a misbehaving receiver OOM the worker. +MAX_RESPONSE_BODY = 64 * 1024 # 64 KiB + +# Per-action input caps. A receiver can't make us do unbounded work +# from one webhook call: extra entries past the cap are silently +# dropped at parse time. +MAX_LABELS_PER_RESPONSE = 50 +MAX_ASSIGN_TO_PER_RESPONSE = 50 +MAX_EVENTS_PER_RESPONSE = 20 +MAX_IM_CONTENT_BYTES = 8 * 1024 # 8 KiB per internal-message comment + +PHASE_BEFORE_SPAM = "before_spam" +PHASE_AFTER_SPAM = "after_spam" +VALID_PHASES = frozenset({PHASE_BEFORE_SPAM, PHASE_AFTER_SPAM}) + + +@dataclass +class _HttpResult: + """Internal: one webhook call's outcome — decision + the side + effects the receiver asked us to apply to the pipeline context. + + The ``UserWebhookStep`` applies these to its ``InboundContext`` + and returns the decision; outside this file the type is invisible. + + Bool flag fields (``mark_starred`` / ``mark_read`` / ``mark_trashed`` + / ``mark_archived`` / ``skip_autoreply``) follow ``true``-only + semantics: a receiver returning ``true`` opts in; anything else + (``false``, missing, non-bool) is "no opinion". This makes the + multi-webhook merge a simple OR so a later receiver can't + accidentally veto an earlier receiver's directive. + """ + + decision: Decision = Decision.CONTINUE + is_spam_override: Optional[bool] = None + labels: Set[str] = field(default_factory=set) + # Ordered, lowercased, deduped — preserves the receiver's intent + # while letting the pipeline use cheap set/list operations downstream. + assign_to: List[str] = field(default_factory=list) + mark_starred: bool = False + mark_read: bool = False + mark_trashed: bool = False + mark_archived: bool = False + skip_autoreply: bool = False + # add_event: each entry is a validated dict ready to be persisted + # as a ThreadEvent. Currently only ``type=im`` is supported. + events: List[Dict[str, Any]] = field(default_factory=list) + # reply_draft: receiver-supplied MessageTemplate UUID. Resolved + + # scope-checked + drafted in the pipeline finalize step. + reply_draft_template_id: Optional[str] = None + + +# HTTP status families that mean "try again later" rather than +# "receiver rejected this message". 408 (timeout), 429 (rate limit), +# and 5xx are conventional retry signals across the webhook ecosystem +# (Stripe, GitHub, etc.). +_RETRY_STATUSES = frozenset({408, 429}) + + +def _read_capped_body(response) -> bytes: + """Read at most ``MAX_RESPONSE_BODY`` bytes from a streaming response. + + The action body contract is tiny (a few hundred bytes). Reading + more is wasted memory at best and a DoS vector at worst — if a + receiver returns a huge payload we keep what we have and ignore + the rest. Network errors mid-stream get logged and the caller + treats the partial body as if the receiver had returned no body. + """ + chunks: List[bytes] = [] + received = 0 + try: + for chunk in response.iter_content(chunk_size=8192, decode_unicode=False): + if not chunk: + continue + remaining = MAX_RESPONSE_BODY - received + if remaining <= 0: + break + if len(chunk) > remaining: + chunks.append(chunk[:remaining]) + received = MAX_RESPONSE_BODY + break + chunks.append(chunk) + received += len(chunk) + except Exception as exc: + logger.warning("Truncated response body read failed: %s", exc) + return b"".join(chunks) + + +def _failure(blocking: bool, decision: Decision) -> _HttpResult: + """Failure-path result: blocking → propagate ``decision`` (DROP / + RETRY); non-blocking → CONTINUE (fire-and-forget, never stalls + delivery).""" + return _HttpResult(decision=decision if blocking else Decision.CONTINUE) + + +def _classify_response_body(body_bytes: bytes) -> _HttpResult: + """Parse a 2xx response body into an ``_HttpResult``. + + Empty body or non-JSON body → plain CONTINUE. + + JSON shape (all keys optional): + - ``action``: ``"drop"`` short-circuits delivery; ``"retry"`` asks + us to re-queue the inbound task; anything else (``"accept"``, + missing) → CONTINUE. + - ``is_spam``: bool; overrides the pipeline's spam verdict. + - ``labels``: list of label UUID strings; the pipeline validates + them against the destination mailbox. + """ + if not body_bytes: + return _HttpResult() + try: + # ``json.loads`` accepts bytes natively and raises ValueError + # (incl. JSONDecodeError) on anything malformed. + payload = json.loads(body_bytes) + except ValueError: + return _HttpResult() + if not isinstance(payload, dict): + return _HttpResult() + + result = _HttpResult() + + action = payload.get("action") + if isinstance(action, str): + action = action.lower() + if action == "drop": + result.decision = Decision.DROP + elif action == "retry": + result.decision = Decision.RETRY + + is_spam = payload.get("is_spam") + if isinstance(is_spam, bool): + result.is_spam_override = is_spam + + labels = payload.get("labels") + if isinstance(labels, list): + for item in labels[:MAX_LABELS_PER_RESPONSE]: + if not isinstance(item, str): + continue + try: + # Normalise to canonical UUID string; rejects garbage + # before it ever hits the DB. + result.labels.add(str(uuid_module.UUID(item))) + except ValueError: + continue + + assign_to = payload.get("assign_to") + if isinstance(assign_to, list): + # Receiver-supplied OIDC emails. Light filter only: must be a + # non-empty string containing '@'. Lowercased + deduped while + # preserving order so a multi-email payload assigns in a + # predictable sequence. The pipeline does the real work + # (resolve to User, check editor rights, skip ambiguous). + seen: Set[str] = set() + for item in assign_to[:MAX_ASSIGN_TO_PER_RESPONSE]: + if not isinstance(item, str): + continue + email = item.strip().lower() + if not email or "@" not in email or email in seen: + continue + seen.add(email) + result.assign_to.append(email) + + # Bool flags. ``true``-only semantics — see ``_HttpResult``. + for key in ( + "mark_starred", + "mark_read", + "mark_trashed", + "mark_archived", + "skip_autoreply", + ): + if payload.get(key) is True: + setattr(result, key, True) + + reply_draft = payload.get("reply_draft") + if isinstance(reply_draft, dict): + candidate = reply_draft.get("template") + if isinstance(candidate, str): + try: + # Normalise to canonical UUID; rejects garbage before + # the DB lookup. + result.reply_draft_template_id = str(uuid_module.UUID(candidate)) + except ValueError: + pass + + add_event = payload.get("add_event") + if isinstance(add_event, list): + for item in add_event[:MAX_EVENTS_PER_RESPONSE]: + if not isinstance(item, dict): + continue + event_type = item.get("type") + if event_type == "im": + content = item.get("content") + if not isinstance(content, str) or not content.strip(): + continue + # Cap per-comment size — the comment is stored in + # ``ThreadEvent.data`` JSONB on every inbound, and we + # don't want a misconfigured receiver to flood the + # timeline with 60KB blobs. + if len(content.encode("utf-8")) > MAX_IM_CONTENT_BYTES: + content = content.encode("utf-8")[:MAX_IM_CONTENT_BYTES].decode( + "utf-8", errors="ignore" + ) + # Mirror the existing IM ThreadEvent shape so the + # pipeline can persist verbatim. ``mentions`` is + # intentionally empty for webhook-driven IMs: receivers + # don't know user UUIDs upfront and we don't want + # email-based mentions sneaking in here without the + # mention-notification semantics being designed. + result.events.append({"type": "im", "content": content, "mentions": []}) + # Unknown event types (incl. future ``type=iframe``) are + # silently dropped here — the classifier doesn't know how + # to validate them yet. Forward-compatible: new types + # become live the moment the classifier learns them, with + # no contract change for receivers that already emit them. + + return result + + +FORMAT_EML = "eml" +FORMAT_JMAP = "jmap" +# ``jmap_without_body`` is the cheap notification variant: same JMAP +# envelope (headers, from/to/subject, messageId, etc.) but no body +# parts, no bodyValues, no attachments. Receivers that only need the +# "a message arrived" signal can use it without ever seeing the body +# content over the wire. +FORMAT_JMAP_WITHOUT_BODY = "jmap_without_body" +VALID_FORMATS = frozenset({FORMAT_EML, FORMAT_JMAP, FORMAT_JMAP_WITHOUT_BODY}) +DEFAULT_FORMAT = FORMAT_EML + +USER_AGENT = "Messages-Webhook/1.0" + +# Signature scheme tag. Bumped when the algorithm changes so receivers +# can pin the version they accept. +SIGNATURE_SCHEME = "v1" + +# JWT in the Authorization header is a short-lived HS256 token covering +# the same envelope as the raw HMAC, intended for receivers that prefer +# a standard JWT verify path (e.g. n8n, Zapier, Make). +JWT_ISSUER = "messages-webhook" +JWT_TTL_SECONDS = 300 # 5 min — same window receivers should accept on the raw HMAC + + +def _resolve_body( + body_format: str, + raw_data: bytes, + parsed_email: JmapEmail, +) -> Tuple[str, bytes, Any]: + """Compute (Content-Type, raw bytes to sign, request kwarg). + + The dispatcher needs: + - the Content-Type to send, + - the exact byte string the signature is computed over, + - the kwarg (``data=`` or ``json=``) to hand off to ``requests``. + + JSON is serialised here once so the signature and the wire bytes + cannot drift (``requests`` would otherwise re-serialise with + different separators/ordering). + """ + if body_format == FORMAT_EML: + return "message/rfc822", raw_data, {"data": raw_data} + include_body = body_format == FORMAT_JMAP + payload = build_jmap_email(parsed_email, include_body=include_body) + # ``separators=(",", ":")`` produces the compact bytes we sign. + # Hand the same bytes to ``requests`` via ``data=`` so what we sign + # is exactly what we POST. + body_bytes = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode( + "utf-8" + ) + return "application/json", body_bytes, {"data": body_bytes} + + +def _sign(secret: str, timestamp: str, body_bytes: bytes) -> str: + """Stripe-style HMAC: HMAC-SHA256 over ``{timestamp}.{body}``. + + Returns hex digest. Receivers MUST compute the same and compare + constant-time, and SHOULD reject timestamps older than ~5 minutes. + """ + msg = timestamp.encode("ascii") + b"." + body_bytes + return hmac.new(secret.encode("utf-8"), msg, hashlib.sha256).hexdigest() + + +def _sign_jwt( + secret: str, + *, + channel: models.Channel, + mailbox: models.Mailbox, + body_bytes: bytes, + issued_at: int, +) -> str: + """Build an HS256 JWT for ``Authorization: Bearer …``. + + Claims: + - ``iss`` — fixed string so receivers can pin issuer. + - ``iat`` / ``exp`` — short TTL, prevents replay. + - ``jti`` — random nonce, for receivers that dedupe replays + beyond timestamp checks. + - ``sub`` — the destination mailbox (informational). + - ``cid`` — channel id (matches ``X-StMsg-Channel-Id``). + - ``body_sha256`` — hex SHA-256 of the request body. Lets the + receiver bind the JWT to the exact bytes posted, rather than + trusting transport. + + Encoded with HS256 using the channel's shared secret — receivers + verify with the same key. + """ + body_hash = hashlib.sha256(body_bytes).hexdigest() + claims = { + "iss": JWT_ISSUER, + "iat": issued_at, + "exp": issued_at + JWT_TTL_SECONDS, + "jti": secrets.token_urlsafe(16), + "sub": str(mailbox), + "cid": str(channel.id), + "body_sha256": body_hash, + } + return jwt.encode(claims, secret, algorithm="HS256") + + +# --- channel lookup --- # + + +def find_webhook_channels_for_mailbox( + mailbox: models.Mailbox, +) -> List[models.Channel]: + """Return every webhook Channel that fires for ``mailbox``. + + Includes: + - mailbox-scoped channels bound to this mailbox + - maildomain-scoped channels bound to this mailbox's domain + - global channels (instance-wide; admin/CLI-only to create) + + Phase filtering is done by the caller because the same channel set is + read twice (before- and after-spam). + """ + return list( + models.Channel.objects.filter( + Q(type=enums.ChannelTypes.WEBHOOK) + & ( + Q( + scope_level=enums.ChannelScopeLevel.MAILBOX, + mailbox=mailbox, + ) + | Q( + scope_level=enums.ChannelScopeLevel.MAILDOMAIN, + maildomain=mailbox.domain, + ) + | Q(scope_level=enums.ChannelScopeLevel.GLOBAL) + ) + ) + ) + + +# --- payload builders --- # + + +def _utcdate(value: Any) -> Optional[str]: + """Format a datetime as a JMAP ``UTCDate`` (RFC 3339 with ``Z`` suffix). + + Falls back to the raw value if it isn't a datetime — the parser may + have given us a pre-formatted string. ``None`` stays ``None``. + """ + if isinstance(value, datetime): + return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + return value + + +def _strip_body_part(part: Dict[str, Any]) -> Dict[str, Any]: + """A JMAP ``EmailBodyPart`` without our parser's project extensions. + + ``content`` (raw bytes on attachment parts) and ``sha256`` are + parser extensions, NOT RFC 8621. Attachment bytes in particular are + never embedded in the JSON body — JMAP keeps them behind a + ``blobId`` (which we don't have at webhook-fire time), and raw bytes + aren't JSON-serialisable anyway. Receivers that need the bytes use + ``format=eml``. + """ + return {k: v for k, v in part.items() if k not in ("content", "sha256")} + + +def build_jmap_email( + parsed_email: JmapEmail, *, include_body: bool = True +) -> Dict[str, Any]: + """Project the parsed JMAP ``Email`` object into the webhook payload. + + ``parse_email`` already returns a strict JMAP Email object + (RFC 8621 §4.1), so this is mostly a copy. We stamp ``receivedAt`` + (the moment the webhook fires) and strip the parser's project + extensions (``_ext`` and per-part ``content`` / ``sha256``) so the + body is strict JMAP. Storage-time fields (``id``, ``blobId``, + ``threadId``, ``mailboxIds``, ``keywords``) don't exist at + webhook-fire time and are simply absent. + + With ``include_body=False`` the body parts, ``bodyValues`` and + ``attachments`` are dropped — receivers get a notification-only + payload (subject + envelope addresses + headers) without the + message body content ever leaving the instance over the wire. + ``hasAttachment`` is preserved so receivers can still tell whether + the message had any. + """ + email: Dict[str, Any] = dict(parsed_email) + email.pop("_ext", None) # project extension, not strict JMAP + email["receivedAt"] = _utcdate(datetime.now(timezone.utc)) + + if include_body: + email["textBody"] = [ + _strip_body_part(p) for p in parsed_email.get("textBody") or [] + ] + email["htmlBody"] = [ + _strip_body_part(p) for p in parsed_email.get("htmlBody") or [] + ] + email["attachments"] = [ + _strip_body_part(p) for p in parsed_email.get("attachments") or [] + ] + else: + for key in ( + "textBody", + "htmlBody", + "bodyValues", + "bodyStructure", + "attachments", + ): + email.pop(key, None) + + return email + + +# --- envelope headers --- # + + +def _envelope_headers( + *, + channel: models.Channel, + phase: str, + mailbox: models.Mailbox, + recipient_email: str, + parsed_email: JmapEmail, + is_spam: Optional[bool], +) -> Dict[str, str]: + """Build the ``X-StMsg-*`` envelope headers attached to every webhook + POST regardless of body format. Same shape for ``eml`` and ``jmap``. + """ + if is_spam is None: + spam_value = "unknown" + else: + spam_value = "true" if is_spam else "false" + mid = first_msgid(parsed_email.get("messageId")) + headers = { + "User-Agent": USER_AGENT, + "X-StMsg-Event": enums.WebhookEvents.MESSAGE_RECEIVED.value, + "X-StMsg-Phase": phase, + "X-StMsg-Channel-Id": str(channel.id), + "X-StMsg-Mailbox": str(mailbox), + "X-StMsg-Recipient": recipient_email, + "X-StMsg-Is-Spam": spam_value, + } + if mid: + # Re-bracket per RFC 5322 for headers (the JMAP body strips them). + # Strip CR/LF defensively: the parser normally does, but a + # malformed inbound Message-Id with embedded newlines would let + # the receiver see "spliced" headers. urllib3 would also reject + # it, but we'd rather drop the dodgy bytes than fail the POST. + mid = mid.replace("\r", "").replace("\n", "").strip() + if mid: + if not (mid.startswith("<") and mid.endswith(">")): + mid = f"<{mid}>" + headers["X-StMsg-Message-Id"] = mid + return headers + + +# --- dispatch --- # + + +class UserWebhookStep: + """Pipeline ``Step`` wrapping one webhook ``Channel``. + + Each matching channel becomes its own step in the inbound pipeline + (one per phase). On call, the step POSTs the configured body to + the channel URL, classifies the response, applies any + ``is_spam`` override and ``labels`` to ``ctx``, and returns a + ``Decision``: + + - non-blocking → always ``CONTINUE`` (fire-and-forget; failures + only logged) + - blocking: + * 2xx + ``{"action":"drop"}`` → DROP + * 2xx + anything else → CONTINUE (with optional side effects) + * 4xx → DROP (receiver definitively rejected) + * 408 / 429 / 5xx → RETRY (transient) + * SSRF / missing secret / unknown auth_method → DROP + * timeout / connection / generic transport → RETRY + """ + + def __init__(self, channel: models.Channel, phase: str): + self.channel = channel + self.phase = phase + # Phase suffix in the name lets the task return value carry + # "which phase did this drop happen at" without a separate field. + self.name = f"webhook[{channel.id}]:{phase}" + + def __call__(self, ctx: InboundContext) -> Decision: + cfg = self.channel.settings or {} + url = cfg["url"] # validated at channel-create time + body_format = cfg.get("format", DEFAULT_FORMAT) + blocking = bool(cfg.get("blocking", False)) + + headers = _envelope_headers( + channel=self.channel, + phase=self.phase, + mailbox=ctx.mailbox, + recipient_email=ctx.recipient_email, + parsed_email=ctx.parsed_email, + is_spam=ctx.is_spam, + ) + result = self._post( + url=url, + body_format=body_format, + headers=headers, + ctx=ctx, + blocking=blocking, + ) + if result.is_spam_override is not None: + ctx.is_spam = result.is_spam_override + ctx.labels |= result.labels + if result.assign_to: + # Defer the actual assignment until after the thread + # exists (post-message-creation). Each blocking webhook + # that asked gets its own ThreadEvent ASSIGN, attributed + # to this channel. + ctx.pending_assigns.append((self.channel.id, result.assign_to)) + # Bool flags OR-merge: any blocking webhook saying true sticks. + ctx.mark_starred = ctx.mark_starred or result.mark_starred + ctx.mark_read = ctx.mark_read or result.mark_read + ctx.mark_trashed = ctx.mark_trashed or result.mark_trashed + ctx.mark_archived = ctx.mark_archived or result.mark_archived + ctx.skip_autoreply = ctx.skip_autoreply or result.skip_autoreply + for event in result.events: + # Per-event attribution like assigns — one ThreadEvent per + # add_event entry, with channel set to the firing webhook. + ctx.pending_events.append((self.channel.id, event)) + if result.reply_draft_template_id: + # Defer template lookup + draft creation until after the + # message + thread land. Each blocking webhook that asked + # produces its own draft, channel-attributed. + ctx.pending_drafts.append((self.channel.id, result.reply_draft_template_id)) + return result.decision + + def _post( + self, + *, + url: str, + body_format: str, + headers: Dict[str, str], + ctx: InboundContext, + blocking: bool, + ) -> _HttpResult: + content_type, body_bytes, body_kwargs = _resolve_body( + body_format, ctx.raw_data, ctx.parsed_email + ) + + secret = (self.channel.encrypted_settings or {}).get("secret") + if not secret: + # The create path always mints a secret; a row without one + # is misconfigured. Fail closed rather than POST something a + # receiver cannot authenticate. + logger.warning( + "Webhook channel %s has no secret — skipping", + self.channel.id, + ) + return _failure(blocking, Decision.DROP) + + auth_headers = self._build_auth_headers(secret, body_bytes, ctx.mailbox) + if auth_headers is None: + return _failure(blocking, Decision.DROP) + + signed_headers = { + **headers, + "Content-Type": content_type, + **auth_headers, + } + kwargs = {"headers": signed_headers, **body_kwargs} + + # ``stream=True`` lets us cap the response body we actually + # read — a misconfigured receiver returning a multi-GB error + # page must not OOM the worker. + try: + response = SSRFSafeSession().post( + url, timeout=WEBHOOK_TIMEOUT, stream=True, **kwargs + ) + except SSRFValidationError as exc: + # SSRF is a config error on our side — retrying won't fix it. + logger.warning( + "Webhook channel %s rejected by SSRF for url=%s: %s", + self.channel.id, + url, + exc, + ) + return _failure(blocking, Decision.DROP) + except Exception as exc: + # Timeout, connection refused, DNS, unknown transport-level + # failure: all transient on the blocking path. The 7-day cap + # in the pipeline runner bounds the retry budget. + logger.exception( + "Webhook channel %s network error for url=%s: %s", + self.channel.id, + url, + exc, + ) + return _failure(blocking, Decision.RETRY) + + try: + status = response.status_code + if 200 <= status < 300: + if not blocking: + # Non-blocking webhooks never influence delivery — + # ignore the body entirely. Avoids surprises if a + # receiver accidentally returns {"action":"drop"}. + return _HttpResult() + body_bytes_response = _read_capped_body(response) + result = _classify_response_body(body_bytes_response) + if result.decision == Decision.DROP: + logger.info( + "Webhook channel %s requested DROP via response body for url=%s", + self.channel.id, + url, + ) + return result + + logger.info( + "Webhook channel %s returned status %s for url=%s", + self.channel.id, + status, + url, + ) + if status in _RETRY_STATUSES or 500 <= status < 600: + return _failure(blocking, Decision.RETRY) + # 3xx and any other non-2xx non-retry code: receiver said no. + return _failure(blocking, Decision.DROP) + finally: + response.close() + + def _build_auth_headers( + self, + secret: str, + body_bytes: bytes, + mailbox: models.Mailbox, + ) -> Optional[Dict[str, str]]: + """Return the auth headers for the channel's ``auth_method``, + or ``None`` when the channel is misconfigured (caller fails + closed).""" + auth_method = (self.channel.settings or {}).get("auth_method") + + if auth_method == "api_key": + # Derived from the root secret via HMAC. The raw root + # never touches the wire — a receiver-side log leak of + # this value reveals nothing about the root, so HMAC/JWT + # verification remains unforgeable. + return {"X-StMsg-Api-Key": self.channel.get_webhook_api_key()} + + if auth_method == "jwt": + # HMAC signature over the body + short-TTL HS256 JWT, + # both keyed by the root secret. + now = int(time.time()) + timestamp = str(now) + signature = _sign(secret, timestamp, body_bytes) + bearer = _sign_jwt( + secret, + channel=self.channel, + mailbox=mailbox, + body_bytes=body_bytes, + issued_at=now, + ) + return { + "X-StMsg-Webhook-Timestamp": timestamp, + "X-StMsg-Webhook-Signature": f"{SIGNATURE_SCHEME}={signature}", + "Authorization": f"Bearer {bearer}", + } + + # Settings validator forbids creating a webhook channel without + # a valid auth_method; an existing row with a missing/unknown + # value is misconfigured. + logger.warning( + "Webhook channel %s has missing/unknown auth_method=%r — skipping", + self.channel.id, + auth_method, + ) + return None + + +def webhook_steps_for_mailbox(mailbox: models.Mailbox, *, phase: str) -> List[Step]: + """Build one ``UserWebhookStep`` per matching channel for the phase. + + Channels are filtered here (phase, events, url present, valid + format) rather than at run time so the pipeline iterator sees a + flat list of ready-to-call steps. + """ + if phase not in VALID_PHASES: + raise ValueError(f"Invalid webhook phase: {phase}") + + steps: List[Step] = [] + for channel in find_webhook_channels_for_mailbox(mailbox): + cfg = channel.settings or {} + if cfg.get("phase", PHASE_AFTER_SPAM) != phase: + continue + events = cfg.get("events") or [enums.WebhookEvents.MESSAGE_RECEIVED.value] + if enums.WebhookEvents.MESSAGE_RECEIVED.value not in events: + continue + if not cfg.get("url"): + continue + body_format = cfg.get("format", DEFAULT_FORMAT) + if body_format not in VALID_FORMATS: + # Serializer should have caught this on write — fail + # closed rather than POST in a shape the receiver wasn't + # promised. + logger.warning( + "Webhook channel %s has invalid format=%r — skipping", + channel.id, + body_format, + ) + continue + steps.append(UserWebhookStep(channel, phase=phase)) + return steps diff --git a/src/backend/core/mda/inbound_create.py b/src/backend/core/mda/inbound_create.py index 7ae1add88..d38c11341 100644 --- a/src/backend/core/mda/inbound_create.py +++ b/src/backend/core/mda/inbound_create.py @@ -211,6 +211,8 @@ def _create_message_from_inbound( # pylint: disable=too-many-arguments imap_flags: list[str] | None = None, channel: models.Channel | None = None, is_spam: bool = False, + is_trashed: bool = False, + is_archived: bool = False, is_outbound: bool = False, ) -> models.Message | None: """Create a message and thread from parsed email data. @@ -432,7 +434,8 @@ def _create_message_from_inbound( # pylint: disable=too-many-arguments sent_at=(None if is_outbound else (sent_at or timezone.now())), is_draft=is_outbound, # Outbound: draft until prepare_outbound_message finalizes is_sender=is_sender, - is_trashed=False, + is_trashed=is_trashed, + is_archived=is_archived, is_spam=is_spam, has_attachments=len(parsed_email.get("attachments", [])) > 0, channel=channel, diff --git a/src/backend/core/mda/inbound_pipeline.py b/src/backend/core/mda/inbound_pipeline.py new file mode 100644 index 000000000..a51216df7 --- /dev/null +++ b/src/backend/core/mda/inbound_pipeline.py @@ -0,0 +1,675 @@ +"""Inbound-message processing pipeline. + +Every "thing we do with an incoming message before it lands as a +``Message`` row" is a **Step**: a callable that takes an +``InboundContext`` and returns a ``Decision``. Steps may also mutate +the context — set ``is_spam``, add ``labels``, cache ``rspamd_result``, +prepend an authentication header, etc. + + pipeline = [ + *user_webhook_steps(mailbox, phase="before_spam"), + hardcoded_rules_step(spam_config), + rspamd_step(spam_config), + inbound_auth_step(spam_config), + *user_webhook_steps(mailbox, phase="after_spam"), + ] + for step in pipeline: + d = step(ctx) + if d != Decision.CONTINUE: + break + +The orchestrator (``run_inbound_pipeline``) iterates and aborts on the +first ``DROP`` / ``RETRY``. The caller turns that decision into a +task-level return value. + +This file deliberately knows nothing about HTTP, JWT, or JMAP — those +live in ``dispatch_webhooks.py`` behind ``UserWebhookStep``. The +pipeline only sees the uniform ``Step → Decision`` interface. +""" + +# pylint: disable=broad-exception-caught + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass, field +from datetime import timedelta +from enum import IntEnum +from typing import Any, Callable, Dict, List, Optional, Set, Tuple + +from django.db.models import Q +from django.db.models.functions import Lower +from django.utils import timezone + +import requests +from jmap_email import JmapEmail, has_header, parse_email + +from core import enums, models +from core.mda.inbound_auth import ( + check_inbound_authentication, + get_inbound_auth_mode, +) +from core.mda.utils import headers_blocks +from core.services.thread_events import assign_users + +logger = logging.getLogger(__name__) + + +class Decision(IntEnum): + """Step control-flow signal. + + Ordered ``DROP > RETRY > CONTINUE``. The pipeline aborts on the + first non-CONTINUE. + """ + + CONTINUE = 0 + RETRY = 1 + DROP = 2 + + +@dataclass +class InboundContext: + """Mutable bag of state flowing through the pipeline. + + Steps read what they need and write what they decide. The post-loop + finalizer reads the final values (``is_spam``, ``labels``, + ``parsed_email``, ``raw_data``) to build the ``Message`` row. + """ + + mailbox: models.Mailbox + inbound_message: models.InboundMessage + recipient_email: str + raw_data: bytes + parsed_email: JmapEmail + spam_config: Dict[str, Any] + + # Verdict, accumulated across steps: + # - None: undecided (no spam step has run, or none had an opinion) + # - True/False: the last decisive step wins + is_spam: Optional[bool] = None + + # Labels webhook receivers have asked us to attach to the thread. + # Validated against the destination mailbox at finalize time; + # unknown UUIDs are dropped silently. + labels: Set[str] = field(default_factory=set) + + # Deferred per-channel assign requests from blocking webhooks. Each + # entry is ``(channel_id, [oidc_email, ...])`` — applied AFTER the + # message + thread exist, one ``ThreadEvent ASSIGN`` per entry so + # the audit trail keeps each channel's contribution separate. + pending_assigns: List[Tuple[Any, List[str]]] = field(default_factory=list) + + # Deferred per-channel ThreadEvents to create after the thread + # exists. Each entry is ``(channel_id, event_dict)`` — currently + # only ``type=im`` events flow here, but the structure is + # forward-compatible for future event types (e.g. ``iframe``). + pending_events: List[Tuple[Any, Dict[str, Any]]] = field(default_factory=list) + + # Deferred per-channel ``reply_draft`` requests. Each entry is + # ``(channel_id, template_id)`` — applied AFTER message + thread + # exist; resolves the template (scope-checked against the mailbox / + # maildomain) and materialises one draft Message per entry via + # the autoreply path's shared record helper. + pending_drafts: List[Tuple[Any, str]] = field(default_factory=list) + + # Blocking-webhook flag actions (OR-merged across webhooks). All + # default to False and are only ever flipped to True by a + # receiver explicitly opting in via the JSON action body. The + # task body applies them to ThreadAccess / Message / autoreply + # after the message is created. + mark_starred: bool = False + mark_read: bool = False + mark_trashed: bool = False + mark_archived: bool = False + skip_autoreply: bool = False + + # Populated by ``rspamd_step`` so ``inbound_auth_step`` can reuse + # the symbols (DKIM/DMARC verdicts) without a second HTTP call. + rspamd_result: Optional[Dict[str, Any]] = None + + +# A Step is just a callable. It MUST have a ``.name`` attribute so +# logs and the task return value can report which step aborted. +Step = Callable[[InboundContext], Decision] + + +# Inbound messages held by a transient RETRY get one more chance every +# 5 minutes via ``process_inbound_messages_queue_task``. After this cap +# we drop and log loudly so a permanently-broken receiver can't pin a +# row in the queue forever. +RETRY_MAX_AGE = timedelta(days=7) + + +# --------------------------------------------------------------------------- +# Spam-check helpers shared by the steps below. +# --------------------------------------------------------------------------- + + +def _check_hardcoded_rules( + parsed_email: JmapEmail, spam_config: Dict[str, Any] +) -> Optional[bool]: + """Apply the per-domain hardcoded ``rules`` list, header-matched + only against headers from trusted relay blocks. Returns ``True`` / + ``False`` on first matching rule, ``None`` if no rule matched.""" + rules = spam_config.get("rules", []) + for rule in rules: + header_match = rule.get("header_match") or rule.get("header_match_regex") + if not header_match: + continue + if ":" not in header_match: + logger.warning( + "Invalid header_match format (missing colon): %s", header_match + ) + continue + key, value = header_match.split(":", 1) + key = key.lower().strip() + value = value.lower().strip() + + # Existence check first; the trusted value is read from the + # Received-bounded blocks below. + if not has_header(parsed_email, key): + continue + + # Trust window is "block 0 (our MTA's Received + headers above it) + # + N upstream relay blocks". Default 0: a sender can prepend + # their own Received lines (landing in block 1+), so trusting + # those by default would let them forge an allowlist match. + # Slicing beyond list length is fine — yields all blocks. + trusted_relays = spam_config.get("trusted_relays", 0) + blocks_to_check = trusted_relays + 1 + found_value = None + for block in headers_blocks(parsed_email)[:blocks_to_check]: + if key in block and block[key]: + # Blocks are ordered most-recent → oldest; first match wins. + found_value = block[key][0] + break + if found_value is None: + continue + + header_value = ( + found_value.lower().strip() + if isinstance(found_value, str) + else str(found_value).lower().strip() + ) + if rule.get("header_match"): + is_match = header_value == value + else: # header_match_regex + is_match = re.fullmatch(value, header_value) is not None + if is_match: + action = rule.get("action") or "spam" + if action in ("spam", "reject"): + return True + if action in ("ham", "no action"): + return False + return None + + +def _call_rspamd( + raw_data: bytes, spam_config: Dict[str, Any] +) -> Tuple[Optional[bool], Optional[str], Optional[Dict[str, Any]]]: + """POST raw RFC-822 bytes to rspamd's ``/checkv2``. + + Returns ``(is_spam_or_None, error_message, result_dict)``. is_spam + is ``None`` only when rspamd is not configured. Errors are + swallowed and surfaced via the error_message channel so a flaky + rspamd never blocks delivery (mirroring the old behaviour). + """ + url = spam_config.get("rspamd_url") + if not url: + logger.debug("SPAM_CONFIG.rspamd_url not configured, skipping rspamd") + return None, None, None + + headers = {"Content-Type": "message/rfc822"} + auth = spam_config.get("rspamd_auth") + if auth: + headers["Authorization"] = auth + + try: + response = requests.post( + f"{url}/checkv2", data=raw_data, headers=headers, timeout=10 + ) + response.raise_for_status() + result = response.json() + except (requests.exceptions.RequestException, ValueError) as exc: + # Network failures, non-2xx (raise_for_status), and a non-JSON + # body (ValueError covers JSONDecodeError) all funnel here. We + # don't let a flaky rspamd block delivery — fall through with + # is_spam=False so the pipeline keeps moving. The + # error_message channel lets the caller log loudly. + logger.exception("Error calling rspamd: %s", exc) + return False, str(exc), None + except Exception as exc: + logger.exception("Unexpected error calling rspamd: %s", exc) + return False, str(exc), None + + if not isinstance(result, dict): + logger.warning("rspamd returned non-object body: %r", result) + return False, "rspamd returned non-object body", None + + action = result.get("action", "") + score = result.get("score", 0.0) + required = result.get("required_score", 15.0) + is_spam = action == "reject" + logger.info( + "Rspamd: action=%s score=%.2f required=%.2f is_spam=%s", + action, + score, + required, + is_spam, + ) + return is_spam, None, result + + +# --------------------------------------------------------------------------- +# Steps. Each is callable as ``step(ctx) -> Decision`` and carries a +# ``.name`` for log/return-value reporting. +# --------------------------------------------------------------------------- + + +def _make_hardcoded_rules_step(spam_config: Dict[str, Any]) -> Step: + def hardcoded_rules(ctx: InboundContext) -> Decision: + if ctx.is_spam is not None: + return Decision.CONTINUE + verdict = _check_hardcoded_rules(ctx.parsed_email, spam_config) + if verdict is not None: + ctx.is_spam = verdict + return Decision.CONTINUE + + hardcoded_rules.name = "hardcoded_rules" + return hardcoded_rules + + +def _make_rspamd_step(spam_config: Dict[str, Any]) -> Step: + """Rspamd as a step. + + Sets ``is_spam`` if no earlier step decided. Always caches the + full ``rspamd_result`` dict on the context — ``inbound_auth_step`` + reuses the symbols (DKIM/DMARC) without a second HTTP call. + """ + + def rspamd(ctx: InboundContext) -> Decision: + if ctx.is_spam is not None: + # Spam verdict already decided — but we still might want + # rspamd's symbols for inbound_auth. The auth step has its + # own fallback so we can cheaply skip rspamd entirely here. + return Decision.CONTINUE + is_spam, err, result = _call_rspamd(ctx.raw_data, spam_config) + if err: + logger.warning( + "rspamd error on inbound message %s: %s (treating as not spam)", + ctx.inbound_message.id, + err, + ) + ctx.rspamd_result = result + if is_spam is not None: + ctx.is_spam = is_spam + return Decision.CONTINUE + + rspamd.name = "rspamd" + return rspamd + + +def _make_inbound_auth_step(spam_config: Dict[str, Any]) -> Step: + """DKIM / DMARC verdict via ``check_inbound_authentication``. + + Reuses ``ctx.rspamd_result`` if populated; otherwise calls rspamd + itself when ``auth_mode='rspamd'``. On a verdict, prepends an + ``X-StMsg-Sender-Auth`` header to both ``raw_data`` and + ``parsed_email`` so subsequent steps + downstream consumers see it. + """ + + def inbound_auth(ctx: InboundContext) -> Decision: + if ctx.rspamd_result is None and get_inbound_auth_mode(spam_config) == "rspamd": + _, _, ctx.rspamd_result = _call_rspamd(ctx.raw_data, spam_config) + verdict = check_inbound_authentication( + ctx.raw_data, ctx.parsed_email, spam_config, ctx.rspamd_result + ) + if not verdict: + return Decision.CONTINUE + prepended = f"X-StMsg-Sender-Auth: {verdict}\r\n".encode("ascii") + ctx.raw_data + reparsed = parse_email(prepended) + if reparsed is not None: + ctx.parsed_email = reparsed + ctx.raw_data = prepended + else: + # Keep raw_data / parsed_email in lockstep: if re-parse breaks, + # we sacrifice the auth banner rather than corrupting the blob. + logger.warning("Failed to re-parse after prepending X-StMsg-Sender-Auth") + return Decision.CONTINUE + + inbound_auth.name = "inbound_auth" + return inbound_auth + + +# --------------------------------------------------------------------------- +# Pipeline construction + runner. +# --------------------------------------------------------------------------- + + +def build_inbound_pipeline(ctx: InboundContext) -> List[Step]: + """Standard pipeline for an inbound message. + + Order matters: + 1. Before-spam user webhooks — may DROP, RETRY, or set is_spam. + 2. ``hardcoded_rules`` — header-match rules per domain config. + 3. ``rspamd`` — fills the gap if nothing decided spam yet, and + caches symbols for the next step. + 4. ``inbound_auth`` — DKIM / DMARC verdict, may mutate parsed_email. + 5. After-spam user webhooks — see the verdict, may override it, + may add labels, may DROP/RETRY. + """ + # Imported here to avoid the inbound_pipeline ↔ dispatch_webhooks + # cycle: webhook_steps_for_mailbox lives next to UserWebhookStep + # because it instantiates one per matching channel. + from core.mda.dispatch_webhooks import ( # pylint: disable=import-outside-toplevel + webhook_steps_for_mailbox, + ) + + return [ + *webhook_steps_for_mailbox(ctx.mailbox, phase="before_spam"), + _make_hardcoded_rules_step(ctx.spam_config), + _make_rspamd_step(ctx.spam_config), + _make_inbound_auth_step(ctx.spam_config), + *webhook_steps_for_mailbox(ctx.mailbox, phase="after_spam"), + ] + + +def run_inbound_pipeline( + pipeline: List[Step], ctx: InboundContext +) -> Tuple[Decision, Optional[str]]: + """Iterate the pipeline. Stop on the first non-CONTINUE decision. + + Returns ``(final_decision, aborting_step_name_or_None)``. The + caller turns that into a Celery-task return value. + """ + for step in pipeline: + decision = step(ctx) + if decision != Decision.CONTINUE: + return decision, getattr(step, "name", step.__class__.__name__) + return Decision.CONTINUE, None + + +# --------------------------------------------------------------------------- +# Finalisation: label application. +# --------------------------------------------------------------------------- + + +def apply_labels_to_thread( + thread: models.Thread, mailbox: models.Mailbox, label_ids: Set[str] +) -> None: + """Attach pipeline-collected labels to a thread. + + Each id is validated against the destination mailbox: unknown + UUIDs are logged and skipped — a misbehaving webhook receiver + must not stall delivery. Label IDs are already UUID-validated + upstream (in the webhook response classifier). + """ + for label_id in label_ids: + try: + label_obj = models.Label.objects.get(id=label_id, mailbox=mailbox) + except models.Label.DoesNotExist: + logger.warning( + "Pipeline label %s not found for mailbox %s — skipping", + label_id, + mailbox.id, + ) + continue + thread.labels.add(label_obj) + + +def _resolve_assignable_users( + thread: models.Thread, emails: List[str] +) -> List[Dict[str, Any]]: + """Resolve OIDC emails → user dicts ready for ``assign_users``. + + A single SQL query case-folds both sides (``Lower("email")``) and + fetches all matching users at once — no N+1. Ambiguity (≥2 users + sharing one email) and unknown emails are logged and skipped. + NEVER auto-creates users: a webhook receiver must not be able to + pollute the ``User`` table. + + The survivors are then filtered to users that currently hold one + of the assignable mailbox roles on this thread (editor / sender / + admin) via ``ThreadAccess.editor_user_ids`` — viewers can't be + assigned, matching the API rule. + """ + if not emails: + return [] + + # The input is already lowercased + deduped by the classifier; + # belt-and-suspenders ``set()`` here in case a future caller + # forgets. + target_emails = {e.lower() for e in emails if e} + if not target_emails: + return [] + + matches = list( + models.User.objects.annotate(_lemail=Lower("email")) + .filter(_lemail__in=target_emails) + .only("id", "email", "full_name") + ) + + # Group by lowercased email to detect ambiguity per address. + by_email: Dict[str, List[models.User]] = {} + for user in matches: + key = (user.email or "").lower() + by_email.setdefault(key, []).append(user) + + candidate_ids: List[Any] = [] + candidate_users: Dict[Any, models.User] = {} + for email in target_emails: + bucket = by_email.get(email) or [] + if not bucket: + logger.warning( + "Webhook assignee email %s does not resolve to any user — skipping", + email, + ) + continue + if len(bucket) > 1: + logger.warning( + "Webhook assignee email %s is ambiguous (multiple matches) — skipping", + email, + ) + continue + user = bucket[0] + if user.id in candidate_users: + continue + candidate_users[user.id] = user + candidate_ids.append(user.id) + + if not candidate_ids: + return [] + + assignable_ids = set( + models.ThreadAccess.objects.editor_user_ids(thread.id, user_ids=candidate_ids) + ) + for uid in candidate_ids: + if uid not in assignable_ids: + logger.warning( + "Webhook assignee %s lacks an assignable role on the thread — skipping", + candidate_users[uid].email, + ) + + return [ + {"id": str(uid), "name": candidate_users[uid].full_name or ""} + for uid in candidate_ids + if uid in assignable_ids + ] + + +def apply_thread_access_flags( + thread: models.Thread, + mailbox: models.Mailbox, + *, + mark_starred: bool, + mark_read: bool, +) -> None: + """Apply per-mailbox flag toggles to the destination ThreadAccess. + + ``mark_starred`` sets ``starred_at`` to now; ``mark_read`` sets + ``read_at`` to now. Both are idempotent — re-applying doesn't + unstar / unread — and both are no-ops when the corresponding bool + is False. The ``ThreadAccess`` row may not exist if the destination + mailbox doesn't have one yet (rare: brand-new thread, race with + deletion); in that case we log and skip rather than fail delivery. + """ + if not (mark_starred or mark_read): + return + access = models.ThreadAccess.objects.filter(thread=thread, mailbox=mailbox).first() + if access is None: + logger.warning( + "ThreadAccess missing for thread %s / mailbox %s — " + "skip mark_starred/mark_read", + thread.id, + mailbox.id, + ) + return + update_fields: List[str] = [] + now = timezone.now() + if mark_starred and access.starred_at is None: + access.starred_at = now + update_fields.append("starred_at") + if mark_read and access.read_at is None: + access.read_at = now + update_fields.append("read_at") + if update_fields: + access.save(update_fields=update_fields) + + +def apply_pending_drafts( + inbound_msg: models.Message, + mailbox: models.Mailbox, + pending: List[Tuple[Any, str]], +) -> None: + """Materialise webhook-driven reply drafts. + + For each ``(channel_id, template_id)`` entry: look up the + ``MessageTemplate`` scoped to the destination mailbox or its + maildomain (out-of-scope templates are silently skipped — a + webhook receiver mustn't be able to draft from another mailbox's + template). Then delegate to ``create_draft_reply_from_template``, + which shares its record-creation path with the autoreply flow and + stores the template's editor-format body as ``draft_blob`` so the + user can refine the draft inline. + """ + # Inline: autoreply → outbound → inbound → inbound_tasks → + # inbound_pipeline is a real import cycle, so this one import can't + # move to the top. + from core.mda.autoreply import ( # pylint: disable=import-outside-toplevel + create_draft_reply_from_template, + ) + + for channel_id, template_id in pending: + template = ( + models.MessageTemplate.objects.filter( + Q(mailbox=mailbox) | Q(maildomain=mailbox.domain), + id=template_id, + type=enums.MessageTemplateTypeChoices.MESSAGE, + is_active=True, + ) + .select_related("blob", "signature__blob") + .first() + ) + if template is None: + logger.warning( + "Webhook reply_draft template %s not found or out of scope " + "for mailbox %s — skipping", + template_id, + mailbox.id, + ) + continue + try: + channel = models.Channel.objects.get(id=channel_id) + except models.Channel.DoesNotExist: + logger.warning( + "Webhook channel %s vanished before reply_draft could land — skipping", + channel_id, + ) + continue + create_draft_reply_from_template( + template, + mailbox, + inbound_msg, + channel=channel, + ) + + +def apply_pending_events( + thread: models.Thread, pending: List[Tuple[Any, Dict[str, Any]]] +) -> None: + """Persist webhook-driven ``ThreadEvent`` rows. + + One row per ``(channel_id, event_dict)`` pair — preserves per- + receiver attribution. Today only ``type=im`` events arrive here + (the classifier dropped unknown types); future types just need + their dispatch case added without touching the contract. + """ + for channel_id, event in pending: + event_type = event.get("type") + if event_type != enums.ThreadEventTypeChoices.IM: + logger.warning("Unknown pending event type %r — skipping", event_type) + continue + try: + channel = models.Channel.objects.get(id=channel_id) + except models.Channel.DoesNotExist: + logger.warning( + "Webhook channel %s vanished before event could land — skipping", + channel_id, + ) + continue + models.ThreadEvent.objects.create( + thread=thread, + author=None, + channel=channel, + type=enums.ThreadEventTypeChoices.IM, + data={ + "content": event["content"], + "mentions": event.get("mentions", []), + }, + ) + + +def apply_pending_assigns( + thread: models.Thread, pending: List[Tuple[Any, List[str]]] +) -> None: + """Replay the per-channel deferred assigns into ``ThreadEvent``s. + + One ``assign_users()`` call per (channel, emails) tuple → one + ``ThreadEvent ASSIGN`` per webhook that asked. The service's + idempotence (partial UniqueConstraint on UserEvent) absorbs a + later webhook re-asking for an already-assigned user, so the + first-to-ask is the canonical attribution. + """ + for channel_id, emails in pending: + assignees_data = _resolve_assignable_users(thread, emails) + if not assignees_data: + continue + try: + channel = models.Channel.objects.get(id=channel_id) + except models.Channel.DoesNotExist: + # The webhook channel was deleted between dispatch and + # finalize (admin churn during processing). Skip the + # assign rather than half-attribute it to a dead row. + logger.warning( + "Webhook channel %s vanished before assign could land — skipping", + channel_id, + ) + continue + try: + assign_users( + thread=thread, + author=None, + assignees_data=assignees_data, + channel=channel, + ) + except ValueError as exc: + # Editor-rights check inside the service. We already + # pre-filtered, so this shouldn't fire — but if a race + # invalidated the rights between filter and service call, + # don't blow up delivery over it. + logger.warning( + "assign_users skipped %d assignee(s) due to race: %s", + len(assignees_data), + exc, + ) diff --git a/src/backend/core/mda/inbound_tasks.py b/src/backend/core/mda/inbound_tasks.py index bb83c1ebd..f4857481d 100644 --- a/src/backend/core/mda/inbound_tasks.py +++ b/src/backend/core/mda/inbound_tasks.py @@ -1,239 +1,144 @@ -"""Message delivery and processing tasks.""" +"""Message delivery and processing tasks. -# pylint: disable=unused-argument, broad-exception-raised, broad-exception-caught, too-many-lines +Per-message processing is a pipeline of ``Step``s — see +``inbound_pipeline.py``. This module is the Celery task wrapper: +acquire a Redis lock, parse the bytes, build the context + pipeline, +iterate, and turn the final ``Decision`` into a task return value. +""" -import re -from typing import Any +# pylint: disable=unused-argument, broad-exception-raised, broad-exception-caught + +from typing import Any, Dict, Optional from django.conf import settings from django.core.cache import cache from django.utils import timezone -import requests from celery.utils.log import get_task_logger -from jmap_email import ( - JmapEmail, - first_address_email, - has_header, - parse_email, -) +from jmap_email import JmapEmail, first_address_email, parse_email from core import models -from core.mda.inbound_auth import ( - check_inbound_authentication, - get_inbound_auth_mode, -) from core.mda.inbound_create import _create_message_from_inbound -from core.mda.utils import headers_blocks +from core.mda.inbound_pipeline import ( + RETRY_MAX_AGE, + Decision, + InboundContext, + apply_labels_to_thread, + apply_pending_assigns, + apply_pending_drafts, + apply_pending_events, + apply_thread_access_flags, + build_inbound_pipeline, + run_inbound_pipeline, +) from messages.celery_app import app as celery_app logger = get_task_logger(__name__) -def _is_selfcheck_message(parsed_email: JmapEmail, recipient_email: str) -> bool: - """Return True when this message is the self-check probe. +def _is_selfcheck(parsed_email: JmapEmail, recipient_email: str) -> bool: + """Strict envelope match for the configured self-probe. - Match is strict on both envelope ends: the From address must equal - MESSAGES_SELFCHECK_FROM and the recipient must equal MESSAGES_SELFCHECK_TO. + The self-probe is an internal liveness check sent from + ``MESSAGES_SELFCHECK_FROM`` to ``MESSAGES_SELFCHECK_TO``. We short- + circuit spam checking for it so the probe is never junked, but it + still flows through the rest of the pipeline (inbound auth, after- + spam webhooks, message creation). """ selfcheck_from = (settings.MESSAGES_SELFCHECK_FROM or "").strip().lower() selfcheck_to = (settings.MESSAGES_SELFCHECK_TO or "").strip().lower() - if not selfcheck_from or not selfcheck_to: return False from_email = first_address_email(parsed_email.get("from")).strip().lower() if from_email != selfcheck_from: return False - return (recipient_email or "").strip().lower() == selfcheck_to -def _check_spam_with_hardcoded_rules( - parsed_email: JmapEmail, spam_config: dict[str, Any] -) -> bool | None: - """Check if a message is spam using hardcoded rules. +def _safe_finalize(label, inbound_message_id, gate, fn): + """Run one finalize step under an isolated try/except. - Args: - parsed_email: Parsed email message - spam_config: Spam configuration + ``gate`` short-circuits the call when the input collection is + empty/false — same semantics as the inline ``if ctx.labels:`` + guards, just lifted out. Exceptions are logged but never + propagated: the message has already landed; failing the whole + task here would only confuse operators.""" + if not gate: + return + try: + fn() + except Exception as exc: # pylint: disable=broad-exception-caught + logger.exception( + "Finalize step %r failed on inbound message %s: %s", + label, + inbound_message_id, + exc, + ) - Returns: - is_spam: True if the message is spam, False otherwise. None if no rules matched. - """ - rules = spam_config.get("rules", []) - - for rule in rules: - if rule.get("header_match") or rule.get("header_match_regex"): - # Split on first colon only, in case value contains colons - header_match = rule.get("header_match") or rule.get("header_match_regex") - if ":" not in header_match: - logger.warning( - "Invalid header_match format (missing colon): %s", header_match - ) - continue - - key, value = header_match.split(":", 1) - key = key.lower().strip() - value = value.lower().strip() - - # Existence check first — the actual value is read from - # ``headersBlocks`` below to apply the trusted-relays cut. - if not has_header(parsed_email, key): - continue - - # Use ``ext.headersBlocks`` to identify which headers to - # trust based on the trusted_relays config. Each block ends - # with a Received header, marking everything above it as - # trusted. - # Block 0: headers before first Received (ours from MTA), - # ending with first Received. - # Block 1: headers between first and second Received, - # ending with second Received (relay 1). - # Block 2+: headers after second Received, ending with - # third Received (relay 2+). - blocks = headers_blocks(parsed_email) - - # Default trusted_relays = 0: trust only block 0 (the Received our - # own MTA prepends, plus the headers above it). A sender can prepend - # their own Received lines, which land in block 1+ — trusting those - # by default would let them slip a forged header (e.g. an - # action="ham" allowlist match) into the trusted slice. Operators - # with real upstream relays opt in by setting trusted_relays to the - # number of hops they actually control. - trusted_relays = spam_config.get("trusted_relays", 0) - # block 0 (our Received) + trusted_relays upstream blocks. - blocks_to_check = trusted_relays + 1 - - # Check only the trusted blocks (slicing beyond list length - # just returns all blocks). Blocks are ordered most recent - # to oldest, so we want the first match (most recent). - found_value = None - for block in blocks[:blocks_to_check]: - if key in block: - block_value = block[key] - # Values inside a block are always lists; first entry - # is the most recent occurrence within that block. - if block_value: - found_value = block_value[0] - break - - if found_value is None: - continue - header_value = found_value - - # Normalize header value for comparison - if isinstance(header_value, str): - header_value = header_value.lower().strip() - else: - header_value = str(header_value).lower().strip() - - if rule.get("header_match"): - is_match = header_value == value - elif rule.get("header_match_regex"): - is_match = re.fullmatch(value, header_value) is not None - else: - raise ValueError("Invalid header match type") - - # Check if header matches - if is_match: - action = rule.get("action") or "spam" - if action in ["spam", "reject"]: - return True - if action in ["ham", "no action"]: - return False - - return None - - -def _check_spam_with_rspamd( - raw_data: bytes, spam_config: dict[str, Any] -) -> tuple[bool, str | None, dict[str, Any] | None]: - """Check if a message is spam using rspamd. - Args: - raw_data: Raw email message bytes - spam_config: Spam configuration +def _handle_retry( + inbound_message: models.InboundMessage, step_name: Optional[str] +) -> Dict[str, Any]: + """Translate a RETRY decision into the task return value. - Returns: - Tuple of (is_spam, error_message, rspamd_result). error_message is None on - success. rspamd_result is the full parsed JSON response when available, so - other inspectors (e.g. inbound auth) can reuse symbols without re-querying. + The InboundMessage row is kept in place unless it's older than + ``RETRY_MAX_AGE`` — the 5-min sweep + (``process_inbound_messages_queue_task``) re-fires the task on the + next cycle. Past the budget we drop and log loudly so a + permanently-broken receiver can't pin a row forever. """ - - spam_url = spam_config.get("rspamd_url") - if not spam_url: - # If rspamd is not configured, treat all messages as not spam - logger.debug("SPAM_CONFIG.rspamd_url not configured, skipping spam check") - return False, None, None - - try: - headers = {"Content-Type": "message/rfc822"} - spam_auth = spam_config.get("rspamd_auth") - if spam_auth: - headers["Authorization"] = spam_auth - - response = requests.post( - f"{spam_url}/checkv2", - data=raw_data, - headers=headers, - timeout=10, - ) - response.raise_for_status() - - result = response.json() - # rspamd returns action: "reject", "add header", "greylist", or "no action" - # We consider it spam if action is "reject" - action = result.get("action", "") - score = result.get("score", 0.0) - required_score = result.get("required_score", 15.0) - - is_spam = action == "reject" - - logger.info( - "Rspamd check result: action=%s, score=%.2f, required=%.2f, is_spam=%s", - action, - score, - required_score, - is_spam, + age = timezone.now() - inbound_message.created_at + if age > RETRY_MAX_AGE: + logger.error( + "Inbound message %s exceeded retry budget (%s old) — dropping at step=%s", + inbound_message.id, + age, + step_name, ) - - return is_spam, None, result - - except requests.exceptions.RequestException as e: - logger.exception("Error checking spam with rspamd: %s", e) - # On error, treat as not spam to avoid blocking legitimate messages - return False, str(e), None - except Exception as e: - logger.exception("Unexpected error checking spam with rspamd: %s", e) - return False, str(e), None + inbound_message.delete() + return { + "success": False, + "inbound_message_id": str(inbound_message.id), + "error": "retry_exhausted", + "step": step_name, + } + logger.info( + "Inbound message %s held for retry at step=%s (age=%s)", + inbound_message.id, + step_name, + age, + ) + return { + "success": False, + "inbound_message_id": str(inbound_message.id), + "error": "retry", + "step": step_name, + } @celery_app.task(bind=True) def process_inbound_message_task(self, inbound_message_id: str): - """Process an inbound message from the queue: check spam and create message. - - Args: - inbound_message_id: The ID of the InboundMessage to process + """Process an inbound message: run the pipeline, persist the result. - Returns: - dict: A dictionary with success status and info + Returns ``{"success": ...}`` so the 5-min retry sweep can tell which + messages still need work. On DROP, the ``InboundMessage`` row is + deleted (we're done with it) and the task reports success. """ - # Create a unique lock key for this inbound message to prevent double processing + # Redis lock keyed on the message id prevents two workers from + # racing on the same row. Auto-expires after 5 min so a hung worker + # doesn't block the next sweep. lock_key = f"process_inbound_message_lock:{inbound_message_id}" - lock_timeout = 300 # 5 minutes timeout for the lock - - # Try to acquire the lock - if not cache.add(lock_key, "locked", lock_timeout): + if not cache.add(lock_key, "locked", 300): logger.warning( - "InboundMessage %s is already being processed by another worker, skipping duplicate processing", + "InboundMessage %s is already being processed — skipping", inbound_message_id, ) return {"success": False, "error": "Message already being processed"} + inbound_message: Optional[models.InboundMessage] = None try: - inbound_message = None try: inbound_message = models.InboundMessage.objects.get(id=inbound_message_id) except models.InboundMessage.DoesNotExist: @@ -241,11 +146,6 @@ def process_inbound_message_task(self, inbound_message_id: str): logger.error(error_msg) return {"success": False, "error": error_msg} - # Redis lock prevents concurrent processing, no need to mark as PROCESSING - mailbox = inbound_message.mailbox - recipient_email = str(mailbox) # Use mailbox email as recipient_email - - # Parse the email from raw_data raw_data_bytes = bytes(inbound_message.raw_data) parsed_email = parse_email(raw_data_bytes) if parsed_email is None: @@ -253,93 +153,130 @@ def process_inbound_message_task(self, inbound_message_id: str): logger.error(error_msg) inbound_message.error_message = error_msg inbound_message.save(update_fields=["error_message"]) - # Keep the message for retry return {"success": False, "error": error_msg} - # Get spam config from maildomain (includes global settings + domain-specific overrides) - spam_config = mailbox.domain.get_spam_config() - - rspamd_result: dict[str, Any] | None = None - if _is_selfcheck_message(parsed_email, recipient_email): + mailbox = inbound_message.mailbox + recipient_email = str(mailbox) + ctx = InboundContext( + mailbox=mailbox, + inbound_message=inbound_message, + recipient_email=recipient_email, + raw_data=raw_data_bytes, + parsed_email=parsed_email, + spam_config=mailbox.domain.get_spam_config(), + ) + if _is_selfcheck(parsed_email, recipient_email): + # System self-probe: short-circuit the spam check before + # the pipeline runs. The hardcoded-rules + rspamd steps + # both no-op when ctx.is_spam is already set. + ctx.is_spam = False logger.debug( - "Bypassing spam checks for selfcheck message %s", inbound_message_id + "Selfcheck probe — pre-setting is_spam=False for %s", + inbound_message_id, ) - is_spam = False - else: - # If we have hardcoded rules, check them sequentially - is_spam = _check_spam_with_hardcoded_rules(parsed_email, spam_config) - - # If no rules matched, check with rspamd - if is_spam is None: - is_spam, spam_check_error, rspamd_result = _check_spam_with_rspamd( - raw_data_bytes, spam_config - ) - if spam_check_error: - logger.warning( - "Spam check error for inbound message %s: %s (treating as not spam)", - inbound_message_id, - spam_check_error, - ) - - # Run inbound authentication checks (DKIM / DMARC). The verdict, if - # any, is stamped as X-StMsg-Sender-Auth so the frontend can render - # "unverified" (none) or "likely forged" (fail) warnings. - if rspamd_result is None and get_inbound_auth_mode(spam_config) == "rspamd": - _, _, rspamd_result = _check_spam_with_rspamd(raw_data_bytes, spam_config) - auth_verdict = check_inbound_authentication( - raw_data_bytes, parsed_email, spam_config, rspamd_result - ) - if auth_verdict: - prepended = ( - f"X-StMsg-Sender-Auth: {auth_verdict}\r\n".encode("ascii") - + raw_data_bytes + + decision, aborted_by = run_inbound_pipeline(build_inbound_pipeline(ctx), ctx) + + if decision == Decision.DROP: + logger.info( + "Inbound message %s dropped by step=%s", + inbound_message_id, + aborted_by, ) - reparsed = parse_email(prepended) - if reparsed is not None: - parsed_email = reparsed - raw_data_bytes = prepended - else: - # Keep raw_data_bytes / parsed_email in lockstep: if the - # re-parse breaks, store the original bytes so the blob stays - # parseable for display (subject/body/recipients). The - # sender-auth banner is sacrificed in this rare case. - logger.warning( - "Failed to re-parse email after prepending auth header, " - "dropping the prepend" - ) + inbound_message.delete() + return { + "success": True, + "inbound_message_id": str(inbound_message_id), + "dropped_by": aborted_by, + } + if decision == Decision.RETRY: + return _handle_retry(inbound_message, aborted_by) - # Create the message using the extracted function inbound_msg = _create_message_from_inbound( - recipient_email=recipient_email, - parsed_email=parsed_email, - raw_data=raw_data_bytes, + recipient_email=ctx.recipient_email, + parsed_email=ctx.parsed_email, + raw_data=ctx.raw_data, mailbox=mailbox, channel=inbound_message.channel, - is_spam=is_spam, + is_spam=bool(ctx.is_spam), + is_trashed=ctx.mark_trashed, + is_archived=ctx.mark_archived, ) if inbound_msg: - # Delete the message after successful processing inbound_message.delete() - # Send autoreply if appropriate (only for real Message objects) if isinstance(inbound_msg, models.Message): + # Each finalize step is isolated — a failure in one + # (DB hiccup, race with admin deletion) must not skip + # the others. The message has landed; best effort. + _safe_finalize( + "labels", + inbound_message_id, + ctx.labels, + lambda: apply_labels_to_thread( + inbound_msg.thread, mailbox, ctx.labels + ), + ) + _safe_finalize( + "assigns", + inbound_message_id, + ctx.pending_assigns, + lambda: apply_pending_assigns( + inbound_msg.thread, ctx.pending_assigns + ), + ) + _safe_finalize( + "events", + inbound_message_id, + ctx.pending_events, + lambda: apply_pending_events( + inbound_msg.thread, ctx.pending_events + ), + ) + _safe_finalize( + "drafts", + inbound_message_id, + ctx.pending_drafts, + lambda: apply_pending_drafts( + inbound_msg, mailbox, ctx.pending_drafts + ), + ) + _safe_finalize( + "flags", + inbound_message_id, + ctx.mark_starred or ctx.mark_read, + lambda: apply_thread_access_flags( + inbound_msg.thread, + mailbox, + mark_starred=ctx.mark_starred, + mark_read=ctx.mark_read, + ), + ) + + if isinstance(inbound_msg, models.Message) and not ctx.skip_autoreply: from core.mda.autoreply import ( # pylint: disable=import-outside-toplevel try_send_autoreply, ) - try_send_autoreply(mailbox, parsed_email, inbound_msg, is_spam=is_spam) + # ``try_send_autoreply`` already suppresses for spam. + # The ``skip_autoreply`` flag wraps the same gate from + # the outside so a non-spam message can also opt out + # (e.g. when the webhook itself replies). + try_send_autoreply( + mailbox, ctx.parsed_email, inbound_msg, is_spam=bool(ctx.is_spam) + ) logger.info( "Successfully processed inbound message %s (is_spam=%s)", inbound_message_id, - is_spam, + ctx.is_spam, ) return { "success": True, "inbound_message_id": str(inbound_message_id), - "is_spam": is_spam, + "is_spam": ctx.is_spam, } error_msg = "Failed to create message from inbound message" diff --git a/src/backend/core/models.py b/src/backend/core/models.py index 7ad6fe62d..bb281f63c 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -5,6 +5,7 @@ import base64 import hashlib +import hmac import json import re import secrets @@ -572,31 +573,94 @@ def __str__(self): # that Q() in Python and raises ValidationError before the row is sent # to the DB. No custom clean() override is needed. - # --- api_key helpers --- # + # --- secret rotation --- # + + # Secret storage shapes per channel type — deliberately divergent. + # ``api_key`` channels store only a SHA-256 hash so a DB read can't + # yield a working credential (server hashes incoming ``X-API-Key`` + # and compares). ``webhook`` channels store plaintext under the + # generic ``encrypted_settings["secret"]`` key because the + # dispatcher needs the raw bytes every fire to sign / send; + # future channel types that need a plaintext root reuse the same + # storage key. Forcing one storage shape would either break + # dispatch (hashing webhooks) or weaken api_key (plaintext at + # rest), so the divergence is load-bearing. + # + # ``settings.auth_method`` ("jwt" or "api_key") on webhook + # channels picks how the dispatcher presents that one stored + # secret on each POST. The root never travels on the wire — + # JWT mode keys an HMAC sig + HS256 JWT, API-key mode sends a + # *derived* value (see ``get_webhook_api_key``). + + # Context label for the webhook API-key derivation. Versioned so + # we can roll the KDF without changing the root secret. + WEBHOOK_API_KEY_KDF_LABEL = b"messages.webhook.api_key.v1" + + def rotate_secret(self, *, save: bool = True) -> str: + """Mint a fresh plaintext secret appropriate for this channel + type, persist it according to the type's storage shape, and + return the plaintext exactly once. + + Single-active rotation: any prior secret is invalidated + immediately. Dual-active "smooth" rotation is intentionally + not exposed here — callers that need it must mutate + ``encrypted_settings`` directly via the Django admin. + + Raises ``ValueError`` for channel types without a rotatable + secret (``widget``, ``mta``, ``caldav`` — these authenticate + differently). + + Set ``save=False`` for the DRF create path, where the row is + being built and ``super().create()`` will persist it shortly + after. + """ + from core.enums import ChannelTypes # pylint: disable=import-outside-toplevel + + if self.type == ChannelTypes.API_KEY: + plaintext = "msgk_" + secrets.token_urlsafe(32) + digest = hashlib.sha256(plaintext.encode("utf-8")).hexdigest() + self.encrypted_settings = { + **(self.encrypted_settings or {}), + "api_key_hashes": [digest], + } + elif self.type == ChannelTypes.WEBHOOK: + plaintext = "whsec_" + secrets.token_urlsafe(32) + self.encrypted_settings = { + **(self.encrypted_settings or {}), + "secret": plaintext, + } + else: + raise ValueError( + f"Channel type {self.type!r} has no rotatable secret" + ) - API_KEY_PREFIX = "msgk_" + if save: + self.save(update_fields=["encrypted_settings", "updated_at"]) + return plaintext - def rotate_api_key(self, *, save: bool = True) -> str: - """Mint a fresh api_key plaintext, replace ``api_key_hashes`` with - the new SHA-256 digest, and return the plaintext exactly once. + def get_webhook_api_key(self) -> Optional[str]: + """Derive the API-key presentation of the root secret. - Single-active rotation: any prior secret is invalidated immediately. - Dual-active "smooth" rotation (appending without removing) is not - exposed here — callers that need it must mutate ``encrypted_settings`` - directly via the Django admin. + ``api_key = "whk_" + HMAC-SHA256(root_secret, label).hex()``. The + HMAC step is one-way: a receiver-side leak of the API key + reveals nothing about the root secret, so JWT/HMAC verification + remains unforgeable. Deterministic — switching ``auth_method`` + between ``jwt`` and ``api_key`` on the same channel doesn't + require rotating, because both presentations are tied to the + same root. - Set ``save=False`` for the DRF create path, where the row is being - built and ``super().create()`` will persist it shortly after. + Returns ``None`` if the channel has no root secret yet (a + misconfigured row that the dispatcher will fail closed on). """ - plaintext = self.API_KEY_PREFIX + secrets.token_urlsafe(32) - digest = hashlib.sha256(plaintext.encode("utf-8")).hexdigest() - self.encrypted_settings = { - **(self.encrypted_settings or {}), - "api_key_hashes": [digest], - } - if save: - self.save(update_fields=["encrypted_settings", "updated_at"]) - return plaintext + root = (self.encrypted_settings or {}).get("secret") + if not root: + return None + derived = hmac.new( + root.encode("utf-8"), + self.WEBHOOK_API_KEY_KDF_LABEL, + hashlib.sha256, + ).hexdigest() + return "whk_" + derived def api_key_covers( self, *, mailbox=None, maildomain=None, mailbox_roles=None diff --git a/src/backend/core/services/ssrf.py b/src/backend/core/services/ssrf.py index e58921d08..25fa9f74d 100644 --- a/src/backend/core/services/ssrf.py +++ b/src/backend/core/services/ssrf.py @@ -223,6 +223,20 @@ def _validate_and_unpack(self, url: str) -> tuple[str, str, str, int]: return valid_ips[0], parsed.hostname, parsed.scheme, port + def _pinned_session(self, url: str) -> tuple[requests.Session, str]: + """Return an SSRF-pinned Session bound to ``url``'s validated IP.""" + validated_ip, hostname, scheme, port = self._validate_and_unpack(url) + session = requests.Session() + adapter = SSRFProtectedAdapter( + dest_ip=validated_ip, + dest_port=port, + original_hostname=hostname, + original_scheme=scheme, + ) + session.mount("http://", adapter) + session.mount("https://", adapter) + return session, hostname + def get(self, url: str, timeout: int, **kwargs) -> requests.Response: """Perform a safe HTTP GET with per-hop SSRF validation on redirects. @@ -237,19 +251,7 @@ def get(self, url: str, timeout: int, **kwargs) -> requests.Response: current_url = url for _ in range(MAX_REDIRECTS + 1): - validated_ip, hostname, scheme, port = self._validate_and_unpack( - current_url - ) - - session = requests.Session() - adapter = SSRFProtectedAdapter( - dest_ip=validated_ip, - dest_port=port, - original_hostname=hostname, - original_scheme=scheme, - ) - session.mount("http://", adapter) - session.mount("https://", adapter) + session, _ = self._pinned_session(current_url) response = session.get( current_url, timeout=timeout, allow_redirects=False, **kwargs @@ -268,3 +270,15 @@ def get(self, url: str, timeout: int, **kwargs) -> requests.Response: current_url = next_url raise SSRFValidationError(f"Too many redirects (max {MAX_REDIRECTS})") + + def post(self, url: str, timeout: int, **kwargs) -> requests.Response: + """Perform a safe HTTP POST. + + Redirects are NOT followed: webhook providers conventionally don't + chase 3xx on POST, and silently turning a 30x into a follow-up GET + (which is what most clients do for 301/302/303) would surprise the + caller. A 3xx is returned to the caller as-is. + """ + kwargs.pop("allow_redirects", None) + session, _ = self._pinned_session(url) + return session.post(url, timeout=timeout, allow_redirects=False, **kwargs) diff --git a/src/backend/core/services/thread_events.py b/src/backend/core/services/thread_events.py index 92f8d4b75..542613951 100644 --- a/src/backend/core/services/thread_events.py +++ b/src/backend/core/services/thread_events.py @@ -314,7 +314,7 @@ def _absorb_unassign_in_undo_window(*, thread, author, assignee_ids, assignees_d @transaction.atomic -def assign_users(*, thread, author, assignees_data): +def assign_users(*, thread, author, assignees_data, channel=None): """Assign users to ``thread`` by creating ASSIGN events. - Idempotent: users already holding a ``UserEvent ASSIGN`` on the @@ -326,6 +326,9 @@ def assign_users(*, thread, author, assignees_data): - Persists a single ThreadEvent ASSIGN containing the new assignees and creates the matching ``UserEvent`` rows in the same atomic transaction. + - ``channel`` is attached to the resulting ``ThreadEvent`` for + audit attribution. The user-driven paths leave it ``None``; + webhook-driven assigns pass the firing webhook ``Channel``. Returns the persisted ThreadEvent, or ``None`` when nothing was new. """ @@ -358,6 +361,7 @@ def assign_users(*, thread, author, assignees_data): thread_event = models.ThreadEvent.objects.create( thread=thread, author=author, + channel=channel, type=enums.ThreadEventTypeChoices.ASSIGN, data={"assignees": new_assignees}, ) diff --git a/src/backend/core/templates/admin/core/channel/regenerated_api_key.html b/src/backend/core/templates/admin/core/channel/regenerated_api_key.html index 89a2c5d13..6cb7b8e30 100644 --- a/src/backend/core/templates/admin/core/channel/regenerated_api_key.html +++ b/src/backend/core/templates/admin/core/channel/regenerated_api_key.html @@ -7,18 +7,18 @@ › {{ opts.app_config.verbose_name }} › {{ opts.verbose_name_plural|capfirst }} › {{ original }} - › {% translate "Regenerated api_key" %} + › {% translate "Regenerated secret" %} {% endblock %} {% block content %} -

{% translate "New api_key generated" %}

+

{% translate "New secret generated" %}

{% translate "Copy this value now. It will not be shown again." %}

-

{% blocktranslate %}The previous secret has been invalidated immediately. Any client still using the old api_key will start failing on its next call.{% endblocktranslate %}

+

{% blocktranslate %}The previous secret has been invalidated immediately. Any client still using the old secret will start failing on its next call.{% endblocktranslate %}

-
{{ api_key }}
+
{{ secret }}

diff --git a/src/backend/core/tests/api/test_channel_scope_level.py b/src/backend/core/tests/api/test_channel_scope_level.py index e8e49fc57..a5f4b0b5d 100644 --- a/src/backend/core/tests/api/test_channel_scope_level.py +++ b/src/backend/core/tests/api/test_channel_scope_level.py @@ -1025,7 +1025,7 @@ def test_personal_webhook_channel(self, api_client): "type": "webhook", "settings": { "url": "https://hook.example.com/me", - "events": ["message.received"], + "events": ["message.received"], "auth_method": "jwt", }, }, format="json", @@ -1081,7 +1081,7 @@ def test_user_delete_with_no_personal_channels_succeeds(self): @pytest.mark.django_db class TestRegenerateApiKey: - """The regenerate-api-key action: single-active replace, never append. + """The regenerate-secret action: single-active replace, never append. DRF's only rotation flow. Smooth (dual-active) rotation would happen in the Django admin or a future CLI command. @@ -1099,7 +1099,7 @@ def _hash(plaintext): def _mailbox_url(self, mailbox, channel): return reverse( - "mailbox-channels-regenerate-api-key", + "mailbox-channels-regenerate-secret", kwargs={"mailbox_id": mailbox.id, "pk": channel.id}, ) @@ -1263,7 +1263,7 @@ def test_regenerate_other_mailbox_admin_returns_403(self, api_client): # ---- /users/me/channels/ -------------------------------------------- # def _user_url(self, channel): - return reverse("user-channels-regenerate-api-key", kwargs={"pk": channel.id}) + return reverse("user-channels-regenerate-secret", kwargs={"pk": channel.id}) def test_regenerate_personal_api_key(self, api_client): user = UserFactory() diff --git a/src/backend/core/tests/api/test_channels.py b/src/backend/core/tests/api/test_channels.py index ee7a21c9b..a54e06f63 100644 --- a/src/backend/core/tests/api/test_channels.py +++ b/src/backend/core/tests/api/test_channels.py @@ -566,3 +566,135 @@ def test_unrelated_settings_keys_pass_through(self, api_client, mailbox): # we still assert the encrypted_settings shape). assert "api_key_hashes" in channel.encrypted_settings assert "api_key_hashes" not in channel.settings + + +@pytest.mark.django_db +class TestWebhookChannelSettings: + """Validation of the outbound-webhook-specific settings fields.""" + + URL_KEY = "url" + + def _post(self, api_client, mailbox, settings): + url = reverse("mailbox-channels-list", kwargs={"mailbox_id": mailbox.id}) + return api_client.post( + url, + data={ + "name": "wh", + "type": "webhook", + "settings": settings, + }, + format="json", + ) + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + def test_create_minimal_webhook(self, api_client, mailbox): + response = self._post( + api_client, + mailbox, + { + "url": "https://hook.example.com/in", + "events": ["message.received"], "auth_method": "jwt", + }, + ) + assert response.status_code == status.HTTP_201_CREATED, response.content + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + def test_create_with_all_dispatcher_options(self, api_client, mailbox): + response = self._post( + api_client, + mailbox, + { + "url": "https://hook.example.com/in", + "events": ["message.received"], "auth_method": "jwt", + "phase": "before_spam", + "blocking": True, + "format": "jmap", + }, + ) + assert response.status_code == status.HTTP_201_CREATED, response.content + channel = models.Channel.objects.get(id=response.data["id"]) + assert channel.settings["phase"] == "before_spam" + assert channel.settings["blocking"] is True + assert channel.settings["format"] == "jmap" + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + def test_rejects_invalid_format(self, api_client, mailbox): + response = self._post( + api_client, + mailbox, + { + "url": "https://hook.example.com/in", + "events": ["message.received"], "auth_method": "jwt", + "format": "yaml", + }, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + def test_accepts_jmap_without_body_format(self, api_client, mailbox): + response = self._post( + api_client, + mailbox, + { + "url": "https://hook.example.com/in", + "events": ["message.received"], "auth_method": "jwt", + "format": "jmap_without_body", + }, + ) + assert response.status_code == status.HTTP_201_CREATED, response.content + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + def test_rejects_invalid_phase(self, api_client, mailbox): + response = self._post( + api_client, + mailbox, + { + "url": "https://hook.example.com/in", + "events": ["message.received"], "auth_method": "jwt", + "phase": "whenever", + }, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + def test_rejects_non_bool_blocking(self, api_client, mailbox): + response = self._post( + api_client, + mailbox, + { + "url": "https://hook.example.com/in", + "events": ["message.received"], "auth_method": "jwt", + "blocking": "yes", + }, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + def test_rejects_patch_with_invalid_phase(self, api_client, mailbox): + """A PATCH that touches only ``settings`` must still re-run the + webhook validator (same airtight rule as api_key scopes).""" + create = self._post( + api_client, + mailbox, + { + "url": "https://hook.example.com/in", + "events": ["message.received"], "auth_method": "jwt", + }, + ) + assert create.status_code == status.HTTP_201_CREATED, create.content + url = reverse( + "mailbox-channels-detail", + kwargs={"mailbox_id": mailbox.id, "pk": create.data["id"]}, + ) + response = api_client.patch( + url, + data={ + "settings": { + "url": "https://hook.example.com/in", + "events": ["message.received"], "auth_method": "jwt", + "phase": "bogus", + } + }, + format="json", + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST diff --git a/src/backend/core/tests/mda/test_dispatch_webhooks.py b/src/backend/core/tests/mda/test_dispatch_webhooks.py new file mode 100644 index 000000000..6394333b6 --- /dev/null +++ b/src/backend/core/tests/mda/test_dispatch_webhooks.py @@ -0,0 +1,2525 @@ +"""Tests for the user-webhook step and the inbound pipeline integration.""" + +# pylint: disable=protected-access + +import hashlib +import hmac +import json +import uuid +from dataclasses import dataclass, field +from typing import Optional, Set +from unittest.mock import Mock, patch + +from django.utils import timezone as dj_timezone + +import pytest +import requests as requests_lib + +from core import enums, factories, models +from core.mda.dispatch_webhooks import ( + DEFAULT_FORMAT, + FORMAT_EML, + FORMAT_JMAP, + PHASE_AFTER_SPAM, + PHASE_BEFORE_SPAM, + _classify_response_body, + build_jmap_email, + find_webhook_channels_for_mailbox, + webhook_steps_for_mailbox, +) +from core.mda.inbound_pipeline import ( + RETRY_MAX_AGE, + Decision, + InboundContext, +) +from core.mda.inbound_tasks import process_inbound_message_task +from core.services.ssrf import SSRFValidationError + + +@dataclass +class _PhaseResult: + """Aggregated result of running every webhook step for a phase + against a fresh ``InboundContext``. + + ``decision`` is the most-severe step decision; ``is_spam_override`` + is the final ``ctx.is_spam`` when a step changed it from the initial + value (``None`` = no step had an opinion); ``labels`` is the set the + context accumulated. + """ + + decision: Decision = Decision.CONTINUE + is_spam_override: Optional[bool] = None + labels: Set[str] = field(default_factory=set) + + +def dispatch_webhooks( + *, + phase, + mailbox, + recipient_email, + parsed_email, + raw_data, + is_spam=None, +): + """Test helper: run every webhook step matching ``phase`` against a + minimal ``InboundContext`` and return a phase-level aggregate.""" + ctx = InboundContext( + mailbox=mailbox, + inbound_message=Mock(id="test-inbound", created_at=dj_timezone.now()), + recipient_email=recipient_email, + raw_data=raw_data, + parsed_email=parsed_email, + spam_config={}, + is_spam=is_spam, + ) + initial_is_spam = is_spam + result = _PhaseResult() + for step in webhook_steps_for_mailbox(mailbox, phase=phase): + d = step(ctx) + if d != Decision.CONTINUE: + result.decision = d + break + if ctx.is_spam != initial_is_spam: + result.is_spam_override = ctx.is_spam + result.labels = ctx.labels + return result + + +# --- shared fixtures --- # + + +@pytest.fixture(name="mailbox") +def fixture_mailbox(): + return factories.MailboxFactory() + + +@pytest.fixture(name="parsed_email") +def fixture_parsed_email(mailbox): + """A strict-JMAP Email object as ``jmap_email.parse_email`` emits it.""" + return { + "subject": "Hello", + "from": [{"email": "sender@example.com", "name": "Sender"}], + "to": [{"email": str(mailbox), "name": ""}], + "cc": [], + "bcc": [], + "sentAt": "2026-01-01T12:00:00Z", + "messageId": ["mid@example.com"], + "inReplyTo": ["parent@example.com"], + "references": ["a@example.com", "b@example.com"], + "textBody": [{"partId": "1", "type": "text/plain"}], + "htmlBody": [{"partId": "2", "type": "text/html"}], + "attachments": [], + "hasAttachment": False, + "bodyValues": { + "1": { + "value": "hi there", + "isEncodingProblem": False, + "isTruncated": False, + }, + "2": { + "value": "

hi

", + "isEncodingProblem": False, + "isTruncated": False, + }, + }, + "headers": [ + {"name": "From", "value": "Sender "}, + {"name": "To", "value": str(mailbox)}, + {"name": "Subject", "value": "Hello"}, + ], + } + + +def _make_response(status_code: int, body: bytes = b"") -> Mock: + response = Mock() + response.status_code = status_code + response.content = body + # The dispatcher now reads the body via iter_content (stream=True) + # with a size cap. The mock yields the whole body in one chunk — + # tests that want to exercise the cap can pass a longer ``body``. + response.iter_content = Mock(return_value=iter([body] if body else [])) + response.close = Mock() + return response + + +# ChannelFactory auto-mints this for type=webhook so test channels are +# never silently skipped by the dispatcher's fail-closed signing path. +FACTORY_WEBHOOK_SECRET = "whsec_factory_test" + + +# --- find_webhook_channels_for_mailbox --- # + + +@pytest.mark.django_db +class TestFindWebhookChannels: + def test_finds_mailbox_scoped(self, mailbox): + ch = factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com/a", + "events": ["message.received"], + }, + ) + assert list(find_webhook_channels_for_mailbox(mailbox)) == [ch] + + def test_finds_maildomain_scoped(self, mailbox): + ch = factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=None, + maildomain=mailbox.domain, + settings={ + "url": "https://hook.example.com/d", + "events": ["message.received"], + }, + ) + result = list(find_webhook_channels_for_mailbox(mailbox)) + assert result == [ch] + + def test_finds_global_scoped(self, mailbox): + """Global (instance-wide) webhooks must fire for every mailbox.""" + ch = models.Channel.objects.create( + name="global-wh", + type=enums.ChannelTypes.WEBHOOK, + scope_level=enums.ChannelScopeLevel.GLOBAL, + settings={ + "url": "https://hook.example.com/g", + "events": ["message.received"], + }, + ) + result = list(find_webhook_channels_for_mailbox(mailbox)) + assert result == [ch] + + def test_global_fires_for_other_mailbox_too(self): + """A global webhook must match an unrelated mailbox.""" + mb_a = factories.MailboxFactory() + mb_b = factories.MailboxFactory() + ch = models.Channel.objects.create( + name="global-wh", + type=enums.ChannelTypes.WEBHOOK, + scope_level=enums.ChannelScopeLevel.GLOBAL, + settings={ + "url": "https://hook.example.com/g", + "events": ["message.received"], + }, + ) + assert ch in find_webhook_channels_for_mailbox(mb_a) + assert ch in find_webhook_channels_for_mailbox(mb_b) + + def test_excludes_other_mailbox(self, mailbox): + other = factories.MailboxFactory() + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=other, + settings={ + "url": "https://hook.example.com/x", + "events": ["message.received"], + }, + ) + assert list(find_webhook_channels_for_mailbox(mailbox)) == [] + + def test_excludes_other_types(self, mailbox): + factories.ChannelFactory( + type="widget", mailbox=mailbox, settings={"config": {"enabled": True}} + ) + assert list(find_webhook_channels_for_mailbox(mailbox)) == [] + + +# --- JMAP body builder --- # + + +class TestBuildJmapEmail: + """``parse_email`` already emits a strict JMAP Email object, so + ``build_jmap_email`` is mostly a pass-through: stamp ``receivedAt``, + strip the parser's project extensions (``_ext`` and per-part + ``content`` / ``sha256``).""" + + def test_minimal_email_shape(self): + parsed = { + "subject": "Hi", + "from": [{"email": "alice@example.org", "name": "Alice"}], + "to": [{"email": "bob@example.org", "name": "Bob"}], + "cc": [], + "bcc": [], + "sentAt": "2026-01-01T00:00:00Z", + "messageId": ["abc@example.org"], + "inReplyTo": [], + "references": [], + "textBody": [{"partId": "1", "type": "text/plain"}], + "htmlBody": [], + "attachments": [], + "hasAttachment": False, + "bodyValues": { + "1": { + "value": "hello", + "isEncodingProblem": False, + "isTruncated": False, + }, + }, + "headers": [{"name": "From", "value": "Alice "}], + } + email = build_jmap_email(parsed) + # Strict-JMAP fields pass through unchanged. + assert email["messageId"] == ["abc@example.org"] + assert email["inReplyTo"] == [] + assert email["references"] == [] + assert email["from"] == [{"email": "alice@example.org", "name": "Alice"}] + assert email["sentAt"] == "2026-01-01T00:00:00Z" + # ``receivedAt`` is stamped at webhook-fire time. + assert email["receivedAt"].endswith("Z") + assert email["headers"] == [ + {"name": "From", "value": "Alice "}, + ] + # bodyValues passes through unchanged. + assert email["bodyValues"]["1"] == { + "value": "hello", + "isEncodingProblem": False, + "isTruncated": False, + } + assert email["textBody"][0]["partId"] == "1" + assert email["textBody"][0]["type"] == "text/plain" + assert email["hasAttachment"] is False + # Storage-time JMAP fields are absent at webhook-fire time. + for absent in ("id", "blobId", "threadId", "mailboxIds", "keywords"): + assert absent not in email + + def test_msgid_lists_pass_through(self): + """``parse_email`` already returns ``Id[]`` lists with the angle + brackets stripped — the builder passes them straight through.""" + parsed = { + "subject": "x", + "from": [{"email": "a@x"}], + "to": [], + "cc": [], + "bcc": [], + "sentAt": None, + "messageId": ["m1"], + "inReplyTo": ["parent@example.org"], + "references": ["r1@x", "r2@x"], + "textBody": [], + "htmlBody": [], + "attachments": [], + "hasAttachment": False, + "bodyValues": {}, + "headers": [], + } + email = build_jmap_email(parsed) + assert email["inReplyTo"] == ["parent@example.org"] + assert email["references"] == ["r1@x", "r2@x"] + + def test_attachments_stripped_of_parser_extensions(self): + """Attachment parts keep their JMAP metadata but drop the + parser's ``content`` bytes and ``sha256`` extension — neither is + strict JMAP, and raw bytes aren't JSON-serialisable.""" + parsed = { + "subject": "x", + "from": [{"email": "a@x"}], + "to": [], + "cc": [], + "bcc": [], + "sentAt": None, + "messageId": ["m1"], + "textBody": [], + "htmlBody": [], + "hasAttachment": True, + "bodyValues": {}, + "headers": [], + "attachments": [ + { + "partId": "att-0", + "blobId": None, + "type": "image/png", + "name": "p.png", + "size": 42, + "disposition": "attachment", + "cid": "img1", + "content": b"\x89PNG\r\n", + "sha256": "deadbeef", + }, + ], + } + email = build_jmap_email(parsed) + assert email["hasAttachment"] is True + assert email["attachments"][0]["type"] == "image/png" + assert email["attachments"][0]["name"] == "p.png" + assert email["attachments"][0]["size"] == 42 + assert email["attachments"][0]["cid"] == "img1" + # Project extensions are stripped — bytes never travel in the body. + assert "content" not in email["attachments"][0] + assert "sha256" not in email["attachments"][0] + assert email["attachments"][0]["blobId"] is None + + +# --- dispatch_webhooks --- # + + +@pytest.mark.django_db +class TestDispatchInboundWebhooks: + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_skips_when_no_channels(self, mock_session, mailbox, parsed_email): + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"", + ) + assert outcome.decision == Decision.CONTINUE + mock_session.assert_not_called() + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_skips_channel_with_wrong_phase(self, mock_session, mailbox, parsed_email): + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "before_spam", + }, + ) + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"", + ) + assert outcome.decision == Decision.CONTINUE + mock_session.assert_not_called() + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_skips_channel_without_matching_event( + self, mock_session, mailbox, parsed_email + ): + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.sent"], + }, + ) + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"", + ) + assert outcome.decision == Decision.CONTINUE + mock_session.assert_not_called() + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_non_blocking_continues_on_5xx(self, mock_session, mailbox, parsed_email): + """Non-blocking webhooks never influence delivery, even on 5xx.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "blocking": False, + }, + ) + mock_session.return_value.post.return_value = _make_response(500) + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + assert outcome.decision == Decision.CONTINUE + mock_session.return_value.post.assert_called_once() + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_blocking_retries_on_5xx(self, mock_session, mailbox, parsed_email): + """5xx is transient: caller should hold the InboundMessage and retry.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "blocking": True, + }, + ) + mock_session.return_value.post.return_value = _make_response(503) + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + assert outcome.decision == Decision.RETRY + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_blocking_drops_on_4xx(self, mock_session, mailbox, parsed_email): + """4xx is a definitive receiver rejection — drop the message.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "blocking": True, + }, + ) + mock_session.return_value.post.return_value = _make_response(403) + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + assert outcome.decision == Decision.DROP + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_blocking_retries_on_408(self, mock_session, mailbox, parsed_email): + """408 Request Timeout is conventionally retriable.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "blocking": True, + }, + ) + mock_session.return_value.post.return_value = _make_response(408) + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + assert outcome.decision == Decision.RETRY + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_blocking_retries_on_429(self, mock_session, mailbox, parsed_email): + """429 Too Many Requests is rate-limit: back off and retry.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "blocking": True, + }, + ) + mock_session.return_value.post.return_value = _make_response(429) + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + assert outcome.decision == Decision.RETRY + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_blocking_drops_on_ssrf_rejection( + self, mock_session, mailbox, parsed_email + ): + """SSRF rejection is a config error on our side — retrying won't help.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://internal.example.com", + "events": ["message.received"], + "blocking": True, + }, + ) + mock_session.return_value.post.side_effect = SSRFValidationError("blocked") + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + assert outcome.decision == Decision.DROP + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_non_blocking_continues_on_ssrf_rejection( + self, mock_session, mailbox, parsed_email + ): + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://internal.example.com", + "events": ["message.received"], + "blocking": False, + }, + ) + mock_session.return_value.post.side_effect = SSRFValidationError("blocked") + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + assert outcome.decision == Decision.CONTINUE + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_blocking_retries_on_timeout(self, mock_session, mailbox, parsed_email): + """A connection timeout is transient: retry rather than lose the message.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "blocking": True, + }, + ) + mock_session.return_value.post.side_effect = requests_lib.Timeout("timed out") + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + assert outcome.decision == Decision.RETRY + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_blocking_retries_on_connection_error( + self, mock_session, mailbox, parsed_email + ): + """Connection refused / DNS failures are transient — retry.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "blocking": True, + }, + ) + mock_session.return_value.post.side_effect = requests_lib.ConnectionError( + "refused" + ) + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + assert outcome.decision == Decision.RETRY + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_blocking_retries_on_unknown_exception( + self, mock_session, mailbox, parsed_email + ): + """Unknown transport-level errors land as RETRY — the 7-day cap + bounds how long we'll keep trying a busted receiver.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "blocking": True, + }, + ) + mock_session.return_value.post.side_effect = RuntimeError("boom") + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + assert outcome.decision == Decision.RETRY + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_phase_filtering_dispatches_only_matching( + self, mock_session, mailbox, parsed_email + ): + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com/before", + "events": ["message.received"], + "phase": "before_spam", + }, + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com/after", + "events": ["message.received"], + "phase": "after_spam", + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + dispatch_webhooks( + phase=PHASE_BEFORE_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"", + ) + called_url = mock_session.return_value.post.call_args[0][0] + assert called_url == "https://hook.example.com/before" + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_eml_format_sends_raw_body(self, mock_session, mailbox, parsed_email): + """Default format=eml posts message/rfc822 raw bytes.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw rfc822 bytes", + is_spam=False, + ) + kwargs = mock_session.return_value.post.call_args.kwargs + assert kwargs["data"] == b"raw rfc822 bytes" + assert "json" not in kwargs + assert kwargs["headers"]["Content-Type"] == "message/rfc822" + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_jmap_format_sends_jmap_email_json( + self, mock_session, mailbox, parsed_email + ): + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "format": "jmap", + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw rfc822 bytes", + is_spam=False, + ) + kwargs = mock_session.return_value.post.call_args.kwargs + # We pre-serialise JSON to bytes so signing covers the exact wire + # bytes — so the body lands in ``data``, not ``json``. + assert "json" not in kwargs + body = json.loads(kwargs["data"].decode("utf-8")) + # Body IS the JMAP Email object — no wrapping envelope. + assert body["messageId"] == ["mid@example.com"] + assert body["from"] == [{"email": "sender@example.com", "name": "Sender"}] + assert "X-StMsg-Event" not in body + assert kwargs["headers"]["Content-Type"] == "application/json" + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_jmap_without_body_skips_body_parts( + self, mock_session, mailbox, parsed_email + ): + """Notification variant: no textBody/htmlBody/bodyValues/attachments.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "format": "jmap_without_body", + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + body = json.loads( + mock_session.return_value.post.call_args.kwargs["data"].decode("utf-8") + ) + # Envelope addresses + headers ARE present. + assert body["subject"] == "Hello" + assert body["from"] == [{"email": "sender@example.com", "name": "Sender"}] + assert body["messageId"] == ["mid@example.com"] + assert "headers" in body + # Body content and attachments are NOT shipped. + for absent in ("textBody", "htmlBody", "bodyValues", "attachments"): + assert absent not in body + # hasAttachment is preserved as a single bool that the receiver + # may want for filtering. + assert body["hasAttachment"] is False + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_envelope_headers_set_for_both_formats( + self, mock_session, mailbox, parsed_email + ): + for fmt in (FORMAT_EML, FORMAT_JMAP): + mock_session.reset_mock() + models.Channel.objects.filter( + type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox + ).delete() + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "format": fmt, + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=True, + ) + headers = mock_session.return_value.post.call_args.kwargs["headers"] + assert headers["X-StMsg-Event"] == "message.received" + assert headers["X-StMsg-Phase"] == "after_spam" + assert headers["X-StMsg-Mailbox"] == str(mailbox) + assert headers["X-StMsg-Recipient"] == str(mailbox) + assert headers["X-StMsg-Is-Spam"] == "true" + assert headers["X-StMsg-Message-Id"] == "" + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_is_spam_header_unknown_when_none( + self, mock_session, mailbox, parsed_email + ): + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "before_spam", + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + dispatch_webhooks( + phase=PHASE_BEFORE_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=None, + ) + headers = mock_session.return_value.post.call_args.kwargs["headers"] + assert headers["X-StMsg-Is-Spam"] == "unknown" + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_invalid_format_skips_dispatch(self, mock_session, mailbox, parsed_email): + """A row that somehow has settings.format = junk must not silently + POST in the wrong shape — skip it instead.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "format": "yaml", + }, + ) + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + mock_session.return_value.post.assert_not_called() + + def test_invalid_phase_raises(self, mailbox, parsed_email): + with pytest.raises(ValueError): + dispatch_webhooks( + phase="never", + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"", + ) + + def test_constants_default(self): + assert DEFAULT_FORMAT == FORMAT_EML + + +@pytest.mark.django_db +class TestWebhookSigning: + """Every outgoing webhook is HMAC-signed; receivers verify by + recomputing HMAC-SHA256 over ``f"{ts}.{body}"`` with the channel + secret and constant-time comparing against + ``X-StMsg-Webhook-Signature``.""" + + SECRET = FACTORY_WEBHOOK_SECRET + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_eml_signature_covers_raw_body(self, mock_session, mailbox, parsed_email): + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "format": "eml", + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + raw = b"From: a\r\n\r\nbody" + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=raw, + is_spam=False, + ) + headers = mock_session.return_value.post.call_args.kwargs["headers"] + ts = headers["X-StMsg-Webhook-Timestamp"] + sig_header = headers["X-StMsg-Webhook-Signature"] + scheme, _, sig = sig_header.partition("=") + assert scheme == "v1" + + expected = hmac.new( + self.SECRET.encode("utf-8"), + ts.encode("ascii") + b"." + raw, + hashlib.sha256, + ).hexdigest() + assert hmac.compare_digest(sig, expected) + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_jmap_signature_covers_exact_serialised_bytes( + self, mock_session, mailbox, parsed_email + ): + """The body we sign MUST equal the body we POST byte-for-byte — + otherwise ``requests`` could re-serialise JSON with different + separators/key order and break the signature.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "format": "jmap", + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + kwargs = mock_session.return_value.post.call_args.kwargs + body_bytes = kwargs["data"] + assert isinstance(body_bytes, bytes) + ts = kwargs["headers"]["X-StMsg-Webhook-Timestamp"] + sig = kwargs["headers"]["X-StMsg-Webhook-Signature"].split("=", 1)[1] + expected = hmac.new( + self.SECRET.encode("utf-8"), + ts.encode("ascii") + b"." + body_bytes, + hashlib.sha256, + ).hexdigest() + assert hmac.compare_digest(sig, expected) + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_api_key_mode_sends_only_api_key_header( + self, mock_session, mailbox, parsed_email + ): + """auth_method=api_key: send X-StMsg-Api-Key, omit HMAC sig / JWT. + + Sending only the credential the receiver verifies keeps the + unused presentation off the wire (and out of any receiver-side + proxy/log).""" + channel = factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "auth_method": "api_key", + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + headers = mock_session.return_value.post.call_args.kwargs["headers"] + # X-StMsg-Api-Key carries the HMAC-DERIVED value, NOT the root + # secret — the root never travels on the wire. + assert headers["X-StMsg-Api-Key"] == channel.get_webhook_api_key() + assert headers["X-StMsg-Api-Key"] != self.SECRET + # The HMAC + JWT presentation is NOT sent — that's the whole + # point of the per-channel auth_method. + assert "X-StMsg-Webhook-Signature" not in headers + assert "X-StMsg-Webhook-Timestamp" not in headers + assert "Authorization" not in headers + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_jwt_mode_sends_only_hmac_and_jwt_headers( + self, mock_session, mailbox, parsed_email + ): + """auth_method=jwt (default): HMAC sig + Authorization Bearer, + but never the raw secret as an API key header.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "auth_method": "jwt", + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + headers = mock_session.return_value.post.call_args.kwargs["headers"] + assert "X-StMsg-Webhook-Signature" in headers + assert headers["Authorization"].startswith("Bearer ") + # Receivers that verify HMAC never need the raw secret — keep it + # off the wire so it can't leak through receiver-side logs. + assert "X-StMsg-Api-Key" not in headers + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_missing_auth_method_fails_closed( + self, mock_session, mailbox, parsed_email + ): + """A row with auth_method missing/unknown is misconfigured — + the dispatcher fails closed rather than POST with no auth.""" + # Bypass the factory's auto-fill so settings has no auth_method. + models.Channel.objects.create( + name="no-auth-method", + type=enums.ChannelTypes.WEBHOOK, + scope_level=enums.ChannelScopeLevel.MAILBOX, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + }, + encrypted_settings={"secret": "whsec_test"}, + ) + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + mock_session.return_value.post.assert_not_called() + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_api_key_value_is_derived_not_raw_secret( + self, mock_session, mailbox, parsed_email + ): + """The X-StMsg-Api-Key value MUST NOT be the raw root secret — + a receiver-side log leak of the API key would otherwise + compromise HMAC/JWT verification.""" + channel = factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "auth_method": "api_key", + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + sent = mock_session.return_value.post.call_args.kwargs["headers"][ + "X-StMsg-Api-Key" + ] + root = channel.encrypted_settings["secret"] + assert sent != root, "raw root secret must never travel as the API key" + assert sent.startswith("whk_"), ( + "API key should use the dedicated prefix so receivers can " + "distinguish it from the root secret" + ) + # And the derivation is the stable one exposed by the model. + assert sent == channel.get_webhook_api_key() + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_missing_secret_fails_closed(self, mock_session, mailbox, parsed_email): + """A webhook channel with no secret is misconfigured — the + dispatcher must skip it rather than POST an unsigned request.""" + # Build a channel directly so we can leave encrypted_settings + # empty (factory would otherwise auto-fill the secret). + models.Channel.objects.create( + name="no-secret", + type=enums.ChannelTypes.WEBHOOK, + scope_level=enums.ChannelScopeLevel.MAILBOX, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + }, + encrypted_settings={}, + ) + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + mock_session.return_value.post.assert_not_called() + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_missing_secret_blocks_when_blocking( + self, mock_session, mailbox, parsed_email + ): + """If the misconfigured channel is also ``blocking``, the + dispatcher MUST drop the message — better than POSTing an + unsigned request that any verifying receiver will reject.""" + models.Channel.objects.create( + name="no-secret-blocking", + type=enums.ChannelTypes.WEBHOOK, + scope_level=enums.ChannelScopeLevel.MAILBOX, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "blocking": True, + }, + encrypted_settings={}, + ) + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + assert outcome.decision == Decision.DROP + mock_session.return_value.post.assert_not_called() + + +# --- integration with process_inbound_message_task --- # + + +@pytest.mark.django_db +class TestPipelineIntegration: + @patch("core.mda.inbound_tasks._create_message_from_inbound") + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_before_spam_blocking_drops_message( + self, mock_session, mock_check_spam, mock_create_message + ): + mailbox = factories.MailboxFactory() + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: test\r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "before_spam", + "blocking": True, + }, + ) + # 4xx → receiver definitively rejects this message → DROP. + mock_session.return_value.post.return_value = _make_response(403) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + result = process_inbound_message_task.run(str(inbound_message.id)) + + assert result["dropped_by"].endswith(":before_spam") + mock_check_spam.assert_not_called() + mock_create_message.assert_not_called() + assert not models.InboundMessage.objects.filter(id=inbound_message.id).exists() + + @patch("core.mda.inbound_tasks._create_message_from_inbound") + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_after_spam_blocking_drops_message( + self, mock_session, mock_check_spam, mock_create_message + ): + mailbox = factories.MailboxFactory() + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: test\r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "after_spam", + "blocking": True, + }, + ) + mock_check_spam.return_value = (False, None, None) + mock_session.return_value.post.return_value = _make_response(403) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + result = process_inbound_message_task.run(str(inbound_message.id)) + + assert result["dropped_by"].endswith(":after_spam") + mock_check_spam.assert_called_once() + mock_create_message.assert_not_called() + + @patch("core.mda.inbound_tasks._create_message_from_inbound") + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_after_spam_is_spam_header( + self, mock_session, mock_check_spam, mock_create_message + ): + mailbox = factories.MailboxFactory() + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: test\r\n" + b"Message-ID: \r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "after_spam", + }, + ) + mock_check_spam.return_value = (True, None, None) + mock_session.return_value.post.return_value = _make_response(200) + mock_create_message.return_value = True + + with patch.object(process_inbound_message_task, "update_state", Mock()): + process_inbound_message_task.run(str(inbound_message.id)) + + # is_spam=True surfaces as the X-StMsg-Is-Spam header. + headers = mock_session.return_value.post.call_args.kwargs["headers"] + assert headers["X-StMsg-Is-Spam"] == "true" + assert headers["X-StMsg-Phase"] == "after_spam" + + +# Keep dj_timezone import used to silence "imported but unused" if the +# linter wakes up after edits; it's referenced from fixtures via factories. +_ = dj_timezone + + +# --- response body parsing --- # + + +class TestClassifyResponseBody: + """``_classify_response_body`` is the only thing that lets a + receiver shape delivery beyond accept/drop. Cover the JSON contract + carefully so a typo in production doesn't silently mis-route mail.""" + + def test_empty_body_is_continue(self): + outcome = _classify_response_body(b"") + assert outcome.decision == Decision.CONTINUE + assert outcome.is_spam_override is None + assert outcome.labels == set() + + def test_non_json_body_is_continue(self): + outcome = _classify_response_body(b"OK") + assert outcome.decision == Decision.CONTINUE + + def test_json_array_is_continue(self): + """Only top-level objects are interpreted as the contract.""" + outcome = _classify_response_body(b'["drop"]') + assert outcome.decision == Decision.CONTINUE + + def test_non_bytes_input_is_continue(self): + """Defensive against Mock or str leaking from tests/middleware.""" + outcome = _classify_response_body(None) # type: ignore[arg-type] + assert outcome.decision == Decision.CONTINUE + + def test_action_drop_sets_drop(self): + outcome = _classify_response_body(b'{"action": "drop"}') + assert outcome.decision == Decision.DROP + + def test_action_accept_is_continue(self): + outcome = _classify_response_body(b'{"action": "accept"}') + assert outcome.decision == Decision.CONTINUE + + def test_action_unknown_is_continue(self): + """An unknown action falls through to CONTINUE — receivers + adding new verbs we don't know about shouldn't surprise-drop.""" + outcome = _classify_response_body(b'{"action": "quarantine"}') + assert outcome.decision == Decision.CONTINUE + + def test_is_spam_true_sets_override(self): + outcome = _classify_response_body(b'{"is_spam": true}') + assert outcome.decision == Decision.CONTINUE + assert outcome.is_spam_override is True + + def test_is_spam_false_sets_override_explicitly(self): + """Distinguish ham (explicit false) from no-opinion (missing).""" + outcome = _classify_response_body(b'{"is_spam": false}') + assert outcome.is_spam_override is False + + def test_is_spam_non_bool_is_ignored(self): + """A receiver returning "true"/"false" as strings is ignored — + keeps the contract strict.""" + outcome = _classify_response_body(b'{"is_spam": "true"}') + assert outcome.is_spam_override is None + + def test_labels_uuids_collected(self): + a = str(uuid.uuid4()) + b = str(uuid.uuid4()) + outcome = _classify_response_body( + json.dumps({"labels": [a, b]}).encode("utf-8") + ) + assert outcome.labels == {a, b} + + def test_labels_non_uuid_strings_skipped(self): + good = str(uuid.uuid4()) + outcome = _classify_response_body( + json.dumps({"labels": [good, "not-a-uuid", "", 42]}).encode("utf-8") + ) + assert outcome.labels == {good} + + def test_labels_non_list_ignored(self): + outcome = _classify_response_body(b'{"labels": "spam"}') + assert outcome.labels == set() + + def test_combined_action_and_labels(self): + """Drop + labels: drop wins; labels are still collected (caller + won't apply them since the thread is never created, but the + merge logic shouldn't lose them either).""" + good = str(uuid.uuid4()) + outcome = _classify_response_body( + json.dumps({"action": "drop", "labels": [good]}).encode("utf-8") + ) + assert outcome.decision == Decision.DROP + assert outcome.labels == {good} + + def test_assign_to_emails_lowercased_and_ordered(self): + outcome = _classify_response_body( + json.dumps({"assign_to": ["Alice@example.org", "bob@example.org"]}).encode( + "utf-8" + ) + ) + # Lowercased, order preserved. + assert outcome.assign_to == ["alice@example.org", "bob@example.org"] + + def test_assign_to_dedupes_case_insensitive(self): + outcome = _classify_response_body( + json.dumps( + {"assign_to": ["alice@example.org", "ALICE@example.org"]} + ).encode("utf-8") + ) + assert outcome.assign_to == ["alice@example.org"] + + def test_assign_to_skips_non_strings_and_non_emails(self): + """Garbage doesn't pollute the list. Real users go through.""" + outcome = _classify_response_body( + json.dumps( + { + "assign_to": [ + "alice@example.org", + "", # empty after strip + "no-at-sign", # no '@' + 42, # not a string + None, # not a string + ] + } + ).encode("utf-8") + ) + assert outcome.assign_to == ["alice@example.org"] + + def test_assign_to_non_list_ignored(self): + outcome = _classify_response_body(b'{"assign_to": "alice@example.org"}') + assert outcome.assign_to == [] + + def test_bool_flags_only_true_is_honoured(self): + """``true``-only semantics — false / missing / non-bool = no opinion.""" + outcome = _classify_response_body( + json.dumps( + { + "mark_starred": True, + "mark_read": True, + "mark_trashed": True, + "mark_archived": True, + "skip_autoreply": True, + } + ).encode("utf-8") + ) + assert outcome.mark_starred is True + assert outcome.mark_read is True + assert outcome.mark_trashed is True + assert outcome.mark_archived is True + assert outcome.skip_autoreply is True + + def test_bool_flags_false_is_no_op(self): + """Explicit ``false`` is the same as missing — no opinion. Lets + a later webhook's ``true`` survive without being veto'd.""" + outcome = _classify_response_body( + json.dumps( + { + "mark_starred": False, + "mark_read": "yes", # non-bool: dropped + "mark_trashed": 1, # non-bool: dropped + } + ).encode("utf-8") + ) + assert outcome.mark_starred is False + assert outcome.mark_read is False + assert outcome.mark_trashed is False + + def test_add_event_im(self): + outcome = _classify_response_body( + json.dumps( + { + "add_event": [ + {"type": "im", "content": "AI flagged: urgent"}, + {"type": "im", "content": " "}, # blank → skip + {"type": "im"}, # no content → skip + {"type": "iframe", "url": "https://x"}, # unknown type → skip + "not a dict", # not a dict → skip + ] + } + ).encode("utf-8") + ) + # Only the well-formed IM survived. + assert outcome.events == [ + {"type": "im", "content": "AI flagged: urgent", "mentions": []} + ] + + def test_add_event_non_list_ignored(self): + outcome = _classify_response_body(b'{"add_event": {"type": "im"}}') + assert outcome.events == [] + + def test_reply_draft_template_uuid_canonicalised(self): + tmpl_id = str(uuid.uuid4()) + outcome = _classify_response_body( + json.dumps({"reply_draft": {"template": tmpl_id}}).encode("utf-8") + ) + assert outcome.reply_draft_template_id == tmpl_id + + def test_reply_draft_non_uuid_template_rejected(self): + outcome = _classify_response_body( + b'{"reply_draft": {"template": "not-a-uuid"}}' + ) + assert outcome.reply_draft_template_id is None + + def test_reply_draft_missing_template_field_rejected(self): + outcome = _classify_response_body(b'{"reply_draft": {}}') + assert outcome.reply_draft_template_id is None + + def test_reply_draft_non_object_ignored(self): + outcome = _classify_response_body(b'{"reply_draft": "template-id"}') + assert outcome.reply_draft_template_id is None + + def test_oversize_arrays_are_capped(self): + """A receiver can't flood us with arbitrary numbers of labels / + assignees / events from one webhook call. Entries past the + per-action cap are silently dropped at parse time.""" + from core.mda.dispatch_webhooks import ( + MAX_ASSIGN_TO_PER_RESPONSE, + MAX_EVENTS_PER_RESPONSE, + MAX_LABELS_PER_RESPONSE, + ) + + labels = [str(uuid.uuid4()) for _ in range(MAX_LABELS_PER_RESPONSE + 10)] + emails = [f"u{i}@example.org" for i in range(MAX_ASSIGN_TO_PER_RESPONSE + 10)] + events = [ + {"type": "im", "content": f"#{i}"} + for i in range(MAX_EVENTS_PER_RESPONSE + 10) + ] + outcome = _classify_response_body( + json.dumps( + {"labels": labels, "assign_to": emails, "add_event": events} + ).encode("utf-8") + ) + assert len(outcome.labels) == MAX_LABELS_PER_RESPONSE + assert len(outcome.assign_to) == MAX_ASSIGN_TO_PER_RESPONSE + assert len(outcome.events) == MAX_EVENTS_PER_RESPONSE + + def test_im_content_is_truncated_at_cap(self): + """A single IM comment is bounded so a misconfigured receiver + can't flood the timeline with multi-KB blobs per inbound.""" + from core.mda.dispatch_webhooks import MAX_IM_CONTENT_BYTES + + big = "x" * (MAX_IM_CONTENT_BYTES + 1000) + outcome = _classify_response_body( + json.dumps({"add_event": [{"type": "im", "content": big}]}).encode("utf-8") + ) + # Truncated; still landed. + assert len(outcome.events) == 1 + assert len(outcome.events[0]["content"]) <= MAX_IM_CONTENT_BYTES + + +# --- WebhookOutcome.merge precedence --- # + + +# (merge() and WebhookOutcome no longer exist — the pipeline applies +# side effects to InboundContext directly. Multi-step semantics +# (DROP-wins / labels-accumulate / is_spam-last-wins) are exercised by +# TestDispatchActionBody and TestPipelineIntegration below.) + + +# --- dispatch_webhooks JSON action body --- # + + +@pytest.mark.django_db +class TestDispatchActionBody: + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_blocking_drops_on_action_drop_body( + self, mock_session, mailbox, parsed_email + ): + """HTTP 200 + {"action":"drop"} → DROP (receiver chose to reject).""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "blocking": True, + }, + ) + mock_session.return_value.post.return_value = _make_response( + 200, body=b'{"action": "drop"}' + ) + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + assert outcome.decision == Decision.DROP + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_blocking_is_spam_override_continues( + self, mock_session, mailbox, parsed_email + ): + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "blocking": True, + }, + ) + mock_session.return_value.post.return_value = _make_response( + 200, body=b'{"is_spam": true}' + ) + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + assert outcome.decision == Decision.CONTINUE + assert outcome.is_spam_override is True + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_non_blocking_ignores_action_body( + self, mock_session, mailbox, parsed_email + ): + """Non-blocking webhooks are fire-and-forget. A receiver's body + should never affect delivery — protects against a non-blocking + webhook accidentally returning {"action":"drop"}.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "blocking": False, + }, + ) + mock_session.return_value.post.return_value = _make_response( + 200, body=b'{"action": "drop", "is_spam": true}' + ) + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + assert outcome.decision == Decision.CONTINUE + assert outcome.is_spam_override is None + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_multi_webhook_drop_wins_and_short_circuits( + self, mock_session, mailbox, parsed_email + ): + """When two blocking webhooks fire, DROP from one stops the chain.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com/first", + "events": ["message.received"], + "blocking": True, + }, + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com/second", + "events": ["message.received"], + "blocking": True, + }, + ) + # First call drops, second should never fire. + mock_session.return_value.post.return_value = _make_response( + 200, body=b'{"action": "drop"}' + ) + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + assert outcome.decision == Decision.DROP + # Exactly one call — the second channel never fires. + assert mock_session.return_value.post.call_count == 1 + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_response_body_is_capped(self, mock_session, mailbox, parsed_email): + """A malicious / misconfigured receiver returning a multi-MB + response must not OOM the worker. We read up to + ``MAX_RESPONSE_BODY`` bytes via ``iter_content`` and ignore the + rest.""" + from core.mda.dispatch_webhooks import MAX_RESPONSE_BODY + + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "blocking": True, + }, + ) + # Feed three chunks that together exceed the cap — the reader + # must stop part-way and never request a fourth chunk. + oversize_chunk = b"x" * (MAX_RESPONSE_BODY // 2) + response = _make_response(200) + response.iter_content = Mock( + return_value=iter([oversize_chunk, oversize_chunk, oversize_chunk]) + ) + mock_session.return_value.post.return_value = response + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + # Body was unparseable (all 'x'), so the result is plain CONTINUE. + assert outcome.decision == Decision.CONTINUE + # And the connection was returned to the pool. + response.close.assert_called_once() + + +# --- pipeline integration: RETRY, label apply, antispam override --- # + + +@pytest.mark.django_db +class TestPipelineRetry: + @patch("core.mda.inbound_tasks._create_message_from_inbound") + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_5xx_retries_and_keeps_inbound_message( + self, mock_session, mock_check_spam, mock_create_message + ): + """Transient 5xx leaves the InboundMessage row in place for the + 5-minute sweep — no rspamd, no message creation.""" + mailbox = factories.MailboxFactory() + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: test\r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "before_spam", + "blocking": True, + }, + ) + mock_session.return_value.post.return_value = _make_response(503) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + result = process_inbound_message_task.run(str(inbound_message.id)) + + assert result["error"] == "retry" + assert result["step"].endswith(":before_spam") + # Row preserved → next sweep can retry. + assert models.InboundMessage.objects.filter(id=inbound_message.id).exists() + mock_check_spam.assert_not_called() + mock_create_message.assert_not_called() + + @patch("core.mda.inbound_tasks._create_message_from_inbound") + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_timeout_retries_and_keeps_inbound_message( + self, mock_session, mock_check_spam, mock_create_message + ): + """A timeout must NOT drop the message — that was the original bug.""" + mailbox = factories.MailboxFactory() + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: test\r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "before_spam", + "blocking": True, + }, + ) + mock_session.return_value.post.side_effect = requests_lib.Timeout("timed out") + + with patch.object(process_inbound_message_task, "update_state", Mock()): + result = process_inbound_message_task.run(str(inbound_message.id)) + + assert result["error"] == "retry" + assert models.InboundMessage.objects.filter(id=inbound_message.id).exists() + mock_check_spam.assert_not_called() + mock_create_message.assert_not_called() + + @patch("core.mda.inbound_tasks._create_message_from_inbound") + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_retry_past_max_age_drops_with_loud_log( + self, mock_session, mock_check_spam, mock_create_message + ): + """An InboundMessage held in retry for >7 days is dropped to + prevent a broken receiver from pinning the row forever.""" + mailbox = factories.MailboxFactory() + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: test\r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + # Backdate past the 7-day cap. + models.InboundMessage.objects.filter(id=inbound_message.id).update( + created_at=dj_timezone.now() + - RETRY_MAX_AGE + - dj_timezone.timedelta(minutes=1) + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "before_spam", + "blocking": True, + }, + ) + mock_session.return_value.post.return_value = _make_response(503) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + result = process_inbound_message_task.run(str(inbound_message.id)) + + assert result["error"] == "retry_exhausted" + # Row is gone — bounded retry budget. + assert not models.InboundMessage.objects.filter(id=inbound_message.id).exists() + mock_check_spam.assert_not_called() + mock_create_message.assert_not_called() + + +@pytest.mark.django_db +class TestPipelineWebhookAntispam: + @patch("core.mda.inbound_tasks._create_message_from_inbound") + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_before_spam_is_spam_override_short_circuits_rspamd( + self, mock_session, mock_check_spam, mock_create_message + ): + """A before_spam webhook returning {"is_spam": true} replaces + the rspamd verdict entirely — receivers can reimplement antispam.""" + mailbox = factories.MailboxFactory() + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: test\r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "before_spam", + "blocking": True, + }, + ) + mock_session.return_value.post.return_value = _make_response( + 200, body=b'{"is_spam": true}' + ) + mock_create_message.return_value = Mock(spec=models.Message) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + result = process_inbound_message_task.run(str(inbound_message.id)) + + # rspamd was skipped because the webhook decided. + mock_check_spam.assert_not_called() + assert result["is_spam"] is True + assert mock_create_message.call_args.kwargs["is_spam"] is True + + @patch("core.mda.inbound_tasks._create_message_from_inbound") + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_after_spam_is_spam_override_replaces_verdict( + self, mock_session, mock_check_spam, mock_create_message + ): + """An after_spam webhook can flip rspamd's verdict — e.g. a + reputation service deciding "actually, this is spam".""" + mailbox = factories.MailboxFactory() + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: test\r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "after_spam", + "blocking": True, + }, + ) + # rspamd says ham; webhook says spam. + mock_check_spam.return_value = (False, None, None) + mock_session.return_value.post.return_value = _make_response( + 200, body=b'{"is_spam": true}' + ) + mock_create_message.return_value = Mock(spec=models.Message) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + process_inbound_message_task.run(str(inbound_message.id)) + + # The webhook flip wins. + assert mock_create_message.call_args.kwargs["is_spam"] is True + + +@pytest.mark.django_db +class TestPipelineWebhookLabels: + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_webhook_label_attached_to_thread(self, mock_session, mock_check_spam): + """Labels from a blocking webhook are attached to the new thread, + but only when the UUID resolves to a label in the receiving + mailbox (unknown UUIDs are skipped, not raised).""" + mailbox = factories.MailboxFactory() + good_label = factories.LabelFactory(mailbox=mailbox) + other_mailbox = factories.MailboxFactory() + other_label = factories.LabelFactory(mailbox=other_mailbox) + unknown_id = str(uuid.uuid4()) + + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: hello\r\n" + b"Message-ID: \r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "after_spam", + "blocking": True, + }, + ) + mock_check_spam.return_value = (False, None, None) + mock_session.return_value.post.return_value = _make_response( + 200, + body=json.dumps( + { + "labels": [ + str(good_label.id), + str(other_label.id), # wrong mailbox → skipped + unknown_id, # unknown UUID → skipped + ] + } + ).encode("utf-8"), + ) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + process_inbound_message_task.run(str(inbound_message.id)) + + message = models.Message.objects.get(mime_id="label-1@example.com") + thread_labels = set(message.thread.labels.values_list("id", flat=True)) + assert good_label.id in thread_labels + assert other_label.id not in thread_labels + + +@pytest.mark.django_db +class TestPipelineWebhookAssign: + """``assign_to`` in the webhook response body resolves OIDC emails + to users, filters by editor-rights on the thread, and produces one + ``ThreadEvent ASSIGN`` per webhook channel that asked. Unknown, + ambiguous, and non-assignable users are silently skipped — delivery + is never blocked because of an assign hiccup.""" + + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_assign_to_resolves_email_and_attributes_channel( + self, mock_session, mock_check_spam + ): + mailbox = factories.MailboxFactory() + editor_user = factories.UserFactory(email="editor@example.org") + factories.MailboxAccessFactory( + mailbox=mailbox, + user=editor_user, + role=enums.MailboxRoleChoices.EDITOR, + ) + + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: assign me\r\n" + b"Message-ID: \r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + channel = factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "after_spam", + "blocking": True, + }, + ) + mock_check_spam.return_value = (False, None, None) + # Email case differs from User.email to exercise iexact. + mock_session.return_value.post.return_value = _make_response( + 200, + body=json.dumps({"assign_to": ["EDITOR@example.org"]}).encode("utf-8"), + ) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + process_inbound_message_task.run(str(inbound_message.id)) + + thread = models.Message.objects.get(mime_id="assign-ok@example.com").thread + events = list( + models.ThreadEvent.objects.filter( + thread=thread, type=enums.ThreadEventTypeChoices.ASSIGN + ) + ) + assert len(events) == 1 + event = events[0] + # Channel FK preserved. + assert event.channel_id == channel.id + # Author intentionally None for webhook-driven assigns. + assert event.author_id is None + # Assignee resolved and present. + assert event.data["assignees"][0]["id"] == str(editor_user.id) + # And the per-user UserEvent landed (source of truth for + # "currently assigned"). + assert models.UserEvent.objects.filter( + user=editor_user, + thread=thread, + type=enums.UserEventTypeChoices.ASSIGN, + ).exists() + + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_assign_to_skips_unknown_ambiguous_and_viewer( + self, mock_session, mock_check_spam + ): + mailbox = factories.MailboxFactory() + editor_user = factories.UserFactory(email="editor@example.org") + factories.MailboxAccessFactory( + mailbox=mailbox, + user=editor_user, + role=enums.MailboxRoleChoices.EDITOR, + ) + # Viewer has access but the role isn't assignable. + viewer_user = factories.UserFactory(email="viewer@example.org") + factories.MailboxAccessFactory( + mailbox=mailbox, + user=viewer_user, + role=enums.MailboxRoleChoices.VIEWER, + ) + # Ambiguous: two distinct users sharing the same email (this is + # storable per the OIDC fallback model — see User.email + # comment). + factories.UserFactory(email="dup@example.org") + factories.UserFactory(email="dup@example.org") + + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: assign mixed\r\n" + b"Message-ID: \r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "after_spam", + "blocking": True, + }, + ) + mock_check_spam.return_value = (False, None, None) + mock_session.return_value.post.return_value = _make_response( + 200, + body=json.dumps( + { + "assign_to": [ + "editor@example.org", # OK + "viewer@example.org", # has access but VIEWER → skipped + "unknown@example.org", # no User row → skipped + "dup@example.org", # ≥2 matches → skipped + ] + } + ).encode("utf-8"), + ) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + process_inbound_message_task.run(str(inbound_message.id)) + + thread = models.Message.objects.get(mime_id="assign-mixed@example.com").thread + # Only the editor lands in the timeline. + assignees = list( + models.UserEvent.objects.filter( + thread=thread, type=enums.UserEventTypeChoices.ASSIGN + ).values_list("user_id", flat=True) + ) + assert assignees == [editor_user.id] + + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_two_webhooks_each_produce_own_threadevent( + self, mock_session, mock_check_spam + ): + """One ``ThreadEvent ASSIGN`` per blocking webhook that asked, + each carrying its own ``channel`` FK. Webhooks asking for the + same user are absorbed by the partial UniqueConstraint, so the + second ThreadEvent simply ends up empty and returns None.""" + mailbox = factories.MailboxFactory() + alice = factories.UserFactory(email="alice@example.org") + bob = factories.UserFactory(email="bob@example.org") + for u in (alice, bob): + factories.MailboxAccessFactory( + mailbox=mailbox, + user=u, + role=enums.MailboxRoleChoices.EDITOR, + ) + + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: multi\r\n" + b"Message-ID: \r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + ch_a = factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com/a", + "events": ["message.received"], + "phase": "after_spam", + "blocking": True, + }, + ) + ch_b = factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com/b", + "events": ["message.received"], + "phase": "after_spam", + "blocking": True, + }, + ) + mock_check_spam.return_value = (False, None, None) + # Each webhook returns a distinct assignee. The mock fires both + # in order (dispatcher iterates channels in DB order). + mock_session.return_value.post.side_effect = [ + _make_response(200, body=b'{"assign_to": ["alice@example.org"]}'), + _make_response(200, body=b'{"assign_to": ["bob@example.org"]}'), + ] + + with patch.object(process_inbound_message_task, "update_state", Mock()): + process_inbound_message_task.run(str(inbound_message.id)) + + thread = models.Message.objects.get(mime_id="assign-multi@example.com").thread + events = list( + models.ThreadEvent.objects.filter( + thread=thread, type=enums.ThreadEventTypeChoices.ASSIGN + ).order_by("created_at") + ) + # One event per webhook that contributed new assignees. + assert len(events) == 2 + assert {e.channel_id for e in events} == {ch_a.id, ch_b.id} + # Both users actually assigned. + assert set( + models.UserEvent.objects.filter( + thread=thread, type=enums.UserEventTypeChoices.ASSIGN + ).values_list("user_id", flat=True) + ) == {alice.id, bob.id} + + +@pytest.mark.django_db +class TestPipelineWebhookFlagActions: + """Blocking webhooks can flip per-message state flags + (``star`` / ``mark_read`` / ``mark_trashed`` / ``mark_archived`` / + ``skip_autoreply``). The pipeline applies them after the message + + thread land; failures never block delivery.""" + + def _send(self, mailbox, mime_id, action_body: bytes): + """Create a minimal inbound message + after-spam blocking webhook + channel, point the SSRFSafeSession mock at ``action_body``, and + run the task. Returns the resulting ``Message``.""" + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: action\r\n" + b"Message-ID: <" + mime_id.encode() + b">\r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "after_spam", + "blocking": True, + }, + ) + with ( + patch("core.mda.inbound_pipeline._call_rspamd") as mock_rspamd, + patch("core.mda.dispatch_webhooks.SSRFSafeSession") as mock_session, + ): + mock_rspamd.return_value = (False, None, None) + mock_session.return_value.post.return_value = _make_response( + 200, body=action_body + ) + with patch.object(process_inbound_message_task, "update_state", Mock()): + process_inbound_message_task.run(str(inbound_message.id)) + return models.Message.objects.get(mime_id=mime_id) + + def test_mark_starred_and_mark_read_set_threadaccess_fields(self): + mailbox = factories.MailboxFactory() + message = self._send( + mailbox, + "flag-starred@example.com", + b'{"mark_starred": true, "mark_read": true}', + ) + access = models.ThreadAccess.objects.get(thread=message.thread, mailbox=mailbox) + assert access.starred_at is not None + assert access.read_at is not None + + def test_mark_trashed_and_archived_set_message_fields(self): + mailbox = factories.MailboxFactory() + message = self._send( + mailbox, + "flag-trash@example.com", + b'{"mark_trashed": true, "mark_archived": true}', + ) + assert message.is_trashed is True + assert message.is_archived is True + + @patch("core.mda.autoreply.try_send_autoreply") + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_skip_autoreply_suppresses_autoreply_call( + self, mock_session, mock_rspamd, mock_autoreply + ): + """``skip_autoreply: true`` short-circuits the autoreply path + entirely — distinct from the ``is_spam=true`` route, which also + suppresses but for a different reason.""" + mailbox = factories.MailboxFactory() + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: noreply\r\n" + b"Message-ID: \r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "after_spam", + "blocking": True, + }, + ) + mock_rspamd.return_value = (False, None, None) + mock_session.return_value.post.return_value = _make_response( + 200, body=b'{"skip_autoreply": true}' + ) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + process_inbound_message_task.run(str(inbound_message.id)) + + mock_autoreply.assert_not_called() + + +@pytest.mark.django_db +class TestPipelineWebhookAddEvent: + """``add_event`` persists ``ThreadEvent`` rows attributed to the + firing channel. Today only ``type=im`` is honoured; unknown types + are silently skipped at the classifier (the contract stays forward- + compatible for future ``type=iframe``).""" + + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_add_event_im_creates_threadevent(self, mock_session, mock_rspamd): + mailbox = factories.MailboxFactory() + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: comment\r\n" + b"Message-ID: \r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + channel = factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "after_spam", + "blocking": True, + }, + ) + mock_rspamd.return_value = (False, None, None) + mock_session.return_value.post.return_value = _make_response( + 200, + body=json.dumps( + { + "add_event": [ + {"type": "im", "content": "AI summary: budget Q4"}, + ] + } + ).encode("utf-8"), + ) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + process_inbound_message_task.run(str(inbound_message.id)) + + message = models.Message.objects.get(mime_id="flag-event@example.com") + events = list( + models.ThreadEvent.objects.filter( + thread=message.thread, type=enums.ThreadEventTypeChoices.IM + ) + ) + assert len(events) == 1 + ev = events[0] + # Channel FK preserved. + assert ev.channel_id == channel.id + # Author intentionally None for webhook-driven IMs. + assert ev.author_id is None + assert ev.data == { + "content": "AI summary: budget Q4", + "mentions": [], + } + + +@pytest.mark.django_db +class TestPipelineWebhookReplyDraft: + """``reply_draft: {"template": }`` materialises a draft reply + using the autoreply path's shared record helper. The template body + lands in ``draft_blob`` (the rich-text editor's JSON shape) so the + user can refine the draft inline before sending — same UI affordance + as a hand-composed draft.""" + + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_reply_draft_creates_draft_with_template_body( + self, mock_session, mock_rspamd + ): + mailbox = factories.MailboxFactory() + template = factories.MessageTemplateFactory( + mailbox=mailbox, + type=enums.MessageTemplateTypeChoices.MESSAGE, + is_active=True, + html_body="

Thanks for your message!

", + text_body="Thanks for your message!", + raw_body={"type": "doc", "content": [{"type": "paragraph"}]}, + ) + raw_data = ( + b"From: customer@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: I need help\r\n" + b"Message-ID: \r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + channel = factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "after_spam", + "blocking": True, + }, + ) + mock_rspamd.return_value = (False, None, None) + mock_session.return_value.post.return_value = _make_response( + 200, + body=json.dumps({"reply_draft": {"template": str(template.id)}}).encode( + "utf-8" + ), + ) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + process_inbound_message_task.run(str(inbound_message.id)) + + inbound = models.Message.objects.get(mime_id="reply-draft@example.com") + draft = models.Message.objects.filter( + thread=inbound.thread, + is_draft=True, + parent=inbound, + ).first() + assert draft is not None + # Draft is attributed to the firing webhook channel. + assert draft.channel_id == channel.id + # Subject auto-prefixed with Re: + assert draft.subject.lower().startswith("re:") + # Body lands in draft_blob (editor JSON) — not in blob (the + # MIME blob the send pipeline produces). That's what lets the + # user edit it inline. + assert draft.draft_blob is not None + assert draft.blob is None + # And the bytes are exactly the template's raw_body json. + assert draft.draft_blob.get_content() == template.raw_body.encode("utf-8") + + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_reply_draft_out_of_scope_template_skipped(self, mock_session, mock_rspamd): + """A template belonging to a different mailbox / maildomain + must not be usable as a webhook reply_draft source.""" + mailbox = factories.MailboxFactory() + # Different domain, no mailbox FK back to ours. + other_mailbox = factories.MailboxFactory() + template = factories.MessageTemplateFactory( + mailbox=other_mailbox, + type=enums.MessageTemplateTypeChoices.MESSAGE, + is_active=True, + ) + raw_data = ( + b"From: c@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: nope\r\n" + b"Message-ID: \r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "after_spam", + "blocking": True, + }, + ) + mock_rspamd.return_value = (False, None, None) + mock_session.return_value.post.return_value = _make_response( + 200, + body=json.dumps({"reply_draft": {"template": str(template.id)}}).encode( + "utf-8" + ), + ) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + process_inbound_message_task.run(str(inbound_message.id)) + + inbound = models.Message.objects.get(mime_id="draft-oos@example.com") + # No draft was created — out-of-scope template silently skipped. + assert not models.Message.objects.filter( + thread=inbound.thread, + is_draft=True, + ).exists() + + +@pytest.mark.django_db +class TestFinalizeStepIsolation: + """A failure in one finalize step (labels / assigns / events / + drafts / flags) must NOT skip the others — the message has already + landed, and a partial failure on a downstream side effect should + log loudly rather than swallow other receiver-requested changes.""" + + @patch("core.mda.inbound_tasks.apply_pending_assigns") + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_assign_failure_does_not_skip_labels( + self, mock_session, mock_rspamd, mock_apply_assigns + ): + mailbox = factories.MailboxFactory() + label = factories.LabelFactory(mailbox=mailbox) + user = factories.UserFactory(email="editor@example.org") + factories.MailboxAccessFactory( + mailbox=mailbox, + user=user, + role=enums.MailboxRoleChoices.EDITOR, + ) + + # Force the assigns step to blow up — labels MUST still apply. + mock_apply_assigns.side_effect = RuntimeError("DB hiccup") + + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: isolation\r\n" + b"Message-ID: \r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.received"], + "phase": "after_spam", + "blocking": True, + }, + ) + mock_rspamd.return_value = (False, None, None) + mock_session.return_value.post.return_value = _make_response( + 200, + body=json.dumps( + { + "labels": [str(label.id)], + "assign_to": ["editor@example.org"], + } + ).encode("utf-8"), + ) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + result = process_inbound_message_task.run(str(inbound_message.id)) + + # Task reports success (message landed, finalize errors logged). + assert result["success"] is True + message = models.Message.objects.get(mime_id="isolation@example.com") + # Labels still got attached even though assigns raised. + assert label in list(message.thread.labels.all()) + mock_apply_assigns.assert_called_once() diff --git a/src/backend/core/tests/mda/test_inbound_auth.py b/src/backend/core/tests/mda/test_inbound_auth.py index d7bdecd13..ad4ebb2e2 100644 --- a/src/backend/core/tests/mda/test_inbound_auth.py +++ b/src/backend/core/tests/mda/test_inbound_auth.py @@ -603,7 +603,7 @@ class TestProcessInboundMessageAuthIntegration: """End-to-end: a verdict prepends X-StMsg-Sender-Auth with its value.""" @override_settings(SPAM_CONFIG={"inbound_auth": "native"}) - @patch("core.mda.inbound_tasks.check_inbound_authentication") + @patch("core.mda.inbound_pipeline.check_inbound_authentication") @patch("core.mda.inbound_tasks._create_message_from_inbound") def test_unverified_verdict_injects_none_header( self, mock_create_message, mock_auth_check @@ -630,7 +630,7 @@ def test_unverified_verdict_injects_none_header( ] == ["none"] @override_settings(SPAM_CONFIG={"inbound_auth": "rspamd"}) - @patch("core.mda.inbound_tasks.check_inbound_authentication") + @patch("core.mda.inbound_pipeline.check_inbound_authentication") @patch("core.mda.inbound_tasks._create_message_from_inbound") def test_forged_verdict_injects_fail_header( self, mock_create_message, mock_auth_check @@ -656,7 +656,7 @@ def test_forged_verdict_injects_fail_header( ] == ["fail"] @override_settings(SPAM_CONFIG={"inbound_auth": "native"}) - @patch("core.mda.inbound_tasks.check_inbound_authentication") + @patch("core.mda.inbound_pipeline.check_inbound_authentication") @patch("core.mda.inbound_tasks._create_message_from_inbound") def test_verified_does_not_inject_header( self, mock_create_message, mock_auth_check @@ -681,7 +681,7 @@ def test_verified_does_not_inject_header( "inbound_auth": "rspamd", } ) - @patch("core.mda.inbound_tasks.requests.post") + @patch("core.mda.inbound_pipeline.requests.post") @patch("core.mda.inbound_tasks._create_message_from_inbound") def test_rspamd_response_reused_by_auth_check(self, mock_create_message, mock_post): """Single rspamd call feeds both spam and auth.""" @@ -714,7 +714,7 @@ def test_rspamd_response_reused_by_auth_check(self, mock_create_message, mock_po "inbound_auth": "rspamd", } ) - @patch("core.mda.inbound_tasks.requests.post") + @patch("core.mda.inbound_pipeline.requests.post") @patch("core.mda.inbound_tasks._create_message_from_inbound") def test_dmarc_fail_injects_fail_header_end_to_end( self, mock_create_message, mock_post @@ -751,7 +751,7 @@ def test_dmarc_fail_injects_fail_header_end_to_end( "rules": [{"header_match": "X-Spam:yes", "action": "ham"}], } ) - @patch("core.mda.inbound_tasks.requests.post") + @patch("core.mda.inbound_pipeline.requests.post") @patch("core.mda.inbound_tasks._create_message_from_inbound") def test_rspamd_fetched_on_demand_when_spam_skipped_rspamd( self, mock_create_message, mock_post @@ -803,11 +803,16 @@ def test_header_injection_propagates_to_stmsg(self): assert values == ["fail"] @override_settings(SPAM_CONFIG={"inbound_auth": "native"}) + @patch("core.mda.inbound_pipeline.parse_email") @patch("core.mda.inbound_tasks.parse_email") - @patch("core.mda.inbound_tasks.check_inbound_authentication") + @patch("core.mda.inbound_pipeline.check_inbound_authentication") @patch("core.mda.inbound_tasks._create_message_from_inbound") def test_reparse_failure_after_prepend_keeps_views_in_sync( - self, mock_create_message, mock_auth_check, mock_parse + self, + mock_create_message, + mock_auth_check, + mock_parse_tasks, + mock_parse_pipeline, ): """If re-parsing the prepended bytes blows up, the prepend is dropped. @@ -815,13 +820,17 @@ def test_reparse_failure_after_prepend_keeps_views_in_sync( diverge, and `Message.get_parsed_data()` later returns {} for the whole message because the same bytes fail to parse at display time. """ - # First call: initial parse succeeds. Second: re-parse after prepend fails. + # Initial parse (inbound_tasks) succeeds; re-parse in the + # inbound_auth_step (inbound_pipeline) raises. original_parsed = { "headers": [{"name": "from", "value": "a@b"}], - "ext": {"headersBlocks": [{}]}, "from": [{"email": "a@b"}], } - mock_parse.side_effect = [original_parsed, None] + # Initial parse (inbound_tasks) succeeds; the inbound_auth_step + # re-parse (inbound_pipeline) returns None — parse_email signals + # failure with None rather than raising. + mock_parse_tasks.return_value = original_parsed + mock_parse_pipeline.return_value = None mock_auth_check.return_value = VERDICT_UNVERIFIED mock_create_message.return_value = True diff --git a/src/backend/core/tests/mda/test_inbound_spoofed_sender.py b/src/backend/core/tests/mda/test_inbound_spoofed_sender.py index 9511f9659..0a7b7e7db 100644 --- a/src/backend/core/tests/mda/test_inbound_spoofed_sender.py +++ b/src/backend/core/tests/mda/test_inbound_spoofed_sender.py @@ -52,7 +52,7 @@ def victim_mailbox(): class TestInboundSpoofedSender: """Inbound delivery with ``From == To`` must not flag ``is_sender=True``.""" - @patch("core.mda.inbound_tasks._check_spam_with_rspamd") + @patch("core.mda.inbound_pipeline._call_rspamd") def test_inbound_spoofed_sender_not_marked_as_sender( self, mock_rspamd, victim_mailbox ): @@ -92,7 +92,7 @@ def test_inbound_spoofed_sender_not_marked_as_sender( "as is_sender=True; otherwise retry_messages_task re-emits them." ) - @patch("core.mda.inbound_tasks._check_spam_with_rspamd") + @patch("core.mda.inbound_pipeline._call_rspamd") @patch("core.mda.outbound_tasks.send_message") def test_inbound_spoofed_sender_not_picked_up_by_retry( self, mock_send_message, mock_rspamd, victim_mailbox diff --git a/src/backend/core/tests/mda/test_spam_processing.py b/src/backend/core/tests/mda/test_spam_processing.py index 74e5dc625..e41850803 100644 --- a/src/backend/core/tests/mda/test_spam_processing.py +++ b/src/backend/core/tests/mda/test_spam_processing.py @@ -11,9 +11,8 @@ from core import factories, models from core.mda.inbound import deliver_inbound_message +from core.mda.inbound_pipeline import _call_rspamd, _check_hardcoded_rules from core.mda.inbound_tasks import ( - _check_spam_with_hardcoded_rules, - _check_spam_with_rspamd, process_inbound_message_task, process_inbound_messages_queue_task, ) @@ -88,7 +87,7 @@ class TestRspamdSpamCheck: """Test rspamd spam checking functionality.""" @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) - @patch("core.mda.inbound_tasks.requests.post") + @patch("core.mda.inbound_pipeline.requests.post") def test_check_spam_with_rspamd_spam(self, mock_post): """Test that spam messages are correctly identified.""" spam_config = {"rspamd_url": "http://rspamd:8010/_api"} @@ -102,7 +101,7 @@ def test_check_spam_with_rspamd_spam(self, mock_post): mock_post.return_value = mock_response raw_data = b"Spam email content" - is_spam, error, rspamd_result = _check_spam_with_rspamd(raw_data, spam_config) + is_spam, error, rspamd_result = _call_rspamd(raw_data, spam_config) assert is_spam is True assert error is None @@ -113,7 +112,7 @@ def test_check_spam_with_rspamd_spam(self, mock_post): assert call_args[1]["data"] == raw_data @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) - @patch("core.mda.inbound_tasks.requests.post") + @patch("core.mda.inbound_pipeline.requests.post") def test_check_spam_with_rspamd_not_spam(self, mock_post): """Test that non-spam messages are correctly identified.""" spam_config = {"rspamd_url": "http://rspamd:8010/_api"} @@ -127,7 +126,7 @@ def test_check_spam_with_rspamd_not_spam(self, mock_post): mock_post.return_value = mock_response raw_data = b"Legitimate email content" - is_spam, error, _rspamd_result = _check_spam_with_rspamd(raw_data, spam_config) + is_spam, error, _rspamd_result = _call_rspamd(raw_data, spam_config) assert is_spam is False assert error is None @@ -138,7 +137,7 @@ def test_check_spam_with_rspamd_not_spam(self, mock_post): "rspamd_auth": "Bearer token123", } ) - @patch("core.mda.inbound_tasks.requests.post") + @patch("core.mda.inbound_pipeline.requests.post") def test_check_spam_with_rspamd_auth_header(self, mock_post): """Test that Authorization header is included when configured.""" spam_config = { @@ -155,31 +154,34 @@ def test_check_spam_with_rspamd_auth_header(self, mock_post): mock_post.return_value = mock_response raw_data = b"Email content" - _check_spam_with_rspamd(raw_data, spam_config) + _call_rspamd(raw_data, spam_config) call_args = mock_post.call_args assert call_args[1]["headers"]["Authorization"] == "Bearer token123" @override_settings(SPAM_CONFIG={}) def test_check_spam_without_rspamd_config(self): - """Test that spam check is skipped when rspamd is not configured.""" + """When rspamd isn't configured the call returns ``is_spam=None`` + (= "no opinion"), so the pipeline falls through to whatever + verdict is already in the context instead of silently marking + the message as ham.""" spam_config = {} raw_data = b"Email content" - is_spam, error, rspamd_result = _check_spam_with_rspamd(raw_data, spam_config) + is_spam, error, rspamd_result = _call_rspamd(raw_data, spam_config) - assert is_spam is False + assert is_spam is None assert error is None assert rspamd_result is None @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) - @patch("core.mda.inbound_tasks.requests.post") + @patch("core.mda.inbound_pipeline.requests.post") def test_check_spam_with_rspamd_error(self, mock_post): """Test that errors in rspamd check are handled gracefully.""" spam_config = {"rspamd_url": "http://rspamd:8010/_api"} mock_post.side_effect = requests.exceptions.RequestException("Connection error") raw_data = b"Email content" - is_spam, error, rspamd_result = _check_spam_with_rspamd(raw_data, spam_config) + is_spam, error, rspamd_result = _call_rspamd(raw_data, spam_config) # On error, treat as not spam to avoid blocking legitimate messages assert is_spam is False @@ -192,7 +194,7 @@ def test_check_spam_with_rspamd_error(self, mock_post): "rspamd_auth": "Bearer global", } ) - @patch("core.mda.inbound_tasks.requests.post") + @patch("core.mda.inbound_pipeline.requests.post") def test_check_spam_with_maildomain_override(self, mock_post): """Test that maildomain custom_settings can override SPAM_CONFIG.""" # Create a maildomain with custom spam config @@ -217,7 +219,7 @@ def test_check_spam_with_maildomain_override(self, mock_post): spam_config = mailbox.domain.get_spam_config() raw_data = b"Email content" - _check_spam_with_rspamd(raw_data, spam_config) + _call_rspamd(raw_data, spam_config) # Verify that the domain-specific URL was used call_args = mock_post.call_args @@ -241,7 +243,7 @@ def test_check_spam_with_hardcoded_rules_spam(self): parsed_email = parse_email(raw_email) spam_config = {"rules": [{"header_match": "X-Spam:yes", "action": "spam"}]} - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -257,7 +259,7 @@ def test_check_spam_with_hardcoded_rules_ham(self): parsed_email = parse_email(raw_email) spam_config = {"rules": [{"header_match": "X-Spam:no", "action": "ham"}]} - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is False @@ -273,7 +275,7 @@ def test_check_spam_with_hardcoded_rules_no_match(self): parsed_email = parse_email(raw_email) spam_config = {"rules": [{"header_match": "X-Spam:yes", "action": "spam"}]} - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is None @@ -289,7 +291,7 @@ def test_check_spam_with_hardcoded_rules_no_rules(self): parsed_email = parse_email(raw_email) spam_config = {} - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is None @@ -310,7 +312,7 @@ def test_check_spam_with_hardcoded_rules_multiple_headers(self): parsed_email = parse_email(raw_email) spam_config = {"rules": [{"header_match": "X-Spam:no", "action": "ham"}]} - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is False @@ -326,7 +328,7 @@ def test_check_spam_with_hardcoded_rules_case_insensitive(self): parsed_email = parse_email(raw_email) spam_config = {"rules": [{"header_match": "X-Spam:yes", "action": "spam"}]} - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -344,7 +346,7 @@ def test_check_spam_with_hardcoded_rules_value_with_colon(self): "rules": [{"header_match": "X-Custom:value:with:colons", "action": "spam"}] } - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -362,7 +364,7 @@ def test_check_spam_with_hardcoded_rules_header_match_regex_spam(self): "rules": [{"header_match_regex": "X-Spam:.*spam.*", "action": "spam"}] } - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -380,7 +382,7 @@ def test_check_spam_with_hardcoded_rules_header_match_regex_spam_no_fullmatch(se "rules": [{"header_match_regex": "X-Spam:spam", "action": "spam"}] } - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is None @@ -398,7 +400,7 @@ def test_check_spam_with_hardcoded_rules_header_match_regex_case_insensitive(sel "rules": [{"header_match_regex": "X-Spam:.*spam.*", "action": "spam"}] } - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -416,7 +418,7 @@ def test_check_spam_with_hardcoded_rules_header_match_regex_pattern(self): "rules": [{"header_match_regex": "X-Spam-Level:[4-9]", "action": "spam"}] } - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -436,7 +438,7 @@ def test_check_spam_with_hardcoded_rules_default_action(self): ] } - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -452,7 +454,7 @@ def test_check_spam_with_hardcoded_rules_reject_action(self): parsed_email = parse_email(raw_email) spam_config = {"rules": [{"header_match": "X-Spam:yes", "action": "reject"}]} - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -468,7 +470,7 @@ def test_check_spam_with_hardcoded_rules_no_action(self): parsed_email = parse_email(raw_email) spam_config = {"rules": [{"header_match": "X-Spam:no", "action": "no action"}]} - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is False @@ -494,7 +496,7 @@ def test_check_spam_with_hardcoded_rules_multiple_rules_order(self): ] } - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) # Should return False (ham) because second rule matched first # Third rule should not be evaluated @@ -519,7 +521,7 @@ def test_check_spam_with_hardcoded_rules_multiple_rules_first_match_wins(self): ] } - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) # Should return True (spam) because first rule matched # Second rule should not be evaluated @@ -547,7 +549,7 @@ def test_check_spam_with_hardcoded_rules_x_spam_single_relay(self): "trusted_relays": 1, # Trust block 0 and block 1 } - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -586,7 +588,7 @@ def test_check_spam_with_hardcoded_rules_x_spam_raw_email_relay_no_header(self): "trusted_relays": 1, # Trust block 0 and block 1 } - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) # Should return None (no match) because sender's X-Spam is in block 2, not in trusted blocks assert result is None @@ -630,7 +632,7 @@ def test_check_spam_with_hardcoded_rules_x_spam_raw_email_with_relay( "trusted_relays": 1, # Trust block 0 and block 1 } - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) # Should match the first X-Spam header (No from last relay), not the sender's (Yes) assert result is False # ham = False (not spam) @@ -689,7 +691,7 @@ def test_check_spam_with_hardcoded_rules_trusted_relays( "trusted_relays": trusted_relays_setting, } - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is expected_result def test_default_ignores_sender_injected_ham_header(self): @@ -723,7 +725,7 @@ def test_default_ignores_sender_injected_ham_header(self): ], } - result = _check_spam_with_hardcoded_rules(parsed_email, spam_config) + result = _check_hardcoded_rules(parsed_email, spam_config) assert result is None # forged ham not honoured @@ -732,7 +734,7 @@ class TestProcessInboundMessageTask: """Test the process_inbound_message_task.""" @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) - @patch("core.mda.inbound_tasks._check_spam_with_rspamd") + @patch("core.mda.inbound_pipeline._call_rspamd") @patch("core.mda.inbound_tasks._create_message_from_inbound") def test_process_inbound_message_task_spam( self, mock_create_message, mock_check_spam @@ -765,7 +767,7 @@ def test_process_inbound_message_task_spam( assert not models.InboundMessage.objects.filter(id=inbound_message.id).exists() @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) - @patch("core.mda.inbound_tasks._check_spam_with_rspamd") + @patch("core.mda.inbound_pipeline._call_rspamd") @patch("core.mda.inbound_tasks._create_message_from_inbound") def test_process_inbound_message_task_not_spam( self, mock_create_message, mock_check_spam @@ -794,7 +796,7 @@ def test_process_inbound_message_task_not_spam( assert call_kwargs["is_spam"] is False @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) - @patch("core.mda.inbound_tasks._check_spam_with_rspamd") + @patch("core.mda.inbound_pipeline._call_rspamd") @patch("core.mda.inbound_tasks._create_message_from_inbound") def test_process_inbound_message_task_failure( self, mock_create_message, mock_check_spam diff --git a/src/backend/messages/settings.py b/src/backend/messages/settings.py index 64e5d0a0b..1df9efc9a 100644 --- a/src/backend/messages/settings.py +++ b/src/backend/messages/settings.py @@ -1101,13 +1101,8 @@ class Base(Configuration): FEATURE_IMPORT_MESSAGES = values.BooleanValue( default=True, environ_name="FEATURE_IMPORT_MESSAGES", environ_prefix=None ) - # NOTE: "webhook" is intentionally NOT in the default list — the - # outbound webhook delivery pipeline is not wired yet. Keeping the - # type creatable would let users mint dead-letter channels that look - # functional. Add "webhook" here once core/mda/webhook_tasks.py and - # the post_save signal land. FEATURE_MAILBOX_ADMIN_CHANNELS = values.ListValue( - default=["api_key"], + default=["api_key", "webhook"], environ_name="FEATURE_MAILBOX_ADMIN_CHANNELS", environ_prefix=None, ) @@ -1481,7 +1476,7 @@ class Test(Base): # Add a test encryption key for django-fernet-encrypted-fields SALT_KEY = ["test-salt-for-development-only"] - FEATURE_MAILBOX_ADMIN_CHANNELS = ["api_key", "widget"] + FEATURE_MAILBOX_ADMIN_CHANNELS = ["api_key", "widget", "webhook"] SCHEMA_CUSTOM_ATTRIBUTES_USER = {} SCHEMA_CUSTOM_ATTRIBUTES_MAILDOMAIN = {} diff --git a/src/frontend/src/features/layouts/components/mailbox-settings/integrations-view/integrations-data-grid.tsx b/src/frontend/src/features/layouts/components/mailbox-settings/integrations-view/integrations-data-grid.tsx index 2d7089939..7c04fd84e 100644 --- a/src/frontend/src/features/layouts/components/mailbox-settings/integrations-view/integrations-data-grid.tsx +++ b/src/frontend/src/features/layouts/components/mailbox-settings/integrations-view/integrations-data-grid.tsx @@ -26,6 +26,8 @@ const getChannelTypeLabel = (type: string | undefined, t: (key: string) => strin return t("Widget"); case "api_key": return t("API Key"); + case "webhook": + return t("Webhook"); default: return type; } @@ -37,6 +39,8 @@ const getChannelTypeIcon = (type: string | undefined) => { return "widgets"; case "api_key": return "key"; + case "webhook": + return "webhook"; default: return "integration_instructions"; } diff --git a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/index.tsx b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/index.tsx index 6213ad254..6decf8225 100644 --- a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/index.tsx +++ b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/index.tsx @@ -5,6 +5,7 @@ import { useTranslation } from "react-i18next"; import { useState, useEffect } from "react"; import { Channel, Mailbox } from "@/features/api/gen"; import { WidgetIntegrationForm } from "./widget-integration-form"; +import { WebhookIntegrationForm } from "./webhook-integration-form"; import { useConfig } from "@/features/providers/config"; import i18n from "@/features/i18n/initI18n"; @@ -16,7 +17,7 @@ type ModalComposeIntegrationProps = { onSuccess?: () => void; }; -type ChannelType = "widget" | "api_key"; +type ChannelType = "widget" | "api_key" | "webhook"; type ViewState = "select_type" | "form"; type ChannelTypeCardProps = { @@ -49,6 +50,12 @@ const CHANNEL_TYPE_METADATA: Record = { icon: "key", disabled: true }, + webhook: { + type: "webhook", + title: i18n.t("Outbound Webhook"), + description: i18n.t("Forward every incoming message to a URL of your choice as a JSON POST."), + icon: "webhook", + }, }; const ChannelTypeCard = ({ title, description, icon, disabled, onClick }: ChannelTypeCardProps) => { @@ -150,6 +157,9 @@ export const ModalComposeIntegration = ({ else if (selectedType === "widget") { label = isEditing ? t("Edit Widget") : t("Create a Widget"); } + else if (selectedType === "webhook") { + label = isEditing ? t("Edit Webhook") : t("Create a Webhook"); + } else { label = t("Integrations"); } @@ -201,6 +211,14 @@ export const ModalComposeIntegration = ({ onClose={onClose} /> )} + {viewState === "form" && selectedType === "webhook" && ( + + )} ); diff --git a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx new file mode 100644 index 000000000..e86567d2d --- /dev/null +++ b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx @@ -0,0 +1,380 @@ +import { Button, Input } from "@gouvfr-lasuite/cunningham-react"; +import { Icon, IconType } from "@gouvfr-lasuite/ui-kit"; +import { useTranslation } from "react-i18next"; +import { useForm, FormProvider } from "react-hook-form"; +import { zodResolver } from "@hookform/resolvers/zod"; +import * as z from "zod"; +import { useState, useMemo } from "react"; +import { useQueryClient } from "@tanstack/react-query"; +import { + Channel, + Mailbox, + useMailboxesChannelsCreate, + useMailboxesChannelsPartialUpdate, + getMailboxesChannelsListUrl, +} from "@/features/api/gen"; +import { + RhfInput, + RhfSelect, + RhfCheckbox, +} from "@/features/forms/components/react-hook-form"; +import { addToast, ToasterItem } from "@/features/ui/components/toaster"; +import { Banner } from "@/features/ui/components/banner"; +import { handle } from "@/features/utils/errors"; + +type WebhookChannelSettings = { + url?: string; + events?: string[]; + phase?: "before_spam" | "after_spam"; + format?: "eml" | "jmap" | "jmap_without_body"; + blocking?: boolean; + auth_method?: "jwt" | "api_key"; +}; + +type CreatedWebhookCredential = { + label: string; + value: string; +}; + +type WebhookIntegrationFormProps = { + mailbox: Mailbox; + channel?: Channel; + onSuccess: (channel: Channel) => void; + onClose: () => void; +}; + +const createFormSchema = (t: (key: string) => string) => + z.object({ + name: z.string().min(1, { error: t("Name is required.") }), + url: z + .string() + .min(1, { error: t("URL is required.") }) + .regex(/^https?:\/\//i, { + error: t("URL must start with http:// or https://"), + }), + phase: z.enum(["before_spam", "after_spam"]), + format: z.enum(["eml", "jmap", "jmap_without_body"]), + blocking: z.boolean(), + auth_method: z.enum(["jwt", "api_key"]), + }); + +type FormFields = z.infer>; + +export const WebhookIntegrationForm = ({ + mailbox, + channel, + onSuccess, + onClose, +}: WebhookIntegrationFormProps) => { + const { t } = useTranslation(); + const queryClient = useQueryClient(); + const [error, setError] = useState(null); + const settings = channel?.settings as WebhookChannelSettings | undefined; + const isEditing = !!channel; + + const createMutation = useMailboxesChannelsCreate(); + const updateMutation = useMailboxesChannelsPartialUpdate(); + + const formSchema = useMemo(() => createFormSchema(t), [t]); + + const form = useForm({ + resolver: zodResolver(formSchema), + defaultValues: { + name: channel?.name || "", + url: settings?.url || "", + phase: settings?.phase || "after_spam", + format: settings?.format || "eml", + blocking: settings?.blocking ?? false, + auth_method: settings?.auth_method || "jwt", + }, + }); + + const [createdCredential, setCreatedCredential] = + useState(null); + + const { + handleSubmit, + formState: { errors }, + } = form; + + const invalidateChannels = async () => { + await queryClient.invalidateQueries({ + queryKey: [getMailboxesChannelsListUrl(mailbox.id)], + exact: false, + }); + }; + + const onSubmit = async (data: FormFields) => { + setError(null); + + const newSettings: WebhookChannelSettings = { + url: data.url, + events: ["message.received"], + phase: data.phase, + format: data.format, + blocking: data.blocking, + auth_method: data.auth_method, + }; + + try { + if (isEditing && channel) { + await updateMutation.mutateAsync({ + mailboxId: mailbox.id, + id: channel.id, + data: { + name: data.name, + settings: newSettings, + }, + }); + addToast( + + {t("Integration updated!")} + , + ); + await invalidateChannels(); + } else { + const newChannel = await createMutation.mutateAsync({ + mailboxId: mailbox.id, + data: { + name: data.name, + type: "webhook", + settings: newSettings, + }, + }); + addToast( + + {t("Integration created!")} + , + ); + await invalidateChannels(); + if (newChannel.status < 300) { + // Surface the freshly minted credential exactly + // once — the receiver needs this value to verify + // every webhook we send. The backend returns + // ``webhook_secret`` for auth_method=jwt and + // ``webhook_api_key`` for auth_method=api_key. + // The one-time ``webhook_secret`` / ``webhook_api_key`` + // are create-only response fields, not part of the + // generated ``Channel`` type — read them off an + // index-signature view. + const payload = newChannel.data as unknown as Record< + string, + unknown + >; + const secret = payload.webhook_secret as + | string + | undefined; + const apiKey = payload.webhook_api_key as + | string + | undefined; + if (secret) { + setCreatedCredential({ + label: t("Webhook signing secret"), + value: secret, + }); + } else if (apiKey) { + setCreatedCredential({ + label: t("Webhook API key"), + value: apiKey, + }); + } else { + onSuccess(newChannel.data); + } + } + } + } catch (err) { + handle(err); + setError(t("An error occurred while saving the integration.")); + } + }; + + if (createdCredential && !isEditing) { + return ( +
+
+

{t("Save this credential now")}

+ + {t( + "This value is shown only once. Configure your receiver with it before closing — you can rotate it later if you need a new one.", + )} + + +
+
+ +
+
+ ); + } + + return ( + +
+
+

{t("General")}

+ +
+ +
+

{t("Endpoint")}

+ + +
+ +
+

{t("Behavior")}

+ + + + +

+ + {t( + "Non-blocking (default) is the safe choice:", + )} + {" "} + {t( + "we POST and ignore the response. Receivers cannot affect the message.", + )} +

+

+ + {t( + "Blocking lets the receiver act on this single message", + )} + {" "} + {t( + "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.", + )} +

+ +
+ + {error && {error}} + +
+ + +
+ + {!isEditing && ( +
+ +

+ {t( + "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).", + )} +

+
+ )} +
+ + ); +}; From 1c7f9991311b955acfb23c3791ed3c1ecb937d02 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Wed, 17 Jun 2026 12:59:01 +0200 Subject: [PATCH 02/21] review fixes --- src/backend/core/api/openapi.json | 165 +++++++++++++-- src/backend/core/api/serializers.py | 13 +- src/backend/core/api/viewsets/channel.py | 6 +- src/backend/core/mda/dispatch_webhooks.py | 32 ++- src/backend/core/mda/inbound_pipeline.py | 16 +- src/backend/core/models.py | 14 +- src/backend/core/tests/api/test_channels.py | 106 +++++++++- .../core/tests/mda/test_dispatch_webhooks.py | 33 ++- .../src/features/api/gen/channels/channels.ts | 191 +++++++++++------- .../api/gen/models/channel_create_response.ts | 65 ++++++ .../src/features/api/gen/models/index.ts | 3 +- .../models/regenerated_api_key_response.ts | 14 -- .../gen/models/regenerated_secret_response.ts | 18 ++ 13 files changed, 526 insertions(+), 150 deletions(-) create mode 100644 src/frontend/src/features/api/gen/models/channel_create_response.ts delete mode 100644 src/frontend/src/features/api/gen/models/regenerated_api_key_response.ts create mode 100644 src/frontend/src/features/api/gen/models/regenerated_secret_response.ts diff --git a/src/backend/core/api/openapi.json b/src/backend/core/api/openapi.json index 8be7b3b19..b2cc9b0ec 100644 --- a/src/backend/core/api/openapi.json +++ b/src/backend/core/api/openapi.json @@ -2978,11 +2978,17 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Channel" + "$ref": "#/components/schemas/ChannelCreateResponse" } } }, - "description": "" + "description": "Channel created successfully. The response carries the one-time plaintext credentials (api_key / webhook_secret / webhook_api_key / password) which are never returned again." + }, + "400": { + "description": "Invalid input data" + }, + "403": { + "description": "Permission denied" } } } @@ -3198,9 +3204,9 @@ } } }, - "/api/v1.0/mailboxes/{mailbox_id}/channels/{id}/regenerate-api-key/": { + "/api/v1.0/mailboxes/{mailbox_id}/channels/{id}/regenerate-secret/": { "post": { - "operationId": "mailboxes_channels_regenerate_api_key_create", + "operationId": "mailboxes_channels_regenerate_secret_create", "description": "Manage integration channels for a mailbox", "parameters": [ { @@ -3234,14 +3240,14 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RegeneratedApiKeyResponse" + "$ref": "#/components/schemas/RegeneratedSecretResponse" } } }, - "description": "Returns the freshly generated plaintext api_key. The previous secret is invalidated immediately. The plaintext is shown ONCE and cannot be retrieved later." + "description": "Rotates the channel's secret. Single-active: the previous credential is invalidated immediately. The response carries exactly one of ``api_key`` / ``webhook_secret`` / ``webhook_api_key`` matching the channel's type (and, for webhooks, its current ``auth_method``)." }, "400": { - "description": "Channel is not an api_key channel" + "description": "Channel type has no rotatable secret" }, "403": { "description": "Permission denied" @@ -6934,11 +6940,17 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Channel" + "$ref": "#/components/schemas/ChannelCreateResponse" } } }, - "description": "" + "description": "Channel created successfully. The response carries the one-time plaintext credentials (api_key / webhook_secret / webhook_api_key / password) which are never returned again." + }, + "400": { + "description": "Invalid input data" + }, + "403": { + "description": "Permission denied" } } } @@ -7118,9 +7130,9 @@ } } }, - "/api/v1.0/users/me/channels/{id}/regenerate-api-key/": { + "/api/v1.0/users/me/channels/{id}/regenerate-secret/": { "post": { - "operationId": "users_me_channels_regenerate_api_key_create", + "operationId": "users_me_channels_regenerate_secret_create", "description": "Manage personal (scope_level=user) integration channels", "parameters": [ { @@ -7145,14 +7157,14 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RegeneratedApiKeyResponse" + "$ref": "#/components/schemas/RegeneratedSecretResponse" } } }, - "description": "Returns the freshly generated plaintext api_key. The previous secret is invalidated immediately. The plaintext is shown ONCE and cannot be retrieved later." + "description": "Rotates the channel's secret. Single-active: the previous credential is invalidated immediately. The response carries exactly one of ``api_key`` / ``webhook_secret`` / ``webhook_api_key`` matching the channel's type (and, for webhooks, its current ``auth_method``)." }, "400": { - "description": "Channel is not an api_key channel" + "description": "Channel type has no rotatable secret" }, "403": { "description": "Permission denied" @@ -7494,6 +7506,116 @@ "user" ] }, + "ChannelCreateResponse": { + "type": "object", + "description": "Schema-only view of the channel-create 201 response.\n\n``ChannelViewSet.create`` returns the full ``ChannelSerializer``\npayload plus the freshly-minted plaintext credentials — surfaced\nexactly once on creation and never retrievable again. They are\ndeclared here as read-only fields so generated API clients see them\nin the OpenAPI schema. This serializer is never used to serialize a\nresponse directly; the view assembles the body by hand.", + "properties": { + "id": { + "type": "string", + "format": "uuid", + "readOnly": true, + "description": "primary key for the record as UUID" + }, + "name": { + "type": "string", + "description": "Human-readable name for this channel", + "maxLength": 255 + }, + "type": { + "type": "string", + "description": "Type of channel", + "maxLength": 255 + }, + "scope_level": { + "allOf": [ + { + "$ref": "#/components/schemas/ScopeLevelEnum" + } + ], + "readOnly": true + }, + "settings": { + "description": "Channel-specific configuration settings" + }, + "mailbox": { + "type": "string", + "format": "uuid", + "description": "primary key for the record as UUID", + "readOnly": true, + "nullable": true + }, + "maildomain": { + "type": "string", + "format": "uuid", + "description": "primary key for the record as UUID", + "readOnly": true, + "nullable": true + }, + "user": { + "type": "string", + "format": "uuid", + "description": "primary key for the record as UUID", + "readOnly": true, + "nullable": true + }, + "last_used_at": { + "type": "string", + "format": "date-time", + "readOnly": true, + "nullable": true + }, + "created_at": { + "type": "string", + "format": "date-time", + "readOnly": true, + "title": "Created on", + "description": "date and time at which a record was created" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "readOnly": true, + "title": "Updated on", + "description": "date and time at which a record was last updated" + }, + "api_key": { + "type": "string", + "readOnly": true, + "description": "api_key channels only — the plaintext API key." + }, + "password": { + "type": "string", + "readOnly": true, + "description": "Plaintext password, when the channel type mints one." + }, + "webhook_secret": { + "type": "string", + "readOnly": true, + "description": "webhook channels with auth_method=jwt — the HMAC/JWT signing secret." + }, + "webhook_api_key": { + "type": "string", + "readOnly": true, + "description": "webhook channels with auth_method=api_key — the derived API key." + } + }, + "required": [ + "api_key", + "created_at", + "id", + "last_used_at", + "mailbox", + "maildomain", + "name", + "password", + "scope_level", + "type", + "updated_at", + "user", + "webhook_api_key", + "webhook_secret" + ] + }, "ChannelRequest": { "type": "object", "description": "Serialize Channel model.", @@ -9693,20 +9815,27 @@ "updated_at" ] }, - "RegeneratedApiKeyResponse": { + "RegeneratedSecretResponse": { "type": "object", "properties": { "id": { "type": "string", - "description": "Channel id (also the X-Channel-Id header value)." + "description": "Channel id." }, "api_key": { "type": "string", - "description": "Freshly generated plaintext api_key. Returned ONCE on regeneration and cannot be retrieved later." + "description": "Present for ``api_key`` channels — the plaintext used in subsequent X-API-Key calls. Returned ONCE." + }, + "webhook_secret": { + "type": "string", + "description": "Present for webhook channels with ``auth_method='jwt'`` — the freshly minted root receivers use to verify the HMAC sig and JWT." + }, + "webhook_api_key": { + "type": "string", + "description": "Present for webhook channels with ``auth_method='api_key'`` — the HMAC-derived value sent as X-StMsg-Api-Key. Changes whenever the root rotates." } }, "required": [ - "api_key", "id" ] }, diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index f3eb092e0..e5e1a18b4 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -2253,16 +2253,23 @@ def _validate_webhook_settings(self, attrs): raise serializers.ValidationError( {"settings": "webhook channels require settings.url (a string)."} ) - if not url.startswith(("http://", "https://")): + # Parse first so the scheme check sees urlparse's normalized + # (lowercased) scheme — a raw ``startswith`` would wrongly reject + # e.g. ``HTTPS://``. + parsed_url = urlparse(url) + if parsed_url.scheme not in ("http", "https"): raise serializers.ValidationError( {"settings": "webhook settings.url must be http:// or https://"} ) + host = (parsed_url.hostname or "").lower() + if not host: + raise serializers.ValidationError( + {"settings": "webhook settings.url must include a host."} + ) # Each POST carries the HMAC/JWT signing material in its headers, # so plaintext http would leak credentials in transit. Require # https except for a local-dev escape hatch: a loopback host, or # DEBUG (where operators routinely point at a local receiver). - parsed_url = urlparse(url) - host = (parsed_url.hostname or "").lower() is_loopback = host in ("localhost", "127.0.0.1", "::1") if parsed_url.scheme != "https" and not (settings.DEBUG or is_loopback): raise serializers.ValidationError( diff --git a/src/backend/core/api/viewsets/channel.py b/src/backend/core/api/viewsets/channel.py index 65c6aa906..dd25a5b4c 100644 --- a/src/backend/core/api/viewsets/channel.py +++ b/src/backend/core/api/viewsets/channel.py @@ -256,7 +256,11 @@ def regenerate_secret(self, request, *args, **kwargs): try: plaintext = instance.rotate_secret() except ValueError as exc: - raise ValidationError({"type": str(exc)}) from exc + # Static message — don't reflect the internal exception text + # back to the API caller. + raise ValidationError( + {"type": "This channel type does not support secret rotation."} + ) from exc # api_key channels store only the hash; stash the just-minted # plaintext on the instance so ``_attach_credential`` can find diff --git a/src/backend/core/mda/dispatch_webhooks.py b/src/backend/core/mda/dispatch_webhooks.py index 2ef3a4af0..8ffd18b44 100644 --- a/src/backend/core/mda/dispatch_webhooks.py +++ b/src/backend/core/mda/dispatch_webhooks.py @@ -41,6 +41,7 @@ from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Set, Tuple +from urllib.parse import urlparse from django.db.models import Q @@ -76,7 +77,7 @@ @dataclass -class _HttpResult: +class _HttpResult: # pylint: disable=too-many-instance-attributes """Internal: one webhook call's outcome — decision + the side effects the receiver asked us to apply to the pipeline context. @@ -146,6 +147,26 @@ def _read_capped_body(response) -> bytes: return b"".join(chunks) +def _sanitize_url(url: str) -> str: + """Reduce a webhook URL to ``scheme://host[:port]`` for safe logging. + + Receivers routinely embed a secret token in the path, query string + or userinfo (e.g. ``https://hook.example.com/in/``); logging + the raw URL would leak it. We keep only the scheme, host and port — + enough to identify the receiver without exposing credentials. + """ + try: + parsed = urlparse(url) + except (ValueError, TypeError): + return "" + if not parsed.hostname: + return "" + host = parsed.hostname + if parsed.port: + host = f"{host}:{parsed.port}" + return f"{parsed.scheme}://{host}" + + def _failure(blocking: bool, decision: Decision) -> _HttpResult: """Failure-path result: blocking → propagate ``decision`` (DROP / RETRY); non-blocking → CONTINUE (fire-and-forget, never stalls @@ -478,6 +499,7 @@ def build_jmap_email( "bodyValues", "bodyStructure", "attachments", + "preview", ): email.pop(key, None) @@ -651,7 +673,7 @@ def _post( logger.warning( "Webhook channel %s rejected by SSRF for url=%s: %s", self.channel.id, - url, + _sanitize_url(url), exc, ) return _failure(blocking, Decision.DROP) @@ -662,7 +684,7 @@ def _post( logger.exception( "Webhook channel %s network error for url=%s: %s", self.channel.id, - url, + _sanitize_url(url), exc, ) return _failure(blocking, Decision.RETRY) @@ -681,7 +703,7 @@ def _post( logger.info( "Webhook channel %s requested DROP via response body for url=%s", self.channel.id, - url, + _sanitize_url(url), ) return result @@ -689,7 +711,7 @@ def _post( "Webhook channel %s returned status %s for url=%s", self.channel.id, status, - url, + _sanitize_url(url), ) if status in _RETRY_STATUSES or 500 <= status < 600: return _failure(blocking, Decision.RETRY) diff --git a/src/backend/core/mda/inbound_pipeline.py b/src/backend/core/mda/inbound_pipeline.py index a51216df7..1eecf2168 100644 --- a/src/backend/core/mda/inbound_pipeline.py +++ b/src/backend/core/mda/inbound_pipeline.py @@ -69,7 +69,7 @@ class Decision(IntEnum): @dataclass -class InboundContext: +class InboundContext: # pylint: disable=too-many-instance-attributes """Mutable bag of state flowing through the pipeline. Steps read what they need and write what they decide. The post-loop @@ -153,13 +153,16 @@ def _check_hardcoded_rules( only against headers from trusted relay blocks. Returns ``True`` / ``False`` on first matching rule, ``None`` if no rule matched.""" rules = spam_config.get("rules", []) - for rule in rules: + for idx, rule in enumerate(rules): header_match = rule.get("header_match") or rule.get("header_match_regex") if not header_match: continue if ":" not in header_match: + # Log the rule position, not its raw value: ``spam_config`` + # also carries spam-service credentials, so we never echo + # values read from it into logs. logger.warning( - "Invalid header_match format (missing colon): %s", header_match + "Invalid header_match format (missing colon) in spam rule #%d", idx ) continue key, value = header_match.split(":", 1) @@ -438,9 +441,10 @@ def _resolve_assignable_users( return [] # The input is already lowercased + deduped by the classifier; - # belt-and-suspenders ``set()`` here in case a future caller - # forgets. - target_emails = {e.lower() for e in emails if e} + # belt-and-suspenders dedup here in case a future caller forgets. + # ``dict.fromkeys`` dedups while preserving input order so the + # resolved assignee payload is deterministic. + target_emails = list(dict.fromkeys(e.lower() for e in emails if e)) if not target_emails: return [] diff --git a/src/backend/core/models.py b/src/backend/core/models.py index bb281f63c..9eddda8d8 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -630,9 +630,7 @@ def rotate_secret(self, *, save: bool = True) -> str: "secret": plaintext, } else: - raise ValueError( - f"Channel type {self.type!r} has no rotatable secret" - ) + raise ValueError(f"Channel type {self.type!r} has no rotatable secret") if save: self.save(update_fields=["encrypted_settings", "updated_at"]) @@ -3259,11 +3257,13 @@ def resolve_placeholder_values(mailbox=None, user=None, message=None): Dictionary mapping placeholder keys to their resolved string values """ context = {} + # Prefer the mailbox contact name, but fall back to the user's + # full name when the contact has no (or an empty) name. context["name"] = ( - mailbox.contact.name - if mailbox and mailbox.contact - else (getattr(user, "full_name", None) if user else "") - ) or "" + (mailbox.contact.name if mailbox and mailbox.contact else None) + or (getattr(user, "full_name", None) if user else None) + or "" + ) context["user_name"] = (getattr(user, "full_name", None) or "") if user else "" schema = settings.SCHEMA_CUSTOM_ATTRIBUTES_USER schema_properties = schema.get("properties", {}) diff --git a/src/backend/core/tests/api/test_channels.py b/src/backend/core/tests/api/test_channels.py index a54e06f63..f2fba4388 100644 --- a/src/backend/core/tests/api/test_channels.py +++ b/src/backend/core/tests/api/test_channels.py @@ -588,24 +588,104 @@ def _post(self, api_client, mailbox, settings): @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) def test_create_minimal_webhook(self, api_client, mailbox): + """A JWT webhook surfaces its one-time ``webhook_secret`` on create.""" response = self._post( api_client, mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], "auth_method": "jwt", + "events": ["message.received"], + "auth_method": "jwt", }, ) assert response.status_code == status.HTTP_201_CREATED, response.content + # The signing secret is returned exactly once, at creation time. + assert response.data.get("webhook_secret") + assert "webhook_api_key" not in response.data + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + def test_create_minimal_webhook_api_key(self, api_client, mailbox): + """An api_key webhook surfaces its one-time ``webhook_api_key``.""" + response = self._post( + api_client, + mailbox, + { + "url": "https://hook.example.com/in", + "events": ["message.received"], + "auth_method": "api_key", + }, + ) + assert response.status_code == status.HTTP_201_CREATED, response.content + # api_key channels return the derived key, never the raw JWT secret. + assert response.data.get("webhook_api_key") + assert "webhook_secret" not in response.data + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + def test_regenerate_secret_jwt_webhook(self, api_client, mailbox): + """Rotating a JWT webhook returns a fresh ``webhook_secret``.""" + create = self._post( + api_client, + mailbox, + { + "url": "https://hook.example.com/in", + "events": ["message.received"], + "auth_method": "jwt", + }, + ) + assert create.status_code == status.HTTP_201_CREATED, create.content + channel_id = create.data["id"] + original_secret = create.data["webhook_secret"] + + url = reverse( + "mailbox-channels-regenerate-secret", + kwargs={"mailbox_id": mailbox.id, "pk": channel_id}, + ) + response = api_client.post(url) + assert response.status_code == status.HTTP_200_OK, response.content + assert response.data["id"] == str(channel_id) + new_secret = response.data["webhook_secret"] + assert new_secret + assert new_secret != original_secret + assert "webhook_api_key" not in response.data + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + def test_regenerate_secret_api_key_webhook(self, api_client, mailbox): + """Rotating an api_key webhook returns a fresh ``webhook_api_key``.""" + create = self._post( + api_client, + mailbox, + { + "url": "https://hook.example.com/in", + "events": ["message.received"], + "auth_method": "api_key", + }, + ) + assert create.status_code == status.HTTP_201_CREATED, create.content + channel_id = create.data["id"] + original_key = create.data["webhook_api_key"] + + url = reverse( + "mailbox-channels-regenerate-secret", + kwargs={"mailbox_id": mailbox.id, "pk": channel_id}, + ) + response = api_client.post(url) + assert response.status_code == status.HTTP_200_OK, response.content + assert response.data["id"] == str(channel_id) + new_key = response.data["webhook_api_key"] + assert new_key + assert new_key != original_key + assert "webhook_secret" not in response.data @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) def test_create_with_all_dispatcher_options(self, api_client, mailbox): + """A webhook channel accepts the full set of dispatcher options.""" response = self._post( api_client, mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], "auth_method": "jwt", + "events": ["message.received"], + "auth_method": "jwt", "phase": "before_spam", "blocking": True, "format": "jmap", @@ -619,12 +699,14 @@ def test_create_with_all_dispatcher_options(self, api_client, mailbox): @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) def test_rejects_invalid_format(self, api_client, mailbox): + """An unknown webhook ``format`` is rejected with HTTP 400.""" response = self._post( api_client, mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], "auth_method": "jwt", + "events": ["message.received"], + "auth_method": "jwt", "format": "yaml", }, ) @@ -632,12 +714,14 @@ def test_rejects_invalid_format(self, api_client, mailbox): @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) def test_accepts_jmap_without_body_format(self, api_client, mailbox): + """The ``jmap_without_body`` format is a valid webhook format.""" response = self._post( api_client, mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], "auth_method": "jwt", + "events": ["message.received"], + "auth_method": "jwt", "format": "jmap_without_body", }, ) @@ -645,12 +729,14 @@ def test_accepts_jmap_without_body_format(self, api_client, mailbox): @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) def test_rejects_invalid_phase(self, api_client, mailbox): + """An unknown webhook ``phase`` is rejected with HTTP 400.""" response = self._post( api_client, mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], "auth_method": "jwt", + "events": ["message.received"], + "auth_method": "jwt", "phase": "whenever", }, ) @@ -658,12 +744,14 @@ def test_rejects_invalid_phase(self, api_client, mailbox): @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) def test_rejects_non_bool_blocking(self, api_client, mailbox): + """A non-boolean ``blocking`` value is rejected with HTTP 400.""" response = self._post( api_client, mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], "auth_method": "jwt", + "events": ["message.received"], + "auth_method": "jwt", "blocking": "yes", }, ) @@ -678,7 +766,8 @@ def test_rejects_patch_with_invalid_phase(self, api_client, mailbox): mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], "auth_method": "jwt", + "events": ["message.received"], + "auth_method": "jwt", }, ) assert create.status_code == status.HTTP_201_CREATED, create.content @@ -691,7 +780,8 @@ def test_rejects_patch_with_invalid_phase(self, api_client, mailbox): data={ "settings": { "url": "https://hook.example.com/in", - "events": ["message.received"], "auth_method": "jwt", + "events": ["message.received"], + "auth_method": "jwt", "phase": "bogus", } }, diff --git a/src/backend/core/tests/mda/test_dispatch_webhooks.py b/src/backend/core/tests/mda/test_dispatch_webhooks.py index 6394333b6..c5c308316 100644 --- a/src/backend/core/tests/mda/test_dispatch_webhooks.py +++ b/src/backend/core/tests/mda/test_dispatch_webhooks.py @@ -1,6 +1,7 @@ """Tests for the user-webhook step and the inbound pipeline integration.""" -# pylint: disable=protected-access +# pylint: disable=protected-access,import-outside-toplevel,missing-function-docstring +# pylint: disable=missing-class-docstring,too-many-lines,too-many-public-methods import hashlib import hmac @@ -216,13 +217,13 @@ def test_excludes_other_mailbox(self, mailbox): "events": ["message.received"], }, ) - assert list(find_webhook_channels_for_mailbox(mailbox)) == [] + assert not list(find_webhook_channels_for_mailbox(mailbox)) def test_excludes_other_types(self, mailbox): factories.ChannelFactory( type="widget", mailbox=mailbox, settings={"config": {"enabled": True}} ) - assert list(find_webhook_channels_for_mailbox(mailbox)) == [] + assert not list(find_webhook_channels_for_mailbox(mailbox)) # --- JMAP body builder --- # @@ -261,8 +262,8 @@ def test_minimal_email_shape(self): email = build_jmap_email(parsed) # Strict-JMAP fields pass through unchanged. assert email["messageId"] == ["abc@example.org"] - assert email["inReplyTo"] == [] - assert email["references"] == [] + assert not email["inReplyTo"] + assert not email["references"] assert email["from"] == [{"email": "alice@example.org", "name": "Alice"}] assert email["sentAt"] == "2026-01-01T00:00:00Z" # ``receivedAt`` is stamped at webhook-fire time. @@ -1666,13 +1667,22 @@ def test_response_body_is_capped(self, mock_session, mailbox, parsed_email): "blocking": True, }, ) - # Feed three chunks that together exceed the cap — the reader - # must stop part-way and never request a fourth chunk. + # Expose a stream far larger than the cap and count how much of + # it the reader actually pulls. The reader must stop on its own + # rather than draining the whole stream — if the cap logic ever + # regresses, ``consumed`` blows past the bound and this test fails. oversize_chunk = b"x" * (MAX_RESPONSE_BODY // 2) + consumed = {"bytes": 0} + + def _counting_iter(*_args, **_kwargs): + # 20x the cap worth of chunks; a working reader takes only a + # couple before stopping. + for _ in range(40): + consumed["bytes"] += len(oversize_chunk) + yield oversize_chunk + response = _make_response(200) - response.iter_content = Mock( - return_value=iter([oversize_chunk, oversize_chunk, oversize_chunk]) - ) + response.iter_content = Mock(side_effect=_counting_iter) mock_session.return_value.post.return_value = response outcome = dispatch_webhooks( phase=PHASE_AFTER_SPAM, @@ -1684,6 +1694,9 @@ def test_response_body_is_capped(self, mock_session, mailbox, parsed_email): ) # Body was unparseable (all 'x'), so the result is plain CONTINUE. assert outcome.decision == Decision.CONTINUE + # The reader stopped at the cap: it consumed at most one chunk + # beyond ``MAX_RESPONSE_BODY``, never the whole oversize stream. + assert consumed["bytes"] <= MAX_RESPONSE_BODY + len(oversize_chunk) # And the connection was returned to the pool. response.close.assert_called_once() diff --git a/src/frontend/src/features/api/gen/channels/channels.ts b/src/frontend/src/features/api/gen/channels/channels.ts index 8710b67fc..9c796bff9 100644 --- a/src/frontend/src/features/api/gen/channels/channels.ts +++ b/src/frontend/src/features/api/gen/channels/channels.ts @@ -23,9 +23,10 @@ import type { import type { Channel, + ChannelCreateResponse, ChannelRequest, PatchedChannelRequest, - RegeneratedApiKeyResponse, + RegeneratedSecretResponse, } from ".././models"; import { fetchAPI } from "../../fetch-api"; @@ -221,16 +222,34 @@ export function useMailboxesChannelsList< * Manage integration channels for a mailbox */ export type mailboxesChannelsCreateResponse201 = { - data: Channel; + data: ChannelCreateResponse; status: 201; }; +export type mailboxesChannelsCreateResponse400 = { + data: void; + status: 400; +}; + +export type mailboxesChannelsCreateResponse403 = { + data: void; + status: 403; +}; + export type mailboxesChannelsCreateResponseSuccess = mailboxesChannelsCreateResponse201 & { headers: Headers; }; +export type mailboxesChannelsCreateResponseError = ( + | mailboxesChannelsCreateResponse400 + | mailboxesChannelsCreateResponse403 +) & { + headers: Headers; +}; + export type mailboxesChannelsCreateResponse = - mailboxesChannelsCreateResponseSuccess; + | mailboxesChannelsCreateResponseSuccess + | mailboxesChannelsCreateResponseError; export const getMailboxesChannelsCreateUrl = (mailboxId: string) => { return `/api/v1.0/mailboxes/${mailboxId}/channels/`; @@ -253,7 +272,7 @@ export const mailboxesChannelsCreate = async ( }; export const getMailboxesChannelsCreateMutationOptions = < - TError = ErrorType, + TError = ErrorType, TContext = unknown, >(options?: { mutation?: UseMutationOptions< @@ -294,10 +313,10 @@ export type MailboxesChannelsCreateMutationResult = NonNullable< Awaited> >; export type MailboxesChannelsCreateMutationBody = ChannelRequest; -export type MailboxesChannelsCreateMutationError = ErrorType; +export type MailboxesChannelsCreateMutationError = ErrorType; export const useMailboxesChannelsCreate = < - TError = ErrorType, + TError = ErrorType, TContext = unknown, >( options?: { @@ -885,56 +904,56 @@ export const useMailboxesChannelsDestroy = < /** * Manage integration channels for a mailbox */ -export type mailboxesChannelsRegenerateApiKeyCreateResponse200 = { - data: RegeneratedApiKeyResponse; +export type mailboxesChannelsRegenerateSecretCreateResponse200 = { + data: RegeneratedSecretResponse; status: 200; }; -export type mailboxesChannelsRegenerateApiKeyCreateResponse400 = { +export type mailboxesChannelsRegenerateSecretCreateResponse400 = { data: void; status: 400; }; -export type mailboxesChannelsRegenerateApiKeyCreateResponse403 = { +export type mailboxesChannelsRegenerateSecretCreateResponse403 = { data: void; status: 403; }; -export type mailboxesChannelsRegenerateApiKeyCreateResponse404 = { +export type mailboxesChannelsRegenerateSecretCreateResponse404 = { data: void; status: 404; }; -export type mailboxesChannelsRegenerateApiKeyCreateResponseSuccess = - mailboxesChannelsRegenerateApiKeyCreateResponse200 & { +export type mailboxesChannelsRegenerateSecretCreateResponseSuccess = + mailboxesChannelsRegenerateSecretCreateResponse200 & { headers: Headers; }; -export type mailboxesChannelsRegenerateApiKeyCreateResponseError = ( - | mailboxesChannelsRegenerateApiKeyCreateResponse400 - | mailboxesChannelsRegenerateApiKeyCreateResponse403 - | mailboxesChannelsRegenerateApiKeyCreateResponse404 +export type mailboxesChannelsRegenerateSecretCreateResponseError = ( + | mailboxesChannelsRegenerateSecretCreateResponse400 + | mailboxesChannelsRegenerateSecretCreateResponse403 + | mailboxesChannelsRegenerateSecretCreateResponse404 ) & { headers: Headers; }; -export type mailboxesChannelsRegenerateApiKeyCreateResponse = - | mailboxesChannelsRegenerateApiKeyCreateResponseSuccess - | mailboxesChannelsRegenerateApiKeyCreateResponseError; +export type mailboxesChannelsRegenerateSecretCreateResponse = + | mailboxesChannelsRegenerateSecretCreateResponseSuccess + | mailboxesChannelsRegenerateSecretCreateResponseError; -export const getMailboxesChannelsRegenerateApiKeyCreateUrl = ( +export const getMailboxesChannelsRegenerateSecretCreateUrl = ( mailboxId: string, id: string, ) => { - return `/api/v1.0/mailboxes/${mailboxId}/channels/${id}/regenerate-api-key/`; + return `/api/v1.0/mailboxes/${mailboxId}/channels/${id}/regenerate-secret/`; }; -export const mailboxesChannelsRegenerateApiKeyCreate = async ( +export const mailboxesChannelsRegenerateSecretCreate = async ( mailboxId: string, id: string, options?: RequestInit, -): Promise => { - return fetchAPI( - getMailboxesChannelsRegenerateApiKeyCreateUrl(mailboxId, id), +): Promise => { + return fetchAPI( + getMailboxesChannelsRegenerateSecretCreateUrl(mailboxId, id), { ...options, method: "POST", @@ -942,24 +961,24 @@ export const mailboxesChannelsRegenerateApiKeyCreate = async ( ); }; -export const getMailboxesChannelsRegenerateApiKeyCreateMutationOptions = < +export const getMailboxesChannelsRegenerateSecretCreateMutationOptions = < TError = ErrorType, TContext = unknown, >(options?: { mutation?: UseMutationOptions< - Awaited>, + Awaited>, TError, { mailboxId: string; id: string }, TContext >; request?: SecondParameter; }): UseMutationOptions< - Awaited>, + Awaited>, TError, { mailboxId: string; id: string }, TContext > => { - const mutationKey = ["mailboxesChannelsRegenerateApiKeyCreate"]; + const mutationKey = ["mailboxesChannelsRegenerateSecretCreate"]; const { mutation: mutationOptions, request: requestOptions } = options ? options.mutation && "mutationKey" in options.mutation && @@ -969,12 +988,12 @@ export const getMailboxesChannelsRegenerateApiKeyCreateMutationOptions = < : { mutation: { mutationKey }, request: undefined }; const mutationFn: MutationFunction< - Awaited>, + Awaited>, { mailboxId: string; id: string } > = (props) => { const { mailboxId, id } = props ?? {}; - return mailboxesChannelsRegenerateApiKeyCreate( + return mailboxesChannelsRegenerateSecretCreate( mailboxId, id, requestOptions, @@ -984,20 +1003,20 @@ export const getMailboxesChannelsRegenerateApiKeyCreateMutationOptions = < return { mutationFn, ...mutationOptions }; }; -export type MailboxesChannelsRegenerateApiKeyCreateMutationResult = NonNullable< - Awaited> +export type MailboxesChannelsRegenerateSecretCreateMutationResult = NonNullable< + Awaited> >; -export type MailboxesChannelsRegenerateApiKeyCreateMutationError = +export type MailboxesChannelsRegenerateSecretCreateMutationError = ErrorType; -export const useMailboxesChannelsRegenerateApiKeyCreate = < +export const useMailboxesChannelsRegenerateSecretCreate = < TError = ErrorType, TContext = unknown, >( options?: { mutation?: UseMutationOptions< - Awaited>, + Awaited>, TError, { mailboxId: string; id: string }, TContext @@ -1006,13 +1025,13 @@ export const useMailboxesChannelsRegenerateApiKeyCreate = < }, queryClient?: QueryClient, ): UseMutationResult< - Awaited>, + Awaited>, TError, { mailboxId: string; id: string }, TContext > => { const mutationOptions = - getMailboxesChannelsRegenerateApiKeyCreateMutationOptions(options); + getMailboxesChannelsRegenerateSecretCreateMutationOptions(options); return useMutation(mutationOptions, queryClient); }; @@ -1185,16 +1204,34 @@ export function useUsersMeChannelsList< * Manage personal (scope_level=user) integration channels */ export type usersMeChannelsCreateResponse201 = { - data: Channel; + data: ChannelCreateResponse; status: 201; }; +export type usersMeChannelsCreateResponse400 = { + data: void; + status: 400; +}; + +export type usersMeChannelsCreateResponse403 = { + data: void; + status: 403; +}; + export type usersMeChannelsCreateResponseSuccess = usersMeChannelsCreateResponse201 & { headers: Headers; }; +export type usersMeChannelsCreateResponseError = ( + | usersMeChannelsCreateResponse400 + | usersMeChannelsCreateResponse403 +) & { + headers: Headers; +}; + export type usersMeChannelsCreateResponse = - usersMeChannelsCreateResponseSuccess; + | usersMeChannelsCreateResponseSuccess + | usersMeChannelsCreateResponseError; export const getUsersMeChannelsCreateUrl = () => { return `/api/v1.0/users/me/channels/`; @@ -1216,7 +1253,7 @@ export const usersMeChannelsCreate = async ( }; export const getUsersMeChannelsCreateMutationOptions = < - TError = ErrorType, + TError = ErrorType, TContext = unknown, >(options?: { mutation?: UseMutationOptions< @@ -1257,10 +1294,10 @@ export type UsersMeChannelsCreateMutationResult = NonNullable< Awaited> >; export type UsersMeChannelsCreateMutationBody = ChannelRequest; -export type UsersMeChannelsCreateMutationError = ErrorType; +export type UsersMeChannelsCreateMutationError = ErrorType; export const useUsersMeChannelsCreate = < - TError = ErrorType, + TError = ErrorType, TContext = unknown, >( options?: { @@ -1819,52 +1856,52 @@ export const useUsersMeChannelsDestroy = < /** * Manage personal (scope_level=user) integration channels */ -export type usersMeChannelsRegenerateApiKeyCreateResponse200 = { - data: RegeneratedApiKeyResponse; +export type usersMeChannelsRegenerateSecretCreateResponse200 = { + data: RegeneratedSecretResponse; status: 200; }; -export type usersMeChannelsRegenerateApiKeyCreateResponse400 = { +export type usersMeChannelsRegenerateSecretCreateResponse400 = { data: void; status: 400; }; -export type usersMeChannelsRegenerateApiKeyCreateResponse403 = { +export type usersMeChannelsRegenerateSecretCreateResponse403 = { data: void; status: 403; }; -export type usersMeChannelsRegenerateApiKeyCreateResponse404 = { +export type usersMeChannelsRegenerateSecretCreateResponse404 = { data: void; status: 404; }; -export type usersMeChannelsRegenerateApiKeyCreateResponseSuccess = - usersMeChannelsRegenerateApiKeyCreateResponse200 & { +export type usersMeChannelsRegenerateSecretCreateResponseSuccess = + usersMeChannelsRegenerateSecretCreateResponse200 & { headers: Headers; }; -export type usersMeChannelsRegenerateApiKeyCreateResponseError = ( - | usersMeChannelsRegenerateApiKeyCreateResponse400 - | usersMeChannelsRegenerateApiKeyCreateResponse403 - | usersMeChannelsRegenerateApiKeyCreateResponse404 +export type usersMeChannelsRegenerateSecretCreateResponseError = ( + | usersMeChannelsRegenerateSecretCreateResponse400 + | usersMeChannelsRegenerateSecretCreateResponse403 + | usersMeChannelsRegenerateSecretCreateResponse404 ) & { headers: Headers; }; -export type usersMeChannelsRegenerateApiKeyCreateResponse = - | usersMeChannelsRegenerateApiKeyCreateResponseSuccess - | usersMeChannelsRegenerateApiKeyCreateResponseError; +export type usersMeChannelsRegenerateSecretCreateResponse = + | usersMeChannelsRegenerateSecretCreateResponseSuccess + | usersMeChannelsRegenerateSecretCreateResponseError; -export const getUsersMeChannelsRegenerateApiKeyCreateUrl = (id: string) => { - return `/api/v1.0/users/me/channels/${id}/regenerate-api-key/`; +export const getUsersMeChannelsRegenerateSecretCreateUrl = (id: string) => { + return `/api/v1.0/users/me/channels/${id}/regenerate-secret/`; }; -export const usersMeChannelsRegenerateApiKeyCreate = async ( +export const usersMeChannelsRegenerateSecretCreate = async ( id: string, options?: RequestInit, -): Promise => { - return fetchAPI( - getUsersMeChannelsRegenerateApiKeyCreateUrl(id), +): Promise => { + return fetchAPI( + getUsersMeChannelsRegenerateSecretCreateUrl(id), { ...options, method: "POST", @@ -1872,24 +1909,24 @@ export const usersMeChannelsRegenerateApiKeyCreate = async ( ); }; -export const getUsersMeChannelsRegenerateApiKeyCreateMutationOptions = < +export const getUsersMeChannelsRegenerateSecretCreateMutationOptions = < TError = ErrorType, TContext = unknown, >(options?: { mutation?: UseMutationOptions< - Awaited>, + Awaited>, TError, { id: string }, TContext >; request?: SecondParameter; }): UseMutationOptions< - Awaited>, + Awaited>, TError, { id: string }, TContext > => { - const mutationKey = ["usersMeChannelsRegenerateApiKeyCreate"]; + const mutationKey = ["usersMeChannelsRegenerateSecretCreate"]; const { mutation: mutationOptions, request: requestOptions } = options ? options.mutation && "mutationKey" in options.mutation && @@ -1899,31 +1936,31 @@ export const getUsersMeChannelsRegenerateApiKeyCreateMutationOptions = < : { mutation: { mutationKey }, request: undefined }; const mutationFn: MutationFunction< - Awaited>, + Awaited>, { id: string } > = (props) => { const { id } = props ?? {}; - return usersMeChannelsRegenerateApiKeyCreate(id, requestOptions); + return usersMeChannelsRegenerateSecretCreate(id, requestOptions); }; return { mutationFn, ...mutationOptions }; }; -export type UsersMeChannelsRegenerateApiKeyCreateMutationResult = NonNullable< - Awaited> +export type UsersMeChannelsRegenerateSecretCreateMutationResult = NonNullable< + Awaited> >; -export type UsersMeChannelsRegenerateApiKeyCreateMutationError = +export type UsersMeChannelsRegenerateSecretCreateMutationError = ErrorType; -export const useUsersMeChannelsRegenerateApiKeyCreate = < +export const useUsersMeChannelsRegenerateSecretCreate = < TError = ErrorType, TContext = unknown, >( options?: { mutation?: UseMutationOptions< - Awaited>, + Awaited>, TError, { id: string }, TContext @@ -1932,13 +1969,13 @@ export const useUsersMeChannelsRegenerateApiKeyCreate = < }, queryClient?: QueryClient, ): UseMutationResult< - Awaited>, + Awaited>, TError, { id: string }, TContext > => { const mutationOptions = - getUsersMeChannelsRegenerateApiKeyCreateMutationOptions(options); + getUsersMeChannelsRegenerateSecretCreateMutationOptions(options); return useMutation(mutationOptions, queryClient); }; diff --git a/src/frontend/src/features/api/gen/models/channel_create_response.ts b/src/frontend/src/features/api/gen/models/channel_create_response.ts new file mode 100644 index 000000000..782c16bdb --- /dev/null +++ b/src/frontend/src/features/api/gen/models/channel_create_response.ts @@ -0,0 +1,65 @@ +/** + * Generated by orval 🍺 + * Do not edit manually. + * messages API + * This is the messages API schema. + * OpenAPI spec version: 1.0.0 (v1.0) + */ +import type { ScopeLevelEnum } from "./scope_level_enum"; + +/** + * Schema-only view of the channel-create 201 response. + +``ChannelViewSet.create`` returns the full ``ChannelSerializer`` +payload plus the freshly-minted plaintext credentials — surfaced +exactly once on creation and never retrievable again. They are +declared here as read-only fields so generated API clients see them +in the OpenAPI schema. This serializer is never used to serialize a +response directly; the view assembles the body by hand. + */ +export interface ChannelCreateResponse { + /** primary key for the record as UUID */ + readonly id: string; + /** + * Human-readable name for this channel + * @maxLength 255 + */ + name: string; + /** + * Type of channel + * @maxLength 255 + */ + type: string; + readonly scope_level: ScopeLevelEnum; + /** Channel-specific configuration settings */ + settings?: unknown; + /** + * primary key for the record as UUID + * @nullable + */ + readonly mailbox: string | null; + /** + * primary key for the record as UUID + * @nullable + */ + readonly maildomain: string | null; + /** + * primary key for the record as UUID + * @nullable + */ + readonly user: string | null; + /** @nullable */ + readonly last_used_at: string | null; + /** date and time at which a record was created */ + readonly created_at: string; + /** date and time at which a record was last updated */ + readonly updated_at: string; + /** api_key channels only — the plaintext API key. */ + readonly api_key: string; + /** Plaintext password, when the channel type mints one. */ + readonly password: string; + /** webhook channels with auth_method=jwt — the HMAC/JWT signing secret. */ + readonly webhook_secret: string; + /** webhook channels with auth_method=api_key — the derived API key. */ + readonly webhook_api_key: string; +} diff --git a/src/frontend/src/features/api/gen/models/index.ts b/src/frontend/src/features/api/gen/models/index.ts index 2b654b0e6..ee07f95c1 100644 --- a/src/frontend/src/features/api/gen/models/index.ts +++ b/src/frontend/src/features/api/gen/models/index.ts @@ -21,6 +21,7 @@ export * from "./calendar_rsvp_request_request"; export * from "./calendar_rsvp_response"; export * from "./change_flag_request_request"; export * from "./channel"; +export * from "./channel_create_response"; export * from "./channel_request"; export * from "./config_retrieve200"; export * from "./config_retrieve200_driv_e"; @@ -147,7 +148,7 @@ export * from "./placeholders_retrieve200"; export * from "./placeholders_retrieve200_i18n"; export * from "./read_message_template"; export * from "./read_message_template_metadata"; -export * from "./regenerated_api_key_response"; +export * from "./regenerated_secret_response"; export * from "./reset_password_error"; export * from "./reset_password_internal_server_error"; export * from "./reset_password_not_found"; diff --git a/src/frontend/src/features/api/gen/models/regenerated_api_key_response.ts b/src/frontend/src/features/api/gen/models/regenerated_api_key_response.ts deleted file mode 100644 index a6ff13cd8..000000000 --- a/src/frontend/src/features/api/gen/models/regenerated_api_key_response.ts +++ /dev/null @@ -1,14 +0,0 @@ -/** - * Generated by orval 🍺 - * Do not edit manually. - * messages API - * This is the messages API schema. - * OpenAPI spec version: 1.0.0 (v1.0) - */ - -export interface RegeneratedApiKeyResponse { - /** Channel id (also the X-Channel-Id header value). */ - id: string; - /** Freshly generated plaintext api_key. Returned ONCE on regeneration and cannot be retrieved later. */ - api_key: string; -} diff --git a/src/frontend/src/features/api/gen/models/regenerated_secret_response.ts b/src/frontend/src/features/api/gen/models/regenerated_secret_response.ts new file mode 100644 index 000000000..6227a6703 --- /dev/null +++ b/src/frontend/src/features/api/gen/models/regenerated_secret_response.ts @@ -0,0 +1,18 @@ +/** + * Generated by orval 🍺 + * Do not edit manually. + * messages API + * This is the messages API schema. + * OpenAPI spec version: 1.0.0 (v1.0) + */ + +export interface RegeneratedSecretResponse { + /** Channel id. */ + id: string; + /** Present for ``api_key`` channels — the plaintext used in subsequent X-API-Key calls. Returned ONCE. */ + api_key?: string; + /** Present for webhook channels with ``auth_method='jwt'`` — the freshly minted root receivers use to verify the HMAC sig and JWT. */ + webhook_secret?: string; + /** Present for webhook channels with ``auth_method='api_key'`` — the HMAC-derived value sent as X-StMsg-Api-Key. Changes whenever the root rotates. */ + webhook_api_key?: string; +} From 12ac6a1109484adc74920cc2d18afbfcf59d9973 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Wed, 17 Jun 2026 15:37:51 +0200 Subject: [PATCH 03/21] review fixes --- src/backend/core/api/openapi.json | 10 +--------- src/backend/core/api/serializers.py | 8 ++++---- src/backend/core/tests/mda/test_dispatch_webhooks.py | 7 +++++-- .../features/api/gen/models/channel_create_response.ts | 8 ++++---- .../webhook-integration-form.tsx | 2 +- 5 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/backend/core/api/openapi.json b/src/backend/core/api/openapi.json index b2cc9b0ec..ef445d433 100644 --- a/src/backend/core/api/openapi.json +++ b/src/backend/core/api/openapi.json @@ -7580,40 +7580,32 @@ }, "api_key": { "type": "string", - "readOnly": true, "description": "api_key channels only — the plaintext API key." }, "password": { "type": "string", - "readOnly": true, "description": "Plaintext password, when the channel type mints one." }, "webhook_secret": { "type": "string", - "readOnly": true, "description": "webhook channels with auth_method=jwt — the HMAC/JWT signing secret." }, "webhook_api_key": { "type": "string", - "readOnly": true, "description": "webhook channels with auth_method=api_key — the derived API key." } }, "required": [ - "api_key", "created_at", "id", "last_used_at", "mailbox", "maildomain", "name", - "password", "scope_level", "type", "updated_at", - "user", - "webhook_api_key", - "webhook_secret" + "user" ] }, "ChannelRequest": { diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index e5e1a18b4..7ce3eff70 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -2405,23 +2405,23 @@ class ChannelCreateResponseSerializer(ChannelSerializer): response directly; the view assembles the body by hand. """ + # ``required=False`` without ``read_only`` so drf-spectacular lists + # these as optional (not ``required``) in the response schema — each + # create returns only the one credential matching the channel type / + # auth_method, so the others are intentionally absent. api_key = serializers.CharField( - read_only=True, required=False, help_text="api_key channels only — the plaintext API key.", ) password = serializers.CharField( - read_only=True, required=False, help_text="Plaintext password, when the channel type mints one.", ) webhook_secret = serializers.CharField( - read_only=True, required=False, help_text="webhook channels with auth_method=jwt — the HMAC/JWT signing secret.", ) webhook_api_key = serializers.CharField( - read_only=True, required=False, help_text="webhook channels with auth_method=api_key — the derived API key.", ) diff --git a/src/backend/core/tests/mda/test_dispatch_webhooks.py b/src/backend/core/tests/mda/test_dispatch_webhooks.py index c5c308316..2da86bff3 100644 --- a/src/backend/core/tests/mda/test_dispatch_webhooks.py +++ b/src/backend/core/tests/mda/test_dispatch_webhooks.py @@ -2,6 +2,7 @@ # pylint: disable=protected-access,import-outside-toplevel,missing-function-docstring # pylint: disable=missing-class-docstring,too-many-lines,too-many-public-methods +# pylint: disable=use-implicit-booleaness-not-comparison import hashlib import hmac @@ -262,8 +263,10 @@ def test_minimal_email_shape(self): email = build_jmap_email(parsed) # Strict-JMAP fields pass through unchanged. assert email["messageId"] == ["abc@example.org"] - assert not email["inReplyTo"] - assert not email["references"] + # Strict JMAP Id[] contract: these must be empty lists, not just + # any falsey value. + assert email["inReplyTo"] == [] + assert email["references"] == [] assert email["from"] == [{"email": "alice@example.org", "name": "Alice"}] assert email["sentAt"] == "2026-01-01T00:00:00Z" # ``receivedAt`` is stamped at webhook-fire time. diff --git a/src/frontend/src/features/api/gen/models/channel_create_response.ts b/src/frontend/src/features/api/gen/models/channel_create_response.ts index 782c16bdb..b8abb14ff 100644 --- a/src/frontend/src/features/api/gen/models/channel_create_response.ts +++ b/src/frontend/src/features/api/gen/models/channel_create_response.ts @@ -55,11 +55,11 @@ export interface ChannelCreateResponse { /** date and time at which a record was last updated */ readonly updated_at: string; /** api_key channels only — the plaintext API key. */ - readonly api_key: string; + api_key?: string; /** Plaintext password, when the channel type mints one. */ - readonly password: string; + password?: string; /** webhook channels with auth_method=jwt — the HMAC/JWT signing secret. */ - readonly webhook_secret: string; + webhook_secret?: string; /** webhook channels with auth_method=api_key — the derived API key. */ - readonly webhook_api_key: string; + webhook_api_key?: string; } diff --git a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx index e86567d2d..594a62965 100644 --- a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx +++ b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx @@ -147,7 +147,7 @@ export const WebhookIntegrationForm = ({ , ); await invalidateChannels(); - if (newChannel.status < 300) { + if (newChannel.status === 201) { // Surface the freshly minted credential exactly // once — the receiver needs this value to verify // every webhook we send. The backend returns From 8f4d9d1067288a7dc8a0fb58a3595c32f07a2fac Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Fri, 19 Jun 2026 02:02:21 +0200 Subject: [PATCH 04/21] review --- docs/webhooks.md | 96 +++++--- src/backend/core/api/serializers.py | 36 ++- src/backend/core/api/viewsets/channel.py | 49 ++-- src/backend/core/enums.py | 4 +- src/backend/core/mda/dispatch_webhooks.py | 93 ++++---- src/backend/core/mda/inbound_pipeline.py | 30 ++- src/backend/core/mda/inbound_tasks.py | 76 +++++-- .../tests/api/test_channel_scope_level.py | 2 +- src/backend/core/tests/api/test_channels.py | 56 ++--- .../core/tests/mda/test_dispatch_webhooks.py | 214 ++++++++++-------- .../core/tests/mda/test_spam_processing.py | 61 ++++- src/backend/messages/settings.py | 9 + src/frontend/public/locales/common/en-US.json | 1 + src/frontend/public/locales/common/fr-FR.json | 1 + src/frontend/public/locales/common/nl-NL.json | 1 + src/frontend/public/locales/common/ru-RU.json | 1 + src/frontend/public/locales/common/uk-UA.json | 1 + .../api/gen/models/channel_create_response.ts | 6 +- .../gen/models/regenerated_secret_response.ts | 6 +- .../webhook-integration-form.tsx | 25 +- .../thread-message/thread-message-header.tsx | 12 + 21 files changed, 479 insertions(+), 301 deletions(-) diff --git a/docs/webhooks.md b/docs/webhooks.md index 96e688be2..6eefa66ea 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -52,7 +52,7 @@ A webhook channel stores its configuration in `Channel.settings` ```json { "url": "https://example.com/inbox-hook", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "after_spam", "format": "eml", "blocking": false, @@ -63,9 +63,9 @@ A webhook channel stores its configuration in `Channel.settings` | Key | Type | Default | Description | | ------------- | -------- | -------------- | --------------------------------------------------------------------------- | | `url` | string | **required** | `http://` or `https://` endpoint. Validated by the SSRF guard at each call. | -| `events` | string[] | **required** | Currently only `message.received` is implemented. | +| `events` | string[] | **required** | Currently only `message.inbound` is implemented. | | `phase` | string | `after_spam` | `before_spam` or `after_spam`. | -| `format` | string | `eml` | `eml`, `jmap`, or `jmap_without_body` (see [Payload formats](#payload-formats)). | +| `format` | string | `eml` | `eml`, `jmap`, or `jmap_metadata` (see [Payload formats](#payload-formats)). | | `blocking` | bool | `false` | If true, the webhook response determines delivery (see [Response contract](#response-contract) below). | | `auth_method` | string | **required** | `jwt` or `api_key` (see [Authentication](#authentication)). | @@ -120,8 +120,8 @@ PATCH the channel's `settings.auth_method`. The root secret is **not** rotated — only the wire presentation changes — but the receiver was given the old method's credential at creation. To get the new method's credential, call `POST /channels/{id}/regenerate-secret/`: the -response returns either `webhook_secret` (jwt) or `webhook_api_key` -(api_key), matching the channel's current method. Rotation invalidates +response returns either `secret` (jwt) or `api_key` (api_key), +matching the channel's current method. Rotation invalidates the previous credential, so update the receiver before the next inbound message lands. @@ -130,33 +130,72 @@ message lands. | Header | Value | | --------------------- | ---------------------------------------------------------------- | | `Content-Type` | `message/rfc822` for `eml`, `application/json` for both JMAP variants | -| `X-StMsg-Event` | `message.received` | +| `X-StMsg-Event` | `message.inbound` | | `X-StMsg-Phase` | `before_spam` or `after_spam` | | `X-StMsg-Channel-Id` | UUID of the firing webhook Channel | | `X-StMsg-Mailbox` | Destination mailbox address | | `X-StMsg-Recipient` | Envelope `RCPT TO` (usually the same as `X-StMsg-Mailbox`) | | `X-StMsg-Is-Spam` | `true`, `false`, or `unknown` (`unknown` in the `before_spam` phase) | -| `X-StMsg-Message-Id` | Original `Message-ID` header value (angle-bracketed), if any | + +The message-id is **not** sent as a header — every body format already +carries it (`messageId` in the JMAP variants, the raw `Message-ID:` +header in `eml`). ### Response contract The classification below applies to **blocking** webhooks. Non-blocking webhooks treat every outcome as success — their bodies are ignored. +The three possible **decisions** are about the inbound email itself: + +* **CONTINUE** — deliver it: the `Message` and its thread are created + normally. +* **DROP** — discard it: **no `Message` and no thread are created**, so + the recipient never sees the email. The original sender is *not* + notified (the inbound SMTP transaction was already accepted). The + short-lived internal `InboundMessage` processing row is also removed — + but that happens on *every* terminal outcome, including normal + delivery, so "the `InboundMessage` is deleted" is not what makes DROP + special; **the email never landing** is. +* **RETRY** — keep the email queued and re-fire the webhook on the next + 5-minute sweep (bounded by the **quarantine window** below). + +**A webhook error never drops the email.** The *only* way a blocking +webhook discards a message is by **explicitly** returning +`{"action": "drop"}` with an HTTP `2xx` (see +[JSON action body](#json-action-body)). Every transport- or status-level +failure is held for RETRY — a receiver bug that answers `4xx`, an +endpoint that is down, or a config error on our side must never cost the +user their mail. + | Outcome | Decision | What happens | | ------------------------------------- | ---------- | ------------------------------------------------------------------------------- | -| HTTP `2xx`, empty / non-JSON body | CONTINUE | Delivery proceeds normally. | -| HTTP `2xx` + JSON action body | see below | Body parsed for `action` / `is_spam` / `labels`. | -| HTTP `4xx` | DROP | Receiver definitively rejected the message; `InboundMessage` deleted. | -| HTTP `408`, `429`, `5xx` | RETRY | Transient — `InboundMessage` kept; the 5-min sweep re-fires the webhook. | +| HTTP `2xx`, empty / non-JSON body | CONTINUE | Email delivered normally. | +| HTTP `2xx` + `{"action": "drop"}` | DROP | The **only** path to DROP — the receiver deliberately discards the email. | +| HTTP `2xx` + other JSON action body | see below | Body parsed for `action` / `is_spam` / `add_labels`; default is CONTINUE. | +| Any non-2xx (`4xx`, `5xx`, `3xx`) | RETRY | The email stays queued; the 5-min sweep re-fires the webhook. | | Connection error, timeout, DNS, etc. | RETRY | Transient. | -| SSRF rejection | DROP | Config error on our side — retrying won't help. | -| Missing signing secret (misconfig) | DROP | The dispatcher fails closed rather than POST an unsigned request. | - -`RETRY` is bounded: an `InboundMessage` held in retry for more than -**7 days** is dropped with a loud `ERROR` log. This prevents a -permanently-broken receiver from pinning a row forever. (When you fix -the receiver within 7 days, the next sweep delivers normally.) +| SSRF rejection | RETRY | Config error on our side (bad URL) — held until fixed, never dropped. | +| Missing signing secret (misconfig) | RETRY | Can't sign the POST — held until the channel is fixed, never dropped. | + +`RETRY` is bounded by a **quarantine window** so a persistently-failing +processing step can neither pin a row forever nor lose mail. If a step +is still failing **48 hours** after the message arrived, we stop holding +and **deliver the message anyway** — landed in the inbox (`is_spam=False` +so it isn't buried) and stamped with an `X-StMsg-Processing-Failed` +marker. The web UI reads that marker and shows a prominent warning banner +(the same surface as the unverified-sender warning), so the recipient +knows the message bypassed a processing step and can review it with +caution. Nothing is ever silently dropped; if the step recovers within +the window, the next sweep delivers normally with no marker. + +The mechanism is generic: a blocking webhook is the trigger today, but +any step that returns `RETRY` (e.g. a persistently-unreachable spam +checker) is quarantined the same way. + +(The marker rides in the stored MIME as an `X-StMsg-*` header. Sender- +supplied `X-StMsg-*` headers are stripped at ingest, so receivers can't +forge it.) #### JSON action body @@ -168,7 +207,7 @@ optional; unknown keys are ignored. { "action": "drop", "is_spam": true, - "labels": ["b3c9c1c3-1f4a-4d4a-9b2d-9c5a2a7c0a01"], + "add_labels": ["b3c9c1c3-1f4a-4d4a-9b2d-9c5a2a7c0a01"], "assign_to": ["alice@example.org"], "mark_starred": true, "mark_read": true, @@ -184,9 +223,9 @@ optional; unknown keys are ignored. | Key | Type | Meaning | | ---------------- | -------------- | ---------------------------------------------------------------------------------------------------------------- | -| `action` | `"drop"` / `"retry"` | `"drop"` drops the message at this phase; `"retry"` re-queues the inbound task (bounded by the 7-day retry budget). Any other value (or omission) is treated as accept. Case-insensitive. | +| `action` | `"drop"` / `"retry"` | `"drop"` drops the message at this phase; `"retry"` re-queues the inbound task (bounded by the 48h quarantine window). Any other value (or omission) is treated as accept. Case-insensitive. | | `is_spam` | bool | Override the spam verdict. Acts as a full antispam: in the `before_spam` phase this **skips rspamd**. | -| `labels` | string[] | UUIDs of `Label` rows in the destination mailbox to attach to the thread once it is created. | +| `add_labels` | string[] | UUIDs of `Label` rows in the destination mailbox to attach to the thread once it is created. | | `assign_to` | string[] | OIDC emails of users to assign to the resulting thread (one `ThreadEvent ASSIGN` per webhook, channel-attributed). | | `mark_starred` | bool (true only) | Star the resulting thread for the destination mailbox. | | `mark_read` | bool (true only) | Mark the resulting thread as read for the destination mailbox. | @@ -199,13 +238,13 @@ optional; unknown keys are ignored. Notes: * `action: "drop"` always wins. Setting `action: "drop"` together with - `labels` or `assign_to` still drops — the thread is never created, + `add_labels` or `assign_to` still drops — the thread is never created, so neither side effect is applied. * `is_spam` discriminates between **explicit false (ham)** and **no opinion**: returning `{}` leaves the dispatcher's verdict (typically rspamd) untouched, while returning `{"is_spam": false}` forces ham. -* `labels` only makes sense for **mailbox-scoped** channels: labels are - per-mailbox. For domain- or global-scoped channels the UUIDs are +* `add_labels` only makes sense for **mailbox-scoped** channels: labels + are per-mailbox. For domain- or global-scoped channels the UUIDs are validated against the receiving mailbox; unknown UUIDs are logged and skipped, not raised — a misbehaving webhook must not stall delivery. * `assign_to` resolves each email to a User row with @@ -285,7 +324,7 @@ merge deterministically: * **decision**: most severe wins (`DROP` > `RETRY` > `CONTINUE`). The dispatcher short-circuits the fan-out as soon as any webhook drops. * **is_spam**: last decisive value wins (DB iteration order). -* **labels**: set union across all webhooks. +* **add_labels**: set union across all webhooks. * **assign_to**: each webhook's list lands as its own ThreadEvent (channel attribution preserved). A user assigned by an earlier webhook is absorbed by the partial UniqueConstraint when a later @@ -312,13 +351,12 @@ MTA received them. ```http POST /inbox-hook HTTP/1.1 Content-Type: message/rfc822 -X-StMsg-Event: message.received +X-StMsg-Event: message.inbound X-StMsg-Phase: after_spam X-StMsg-Channel-Id: 05f1f991-c2e9-4fa7-8a78-98c3aa904c7c X-StMsg-Mailbox: alice@example.com X-StMsg-Recipient: alice@example.com X-StMsg-Is-Spam: false -X-StMsg-Message-Id: From: Bob To: alice@example.com @@ -393,7 +431,7 @@ and `cid`. If you need the raw bytes pick `format: "eml"` instead. with an explicit `Z` suffix, e.g. `2026-01-01T12:00:00Z` (not `+00:00`). This matches RFC 8621 §1.4. -### `jmap_without_body` +### `jmap_metadata` Same JMAP `Email` shape as `jmap`, but the body content and attachments are dropped: @@ -428,7 +466,7 @@ def inbox_hook(): elif content_type.startswith("application/json"): body = request.get_json() print("JMAP subject:", body["subject"]) - # Body content may not be there in jmap_without_body mode. + # Body content may not be there in jmap_metadata mode. body_values = body.get("bodyValues") or {} for part_id, value in body_values.items(): print(f" part {part_id}: {value['value'][:80]}") diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index 7ce3eff70..17534dce7 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -2043,8 +2043,8 @@ class Meta: # Channel types that mint a plaintext secret on create. The # viewset surfaces the minted value via the right response key - # (api_key for api_key channels, webhook_secret / webhook_api_key - # for webhook channels per ``settings.auth_method``). + # (``api_key`` for api_key channels and api_key webhooks, ``secret`` + # for jwt webhooks — keyed by ``settings.auth_method``). _ROTATABLE_TYPES = frozenset( {enums.ChannelTypes.API_KEY, enums.ChannelTypes.WEBHOOK} ) @@ -2268,17 +2268,15 @@ def _validate_webhook_settings(self, attrs): ) # Each POST carries the HMAC/JWT signing material in its headers, # so plaintext http would leak credentials in transit. Require - # https except for a local-dev escape hatch: a loopback host, or - # DEBUG (where operators routinely point at a local receiver). - is_loopback = host in ("localhost", "127.0.0.1", "::1") - if parsed_url.scheme != "https" and not (settings.DEBUG or is_loopback): + # https except under DEBUG, the local-dev escape hatch where + # operators routinely point at a local receiver. We deliberately + # do *not* special-case loopback hosts here: in production the + # shared SSRF guard (``services.ssrf``) rejects loopback targets + # at dispatch anyway, so a hand-rolled allowance would only ever + # apply under DEBUG — which the check below already covers. + if parsed_url.scheme != "https" and not settings.DEBUG: raise serializers.ValidationError( - { - "settings": ( - "webhook settings.url must use https:// " - "(http:// is only allowed for localhost)." - ) - } + {"settings": "webhook settings.url must use https://."} ) events = settings_data.get("events") @@ -2411,27 +2409,25 @@ class ChannelCreateResponseSerializer(ChannelSerializer): # auth_method, so the others are intentionally absent. api_key = serializers.CharField( required=False, - help_text="api_key channels only — the plaintext API key.", + help_text=( + "Plaintext API key — api_key channels and webhook channels " + "with auth_method=api_key." + ), ) password = serializers.CharField( required=False, help_text="Plaintext password, when the channel type mints one.", ) - webhook_secret = serializers.CharField( + secret = serializers.CharField( required=False, help_text="webhook channels with auth_method=jwt — the HMAC/JWT signing secret.", ) - webhook_api_key = serializers.CharField( - required=False, - help_text="webhook channels with auth_method=api_key — the derived API key.", - ) class Meta(ChannelSerializer.Meta): fields = ChannelSerializer.Meta.fields + [ "api_key", "password", - "webhook_secret", - "webhook_api_key", + "secret", ] diff --git a/src/backend/core/api/viewsets/channel.py b/src/backend/core/api/viewsets/channel.py index dd25a5b4c..b3e130f74 100644 --- a/src/backend/core/api/viewsets/channel.py +++ b/src/backend/core/api/viewsets/channel.py @@ -23,14 +23,16 @@ def _attach_credential(data: dict, channel: models.Channel) -> None: """Add the channel's freshly-minted credential to ``data``. - Type-specific response key — lets the frontend branch on presence - instead of sniffing prefixes: + Response key by credential *kind* (the channel's type + auth_method + already tell the caller which to expect, so the keys are shared + across channel types rather than prefixed per type): - - ``api_key`` channels → ``api_key`` - - ``webhook`` channels (``auth_method='jwt'``) → ``webhook_secret`` + - ``api_key`` channels → ``api_key`` (plaintext, one-shot) + - ``webhook`` channels (``auth_method='jwt'``) → ``secret`` (the raw root from ``encrypted_settings["secret"]``) - - ``webhook`` channels (``auth_method='api_key'``) → - ``webhook_api_key`` (HMAC-derived from the root) + - ``webhook`` channels (``auth_method='api_key'``) → ``api_key`` + (HMAC-derived from the root) — same key name as api_key + channels: both are an API key presented in a request header For api_key channels the plaintext is one-shot (we only store the hash), so callers must stash it on ``instance._generated_api_key`` @@ -48,11 +50,11 @@ def _attach_credential(data: dict, channel: models.Channel) -> None: if auth_method == "jwt": root = (channel.encrypted_settings or {}).get("secret") if root: - data["webhook_secret"] = root + data["secret"] = root elif auth_method == "api_key": derived = channel.get_webhook_api_key() if derived: - data["webhook_api_key"] = derived + data["api_key"] = derived @extend_schema( @@ -120,8 +122,8 @@ def get_save_kwargs(self): response=serializers.ChannelCreateResponseSerializer, description=( "Channel created successfully. The response carries the " - "one-time plaintext credentials (api_key / webhook_secret / " - "webhook_api_key / password) which are never returned again." + "one-time plaintext credentials (api_key / secret / " + "password) which are never returned again." ), ), 400: OpenApiResponse(description="Invalid input data"), @@ -197,12 +199,16 @@ def destroy(self, request, *args, **kwargs): "api_key": drf_serializers.CharField( required=False, help_text=( - "Present for ``api_key`` channels — the " - "plaintext used in subsequent X-API-Key " - "calls. Returned ONCE." + "Present for ``api_key`` channels and " + "webhook channels with " + "``auth_method='api_key'`` — the plaintext " + "API key presented in a request header " + "(X-API-Key / X-StMsg-Api-Key). Returned " + "ONCE; for api_key webhooks it changes " + "whenever the root rotates." ), ), - "webhook_secret": drf_serializers.CharField( + "secret": drf_serializers.CharField( required=False, help_text=( "Present for webhook channels with " @@ -211,25 +217,14 @@ def destroy(self, request, *args, **kwargs): "HMAC sig and JWT." ), ), - "webhook_api_key": drf_serializers.CharField( - required=False, - help_text=( - "Present for webhook channels with " - "``auth_method='api_key'`` — the " - "HMAC-derived value sent as " - "X-StMsg-Api-Key. Changes whenever the " - "root rotates." - ), - ), }, ), description=( "Rotates the channel's secret. Single-active: the " "previous credential is invalidated immediately. " "The response carries exactly one of ``api_key`` / " - "``webhook_secret`` / ``webhook_api_key`` matching " - "the channel's type (and, for webhooks, its current " - "``auth_method``)." + "``secret`` matching the channel's type (and, for " + "webhooks, its current ``auth_method``)." ), ), 400: OpenApiResponse(description="Channel type has no rotatable secret"), diff --git a/src/backend/core/enums.py b/src/backend/core/enums.py index 23a70a89d..cfa16b923 100644 --- a/src/backend/core/enums.py +++ b/src/backend/core/enums.py @@ -228,8 +228,8 @@ class WebhookEvents(StrEnum): serializer at write time. Adding a new event is a Python-only change. """ - MESSAGE_RECEIVED = "message.received" - MESSAGE_SENT = "message.sent" + MESSAGE_INBOUND = "message.inbound" + MESSAGE_OUTBOUND = "message.outbound" class ChannelApiKeyScope(models.TextChoices): diff --git a/src/backend/core/mda/dispatch_webhooks.py b/src/backend/core/mda/dispatch_webhooks.py index 8ffd18b44..45f8483e5 100644 --- a/src/backend/core/mda/dispatch_webhooks.py +++ b/src/backend/core/mda/dispatch_webhooks.py @@ -46,7 +46,7 @@ from django.db.models import Q import jwt -from jmap_email import JmapEmail, first_msgid +from jmap_email import JmapEmail from core import enums, models from core.mda.inbound_pipeline import Decision, InboundContext, Step @@ -111,13 +111,6 @@ class _HttpResult: # pylint: disable=too-many-instance-attributes reply_draft_template_id: Optional[str] = None -# HTTP status families that mean "try again later" rather than -# "receiver rejected this message". 408 (timeout), 429 (rate limit), -# and 5xx are conventional retry signals across the webhook ecosystem -# (Stripe, GitHub, etc.). -_RETRY_STATUSES = frozenset({408, 429}) - - def _read_capped_body(response) -> bytes: """Read at most ``MAX_RESPONSE_BODY`` bytes from a streaming response. @@ -184,8 +177,8 @@ def _classify_response_body(body_bytes: bytes) -> _HttpResult: us to re-queue the inbound task; anything else (``"accept"``, missing) → CONTINUE. - ``is_spam``: bool; overrides the pipeline's spam verdict. - - ``labels``: list of label UUID strings; the pipeline validates - them against the destination mailbox. + - ``add_labels``: list of label UUID strings; the pipeline + validates them against the destination mailbox. """ if not body_bytes: return _HttpResult() @@ -212,7 +205,7 @@ def _classify_response_body(body_bytes: bytes) -> _HttpResult: if isinstance(is_spam, bool): result.is_spam_override = is_spam - labels = payload.get("labels") + labels = payload.get("add_labels") if isinstance(labels, list): for item in labels[:MAX_LABELS_PER_RESPONSE]: if not isinstance(item, str): @@ -299,13 +292,13 @@ def _classify_response_body(body_bytes: bytes) -> _HttpResult: FORMAT_EML = "eml" FORMAT_JMAP = "jmap" -# ``jmap_without_body`` is the cheap notification variant: same JMAP +# ``jmap_metadata`` is the cheap notification variant: same JMAP # envelope (headers, from/to/subject, messageId, etc.) but no body # parts, no bodyValues, no attachments. Receivers that only need the # "a message arrived" signal can use it without ever seeing the body # content over the wire. -FORMAT_JMAP_WITHOUT_BODY = "jmap_without_body" -VALID_FORMATS = frozenset({FORMAT_EML, FORMAT_JMAP, FORMAT_JMAP_WITHOUT_BODY}) +FORMAT_JMAP_METADATA = "jmap_metadata" +VALID_FORMATS = frozenset({FORMAT_EML, FORMAT_JMAP, FORMAT_JMAP_METADATA}) DEFAULT_FORMAT = FORMAT_EML USER_AGENT = "Messages-Webhook/1.0" @@ -515,38 +508,29 @@ def _envelope_headers( phase: str, mailbox: models.Mailbox, recipient_email: str, - parsed_email: JmapEmail, is_spam: Optional[bool], ) -> Dict[str, str]: """Build the ``X-StMsg-*`` envelope headers attached to every webhook POST regardless of body format. Same shape for ``eml`` and ``jmap``. + + The message-id is intentionally *not* a header: every body format + already carries it (``messageId`` in the jmap variants, the raw + ``Message-ID:`` header in ``eml``), so a header would only duplicate + it. """ if is_spam is None: spam_value = "unknown" else: spam_value = "true" if is_spam else "false" - mid = first_msgid(parsed_email.get("messageId")) - headers = { + return { "User-Agent": USER_AGENT, - "X-StMsg-Event": enums.WebhookEvents.MESSAGE_RECEIVED.value, + "X-StMsg-Event": enums.WebhookEvents.MESSAGE_INBOUND.value, "X-StMsg-Phase": phase, "X-StMsg-Channel-Id": str(channel.id), "X-StMsg-Mailbox": str(mailbox), "X-StMsg-Recipient": recipient_email, "X-StMsg-Is-Spam": spam_value, } - if mid: - # Re-bracket per RFC 5322 for headers (the JMAP body strips them). - # Strip CR/LF defensively: the parser normally does, but a - # malformed inbound Message-Id with embedded newlines would let - # the receiver see "spliced" headers. urllib3 would also reject - # it, but we'd rather drop the dodgy bytes than fail the POST. - mid = mid.replace("\r", "").replace("\n", "").strip() - if mid: - if not (mid.startswith("<") and mid.endswith(">")): - mid = f"<{mid}>" - headers["X-StMsg-Message-Id"] = mid - return headers # --- dispatch --- # @@ -564,12 +548,16 @@ class UserWebhookStep: - non-blocking → always ``CONTINUE`` (fire-and-forget; failures only logged) - blocking: - * 2xx + ``{"action":"drop"}`` → DROP + * 2xx + ``{"action":"drop"}`` → DROP (the *only* path to DROP) + * 2xx + ``{"action":"retry"}`` → RETRY * 2xx + anything else → CONTINUE (with optional side effects) - * 4xx → DROP (receiver definitively rejected) - * 408 / 429 / 5xx → RETRY (transient) - * SSRF / missing secret / unknown auth_method → DROP + * any non-2xx (4xx / 5xx / 3xx) → RETRY + * SSRF / missing secret / unknown auth_method → RETRY * timeout / connection / generic transport → RETRY + + A webhook error never drops the user's email — only an explicit + ``{"action": "drop"}`` does. Every failure is held for retry, + bounded by the pipeline's 7-day budget. """ def __init__(self, channel: models.Channel, phase: str): @@ -590,7 +578,6 @@ def __call__(self, ctx: InboundContext) -> Decision: phase=self.phase, mailbox=ctx.mailbox, recipient_email=ctx.recipient_email, - parsed_email=ctx.parsed_email, is_spam=ctx.is_spam, ) result = self._post( @@ -642,17 +629,22 @@ def _post( secret = (self.channel.encrypted_settings or {}).get("secret") if not secret: # The create path always mints a secret; a row without one - # is misconfigured. Fail closed rather than POST something a - # receiver cannot authenticate. + # is misconfigured. We can't sign the POST, so we hold for + # RETRY rather than drop the user's mail — re-minting the + # secret lets the next sweep deliver. A webhook failure must + # never silently discard the email (only an explicit + # ``{"action": "drop"}`` on a 2xx does that). logger.warning( - "Webhook channel %s has no secret — skipping", + "Webhook channel %s has no secret — holding for retry", self.channel.id, ) - return _failure(blocking, Decision.DROP) + return _failure(blocking, Decision.RETRY) auth_headers = self._build_auth_headers(secret, body_bytes, ctx.mailbox) if auth_headers is None: - return _failure(blocking, Decision.DROP) + # Unknown/misconfigured auth_method — same reasoning: hold + # for retry, never drop the email on our config error. + return _failure(blocking, Decision.RETRY) signed_headers = { **headers, @@ -669,14 +661,17 @@ def _post( url, timeout=WEBHOOK_TIMEOUT, stream=True, **kwargs ) except SSRFValidationError as exc: - # SSRF is a config error on our side — retrying won't fix it. + # SSRF block is a config error on our side (the URL points at a + # disallowed address). Hold for RETRY rather than drop — fixing + # the URL lets the next sweep deliver. We never discard the + # user's mail because of a webhook/config failure. logger.warning( "Webhook channel %s rejected by SSRF for url=%s: %s", self.channel.id, _sanitize_url(url), exc, ) - return _failure(blocking, Decision.DROP) + return _failure(blocking, Decision.RETRY) except Exception as exc: # Timeout, connection refused, DNS, unknown transport-level # failure: all transient on the blocking path. The 7-day cap @@ -713,10 +708,12 @@ def _post( status, _sanitize_url(url), ) - if status in _RETRY_STATUSES or 500 <= status < 600: - return _failure(blocking, Decision.RETRY) - # 3xx and any other non-2xx non-retry code: receiver said no. - return _failure(blocking, Decision.DROP) + # Any non-2xx status is a transient failure → RETRY. A + # blocking webhook DROPs an email *only* when it explicitly + # returns ``{"action": "drop"}`` with a 2xx (handled above). + # A receiver bug that answers 4xx must never cost the user + # their mail — the 7-day retry budget bounds the hold. + return _failure(blocking, Decision.RETRY) finally: response.close() @@ -783,8 +780,8 @@ def webhook_steps_for_mailbox(mailbox: models.Mailbox, *, phase: str) -> List[St cfg = channel.settings or {} if cfg.get("phase", PHASE_AFTER_SPAM) != phase: continue - events = cfg.get("events") or [enums.WebhookEvents.MESSAGE_RECEIVED.value] - if enums.WebhookEvents.MESSAGE_RECEIVED.value not in events: + events = cfg.get("events") or [enums.WebhookEvents.MESSAGE_INBOUND.value] + if enums.WebhookEvents.MESSAGE_INBOUND.value not in events: continue if not cfg.get("url"): continue diff --git a/src/backend/core/mda/inbound_pipeline.py b/src/backend/core/mda/inbound_pipeline.py index 1eecf2168..04df6d883 100644 --- a/src/backend/core/mda/inbound_pipeline.py +++ b/src/backend/core/mda/inbound_pipeline.py @@ -135,10 +135,19 @@ class InboundContext: # pylint: disable=too-many-instance-attributes # Inbound messages held by a transient RETRY get one more chance every -# 5 minutes via ``process_inbound_messages_queue_task``. After this cap -# we drop and log loudly so a permanently-broken receiver can't pin a -# row in the queue forever. -RETRY_MAX_AGE = timedelta(days=7) +# 5 minutes via ``process_inbound_messages_queue_task``. The webhook step +# is the only producer of a RETRY, and it is bounded by the quarantine +# window below — so a held message is never dropped, only delivered +# (flagged) once it gives up. +# +# A processing step that keeps failing (a blocking webhook today; rspamd +# or any future RETRY-returning step tomorrow) must not hold a message +# forever *or* silently lose it. After this window the task stops holding +# and delivers the message anyway, stamped with ``X-StMsg-Processing- +# Failed`` so the UI warns the recipient it bypassed a processing step. +# Generic on purpose — see the RETRY branch in +# ``process_inbound_message_task``. +QUARANTINE_AFTER = timedelta(hours=48) # --------------------------------------------------------------------------- @@ -289,6 +298,13 @@ def _make_rspamd_step(spam_config: Dict[str, Any]) -> Step: Sets ``is_spam`` if no earlier step decided. Always caches the full ``rspamd_result`` dict on the context — ``inbound_auth_step`` reuses the symbols (DKIM/DMARC) without a second HTTP call. + + An rspamd *error* never fails open: we don't deliver mail that + couldn't be spam-checked. The step RETRYs instead, so the message is + held and retried; if the outage lasts past ``QUARANTINE_AFTER`` the + quarantine path delivers it flagged rather than silently unchecked. + (rspamd simply not being configured is not an error — ``_call_rspamd`` + returns no opinion and the pipeline moves on.) """ def rspamd(ctx: InboundContext) -> Decision: @@ -299,11 +315,15 @@ def rspamd(ctx: InboundContext) -> Decision: return Decision.CONTINUE is_spam, err, result = _call_rspamd(ctx.raw_data, spam_config) if err: + # Don't fail open — hold for retry rather than deliver + # unchecked. A sustained outage is bounded by the quarantine + # window (the message is then delivered flagged). logger.warning( - "rspamd error on inbound message %s: %s (treating as not spam)", + "rspamd error on inbound message %s: %s (holding for retry)", ctx.inbound_message.id, err, ) + return Decision.RETRY ctx.rspamd_result = result if is_spam is not None: ctx.is_spam = is_spam diff --git a/src/backend/core/mda/inbound_tasks.py b/src/backend/core/mda/inbound_tasks.py index f4857481d..3c2424bc8 100644 --- a/src/backend/core/mda/inbound_tasks.py +++ b/src/backend/core/mda/inbound_tasks.py @@ -20,7 +20,7 @@ from core import models from core.mda.inbound_create import _create_message_from_inbound from core.mda.inbound_pipeline import ( - RETRY_MAX_AGE, + QUARANTINE_AFTER, Decision, InboundContext, apply_labels_to_thread, @@ -83,33 +83,27 @@ def _handle_retry( ) -> Dict[str, Any]: """Translate a RETRY decision into the task return value. - The InboundMessage row is kept in place unless it's older than - ``RETRY_MAX_AGE`` — the 5-min sweep + The InboundMessage row is kept in place — the 5-min sweep (``process_inbound_messages_queue_task``) re-fires the task on the - next cycle. Past the budget we drop and log loudly so a - permanently-broken receiver can't pin a row forever. + next cycle. We never drop here: a persistently-failing blocking + webhook is bounded instead by ``QUARANTINE_AFTER`` (the message is + then delivered flagged, see ``_stamp_processing_failed``), and the + webhook step is the only thing that produces a RETRY. """ age = timezone.now() - inbound_message.created_at - if age > RETRY_MAX_AGE: - logger.error( - "Inbound message %s exceeded retry budget (%s old) — dropping at step=%s", - inbound_message.id, - age, - step_name, - ) - inbound_message.delete() - return { - "success": False, - "inbound_message_id": str(inbound_message.id), - "error": "retry_exhausted", - "step": step_name, - } logger.info( "Inbound message %s held for retry at step=%s (age=%s)", inbound_message.id, step_name, age, ) + # Record why the row is parked so the queue is diagnosable straight + # from the admin / DB without grepping logs — important now that a + # webhook failure holds the message here instead of dropping it. + inbound_message.error_message = ( + f"Held for retry at step={step_name}" if step_name else "Held for retry" + ) + inbound_message.save(update_fields=["error_message"]) return { "success": False, "inbound_message_id": str(inbound_message.id), @@ -118,6 +112,28 @@ def _handle_retry( } +def _stamp_processing_failed(ctx: InboundContext) -> None: + """Prepend the ``X-StMsg-Processing-Failed`` marker to the message. + + Mirrors the ``X-StMsg-Sender-Auth`` prepend in the pipeline: the + header rides in the stored MIME, ``Message.get_stmsg_headers()`` + surfaces it as ``processing-failed``, and the frontend renders a + warning banner. Deliberately generic — any processing step that + fails persistently (a blocking webhook, rspamd, …) lands here. + Sender-supplied ``X-StMsg-*`` headers are stripped at ingest, so this + namespace is ours alone — the flag can't be forged. + """ + prepended = b"X-StMsg-Processing-Failed: true\r\n" + ctx.raw_data + reparsed = parse_email(prepended) + if reparsed is not None: + ctx.parsed_email = reparsed + ctx.raw_data = prepended + else: + # Keep raw_data / parsed_email in lockstep — drop the marker + # rather than corrupt the blob (same fallback as Sender-Auth). + logger.warning("Failed to re-parse after prepending X-StMsg-Processing-Failed") + + @celery_app.task(bind=True) def process_inbound_message_task(self, inbound_message_id: str): """Process an inbound message: run the pipeline, persist the result. @@ -189,8 +205,26 @@ def process_inbound_message_task(self, inbound_message_id: str): "inbound_message_id": str(inbound_message_id), "dropped_by": aborted_by, } + quarantined = False if decision == Decision.RETRY: - return _handle_retry(inbound_message, aborted_by) + age = timezone.now() - inbound_message.created_at + if age <= QUARANTINE_AFTER: + return _handle_retry(inbound_message, aborted_by) + # Past the quarantine window: a processing step (blocking + # webhook, rspamd, …) has failed persistently. Stop holding — + # deliver the message anyway so it's never lost, but stamp it + # so the UI warns the recipient it bypassed a processing step, + # and land it in the inbox (is_spam=False) so the warning is + # actually seen rather than buried in the spam folder. + logger.warning( + "Inbound message %s quarantine-delivered after persistent " + "failure at step=%s (age=%s)", + inbound_message_id, + aborted_by, + age, + ) + _stamp_processing_failed(ctx) + quarantined = True inbound_msg = _create_message_from_inbound( recipient_email=ctx.recipient_email, @@ -198,7 +232,7 @@ def process_inbound_message_task(self, inbound_message_id: str): raw_data=ctx.raw_data, mailbox=mailbox, channel=inbound_message.channel, - is_spam=bool(ctx.is_spam), + is_spam=False if quarantined else bool(ctx.is_spam), is_trashed=ctx.mark_trashed, is_archived=ctx.mark_archived, ) diff --git a/src/backend/core/tests/api/test_channel_scope_level.py b/src/backend/core/tests/api/test_channel_scope_level.py index a5f4b0b5d..1e8473bf7 100644 --- a/src/backend/core/tests/api/test_channel_scope_level.py +++ b/src/backend/core/tests/api/test_channel_scope_level.py @@ -1025,7 +1025,7 @@ def test_personal_webhook_channel(self, api_client): "type": "webhook", "settings": { "url": "https://hook.example.com/me", - "events": ["message.received"], "auth_method": "jwt", + "events": ["message.inbound"], "auth_method": "jwt", }, }, format="json", diff --git a/src/backend/core/tests/api/test_channels.py b/src/backend/core/tests/api/test_channels.py index f2fba4388..03bcb0800 100644 --- a/src/backend/core/tests/api/test_channels.py +++ b/src/backend/core/tests/api/test_channels.py @@ -588,53 +588,53 @@ def _post(self, api_client, mailbox, settings): @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) def test_create_minimal_webhook(self, api_client, mailbox): - """A JWT webhook surfaces its one-time ``webhook_secret`` on create.""" + """A JWT webhook surfaces its one-time ``secret`` on create.""" response = self._post( api_client, mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], + "events": ["message.inbound"], "auth_method": "jwt", }, ) assert response.status_code == status.HTTP_201_CREATED, response.content # The signing secret is returned exactly once, at creation time. - assert response.data.get("webhook_secret") - assert "webhook_api_key" not in response.data + assert response.data.get("secret") + assert "api_key" not in response.data @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) def test_create_minimal_webhook_api_key(self, api_client, mailbox): - """An api_key webhook surfaces its one-time ``webhook_api_key``.""" + """An api_key webhook surfaces its one-time ``api_key``.""" response = self._post( api_client, mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], + "events": ["message.inbound"], "auth_method": "api_key", }, ) assert response.status_code == status.HTTP_201_CREATED, response.content # api_key channels return the derived key, never the raw JWT secret. - assert response.data.get("webhook_api_key") - assert "webhook_secret" not in response.data + assert response.data.get("api_key") + assert "secret" not in response.data @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) def test_regenerate_secret_jwt_webhook(self, api_client, mailbox): - """Rotating a JWT webhook returns a fresh ``webhook_secret``.""" + """Rotating a JWT webhook returns a fresh ``secret``.""" create = self._post( api_client, mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], + "events": ["message.inbound"], "auth_method": "jwt", }, ) assert create.status_code == status.HTTP_201_CREATED, create.content channel_id = create.data["id"] - original_secret = create.data["webhook_secret"] + original_secret = create.data["secret"] url = reverse( "mailbox-channels-regenerate-secret", @@ -643,26 +643,26 @@ def test_regenerate_secret_jwt_webhook(self, api_client, mailbox): response = api_client.post(url) assert response.status_code == status.HTTP_200_OK, response.content assert response.data["id"] == str(channel_id) - new_secret = response.data["webhook_secret"] + new_secret = response.data["secret"] assert new_secret assert new_secret != original_secret - assert "webhook_api_key" not in response.data + assert "api_key" not in response.data @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) def test_regenerate_secret_api_key_webhook(self, api_client, mailbox): - """Rotating an api_key webhook returns a fresh ``webhook_api_key``.""" + """Rotating an api_key webhook returns a fresh ``api_key``.""" create = self._post( api_client, mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], + "events": ["message.inbound"], "auth_method": "api_key", }, ) assert create.status_code == status.HTTP_201_CREATED, create.content channel_id = create.data["id"] - original_key = create.data["webhook_api_key"] + original_key = create.data["api_key"] url = reverse( "mailbox-channels-regenerate-secret", @@ -671,10 +671,10 @@ def test_regenerate_secret_api_key_webhook(self, api_client, mailbox): response = api_client.post(url) assert response.status_code == status.HTTP_200_OK, response.content assert response.data["id"] == str(channel_id) - new_key = response.data["webhook_api_key"] + new_key = response.data["api_key"] assert new_key assert new_key != original_key - assert "webhook_secret" not in response.data + assert "secret" not in response.data @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) def test_create_with_all_dispatcher_options(self, api_client, mailbox): @@ -684,7 +684,7 @@ def test_create_with_all_dispatcher_options(self, api_client, mailbox): mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], + "events": ["message.inbound"], "auth_method": "jwt", "phase": "before_spam", "blocking": True, @@ -705,7 +705,7 @@ def test_rejects_invalid_format(self, api_client, mailbox): mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], + "events": ["message.inbound"], "auth_method": "jwt", "format": "yaml", }, @@ -713,16 +713,16 @@ def test_rejects_invalid_format(self, api_client, mailbox): assert response.status_code == status.HTTP_400_BAD_REQUEST @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) - def test_accepts_jmap_without_body_format(self, api_client, mailbox): - """The ``jmap_without_body`` format is a valid webhook format.""" + def test_accepts_jmap_metadata_format(self, api_client, mailbox): + """The ``jmap_metadata`` format is a valid webhook format.""" response = self._post( api_client, mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], + "events": ["message.inbound"], "auth_method": "jwt", - "format": "jmap_without_body", + "format": "jmap_metadata", }, ) assert response.status_code == status.HTTP_201_CREATED, response.content @@ -735,7 +735,7 @@ def test_rejects_invalid_phase(self, api_client, mailbox): mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], + "events": ["message.inbound"], "auth_method": "jwt", "phase": "whenever", }, @@ -750,7 +750,7 @@ def test_rejects_non_bool_blocking(self, api_client, mailbox): mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], + "events": ["message.inbound"], "auth_method": "jwt", "blocking": "yes", }, @@ -766,7 +766,7 @@ def test_rejects_patch_with_invalid_phase(self, api_client, mailbox): mailbox, { "url": "https://hook.example.com/in", - "events": ["message.received"], + "events": ["message.inbound"], "auth_method": "jwt", }, ) @@ -780,7 +780,7 @@ def test_rejects_patch_with_invalid_phase(self, api_client, mailbox): data={ "settings": { "url": "https://hook.example.com/in", - "events": ["message.received"], + "events": ["message.inbound"], "auth_method": "jwt", "phase": "bogus", } diff --git a/src/backend/core/tests/mda/test_dispatch_webhooks.py b/src/backend/core/tests/mda/test_dispatch_webhooks.py index 2da86bff3..2f1a6a907 100644 --- a/src/backend/core/tests/mda/test_dispatch_webhooks.py +++ b/src/backend/core/tests/mda/test_dispatch_webhooks.py @@ -30,7 +30,7 @@ webhook_steps_for_mailbox, ) from core.mda.inbound_pipeline import ( - RETRY_MAX_AGE, + QUARANTINE_AFTER, Decision, InboundContext, ) @@ -160,7 +160,7 @@ def test_finds_mailbox_scoped(self, mailbox): mailbox=mailbox, settings={ "url": "https://hook.example.com/a", - "events": ["message.received"], + "events": ["message.inbound"], }, ) assert list(find_webhook_channels_for_mailbox(mailbox)) == [ch] @@ -172,7 +172,7 @@ def test_finds_maildomain_scoped(self, mailbox): maildomain=mailbox.domain, settings={ "url": "https://hook.example.com/d", - "events": ["message.received"], + "events": ["message.inbound"], }, ) result = list(find_webhook_channels_for_mailbox(mailbox)) @@ -186,7 +186,7 @@ def test_finds_global_scoped(self, mailbox): scope_level=enums.ChannelScopeLevel.GLOBAL, settings={ "url": "https://hook.example.com/g", - "events": ["message.received"], + "events": ["message.inbound"], }, ) result = list(find_webhook_channels_for_mailbox(mailbox)) @@ -202,7 +202,7 @@ def test_global_fires_for_other_mailbox_too(self): scope_level=enums.ChannelScopeLevel.GLOBAL, settings={ "url": "https://hook.example.com/g", - "events": ["message.received"], + "events": ["message.inbound"], }, ) assert ch in find_webhook_channels_for_mailbox(mb_a) @@ -215,7 +215,7 @@ def test_excludes_other_mailbox(self, mailbox): mailbox=other, settings={ "url": "https://hook.example.com/x", - "events": ["message.received"], + "events": ["message.inbound"], }, ) assert not list(find_webhook_channels_for_mailbox(mailbox)) @@ -378,7 +378,7 @@ def test_skips_channel_with_wrong_phase(self, mock_session, mailbox, parsed_emai mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "before_spam", }, ) @@ -422,7 +422,7 @@ def test_non_blocking_continues_on_5xx(self, mock_session, mailbox, parsed_email mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": False, }, ) @@ -446,7 +446,7 @@ def test_blocking_retries_on_5xx(self, mock_session, mailbox, parsed_email): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": True, }, ) @@ -462,14 +462,15 @@ def test_blocking_retries_on_5xx(self, mock_session, mailbox, parsed_email): assert outcome.decision == Decision.RETRY @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_blocking_drops_on_4xx(self, mock_session, mailbox, parsed_email): - """4xx is a definitive receiver rejection — drop the message.""" + def test_blocking_retries_on_4xx(self, mock_session, mailbox, parsed_email): + """A webhook error never drops the email: 4xx is held for RETRY. + Only an explicit {"action": "drop"} on a 2xx drops the message.""" factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": True, }, ) @@ -482,7 +483,7 @@ def test_blocking_drops_on_4xx(self, mock_session, mailbox, parsed_email): raw_data=b"raw", is_spam=False, ) - assert outcome.decision == Decision.DROP + assert outcome.decision == Decision.RETRY @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_blocking_retries_on_408(self, mock_session, mailbox, parsed_email): @@ -492,7 +493,7 @@ def test_blocking_retries_on_408(self, mock_session, mailbox, parsed_email): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": True, }, ) @@ -515,7 +516,7 @@ def test_blocking_retries_on_429(self, mock_session, mailbox, parsed_email): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": True, }, ) @@ -531,16 +532,17 @@ def test_blocking_retries_on_429(self, mock_session, mailbox, parsed_email): assert outcome.decision == Decision.RETRY @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_blocking_drops_on_ssrf_rejection( + def test_blocking_retries_on_ssrf_rejection( self, mock_session, mailbox, parsed_email ): - """SSRF rejection is a config error on our side — retrying won't help.""" + """SSRF rejection is a config error on our side — held for RETRY + (fix the URL and the next sweep delivers), never dropped.""" factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, settings={ "url": "https://internal.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": True, }, ) @@ -553,7 +555,7 @@ def test_blocking_drops_on_ssrf_rejection( raw_data=b"raw", is_spam=False, ) - assert outcome.decision == Decision.DROP + assert outcome.decision == Decision.RETRY @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_non_blocking_continues_on_ssrf_rejection( @@ -564,7 +566,7 @@ def test_non_blocking_continues_on_ssrf_rejection( mailbox=mailbox, settings={ "url": "https://internal.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": False, }, ) @@ -587,7 +589,7 @@ def test_blocking_retries_on_timeout(self, mock_session, mailbox, parsed_email): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": True, }, ) @@ -612,7 +614,7 @@ def test_blocking_retries_on_connection_error( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": True, }, ) @@ -640,7 +642,7 @@ def test_blocking_retries_on_unknown_exception( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": True, }, ) @@ -664,7 +666,7 @@ def test_phase_filtering_dispatches_only_matching( mailbox=mailbox, settings={ "url": "https://hook.example.com/before", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "before_spam", }, ) @@ -673,7 +675,7 @@ def test_phase_filtering_dispatches_only_matching( mailbox=mailbox, settings={ "url": "https://hook.example.com/after", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "after_spam", }, ) @@ -696,7 +698,7 @@ def test_eml_format_sends_raw_body(self, mock_session, mailbox, parsed_email): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -722,7 +724,7 @@ def test_jmap_format_sends_jmap_email_json( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "format": "jmap", }, ) @@ -747,7 +749,7 @@ def test_jmap_format_sends_jmap_email_json( assert kwargs["headers"]["Content-Type"] == "application/json" @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_jmap_without_body_skips_body_parts( + def test_jmap_metadata_skips_body_parts( self, mock_session, mailbox, parsed_email ): """Notification variant: no textBody/htmlBody/bodyValues/attachments.""" @@ -756,8 +758,8 @@ def test_jmap_without_body_skips_body_parts( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], - "format": "jmap_without_body", + "events": ["message.inbound"], + "format": "jmap_metadata", }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -798,7 +800,7 @@ def test_envelope_headers_set_for_both_formats( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "format": fmt, }, ) @@ -812,12 +814,14 @@ def test_envelope_headers_set_for_both_formats( is_spam=True, ) headers = mock_session.return_value.post.call_args.kwargs["headers"] - assert headers["X-StMsg-Event"] == "message.received" + assert headers["X-StMsg-Event"] == "message.inbound" assert headers["X-StMsg-Phase"] == "after_spam" assert headers["X-StMsg-Mailbox"] == str(mailbox) assert headers["X-StMsg-Recipient"] == str(mailbox) assert headers["X-StMsg-Is-Spam"] == "true" - assert headers["X-StMsg-Message-Id"] == "" + # The message-id is NOT a header — every body format already + # carries it (messageId in jmap, raw Message-ID: in eml). + assert "X-StMsg-Message-Mime-Id" not in headers @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_is_spam_header_unknown_when_none( @@ -828,7 +832,7 @@ def test_is_spam_header_unknown_when_none( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "before_spam", }, ) @@ -853,7 +857,7 @@ def test_invalid_format_skips_dispatch(self, mock_session, mailbox, parsed_email mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "format": "yaml", }, ) @@ -897,7 +901,7 @@ def test_eml_signature_covers_raw_body(self, mock_session, mailbox, parsed_email mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "format": "eml", }, ) @@ -936,7 +940,7 @@ def test_jmap_signature_covers_exact_serialised_bytes( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "format": "jmap", }, ) @@ -975,7 +979,7 @@ def test_api_key_mode_sends_only_api_key_header( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "auth_method": "api_key", }, ) @@ -1010,7 +1014,7 @@ def test_jwt_mode_sends_only_hmac_and_jwt_headers( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "auth_method": "jwt", }, ) @@ -1044,7 +1048,7 @@ def test_missing_auth_method_fails_closed( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], }, encrypted_settings={"secret": "whsec_test"}, ) @@ -1070,7 +1074,7 @@ def test_api_key_value_is_derived_not_raw_secret( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "auth_method": "api_key", }, ) @@ -1108,7 +1112,7 @@ def test_missing_secret_fails_closed(self, mock_session, mailbox, parsed_email): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], }, encrypted_settings={}, ) @@ -1123,12 +1127,13 @@ def test_missing_secret_fails_closed(self, mock_session, mailbox, parsed_email): mock_session.return_value.post.assert_not_called() @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_missing_secret_blocks_when_blocking( + def test_missing_secret_retries_when_blocking( self, mock_session, mailbox, parsed_email ): - """If the misconfigured channel is also ``blocking``, the - dispatcher MUST drop the message — better than POSTing an - unsigned request that any verifying receiver will reject.""" + """A misconfigured (secret-less) ``blocking`` channel can't sign + the POST, so the dispatcher holds the message for RETRY rather + than dropping it — re-minting the secret lets the next sweep + deliver. A config error must never discard the user's mail.""" models.Channel.objects.create( name="no-secret-blocking", type=enums.ChannelTypes.WEBHOOK, @@ -1136,7 +1141,7 @@ def test_missing_secret_blocks_when_blocking( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": True, }, encrypted_settings={}, @@ -1149,7 +1154,7 @@ def test_missing_secret_blocks_when_blocking( raw_data=b"raw", is_spam=False, ) - assert outcome.decision == Decision.DROP + assert outcome.decision == Decision.RETRY mock_session.return_value.post.assert_not_called() @@ -1161,7 +1166,7 @@ class TestPipelineIntegration: @patch("core.mda.inbound_tasks._create_message_from_inbound") @patch("core.mda.inbound_pipeline._call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_before_spam_blocking_drops_message( + def test_before_spam_blocking_retries_message( self, mock_session, mock_check_spam, mock_create_message ): mailbox = factories.MailboxFactory() @@ -1178,26 +1183,28 @@ def test_before_spam_blocking_drops_message( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "before_spam", "blocking": True, }, ) - # 4xx → receiver definitively rejects this message → DROP. + # 4xx is a webhook error, not an explicit drop → hold for RETRY. mock_session.return_value.post.return_value = _make_response(403) with patch.object(process_inbound_message_task, "update_state", Mock()): result = process_inbound_message_task.run(str(inbound_message.id)) - assert result["dropped_by"].endswith(":before_spam") + assert result["error"] == "retry" + assert result["step"].endswith(":before_spam") mock_check_spam.assert_not_called() mock_create_message.assert_not_called() - assert not models.InboundMessage.objects.filter(id=inbound_message.id).exists() + # The email is NOT dropped — its row is kept for the next sweep. + assert models.InboundMessage.objects.filter(id=inbound_message.id).exists() @patch("core.mda.inbound_tasks._create_message_from_inbound") @patch("core.mda.inbound_pipeline._call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_after_spam_blocking_drops_message( + def test_after_spam_blocking_retries_message( self, mock_session, mock_check_spam, mock_create_message ): mailbox = factories.MailboxFactory() @@ -1214,18 +1221,20 @@ def test_after_spam_blocking_drops_message( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "after_spam", "blocking": True, }, ) mock_check_spam.return_value = (False, None, None) + # 4xx is a webhook error, not an explicit drop → hold for RETRY. mock_session.return_value.post.return_value = _make_response(403) with patch.object(process_inbound_message_task, "update_state", Mock()): result = process_inbound_message_task.run(str(inbound_message.id)) - assert result["dropped_by"].endswith(":after_spam") + assert result["error"] == "retry" + assert result["step"].endswith(":after_spam") mock_check_spam.assert_called_once() mock_create_message.assert_not_called() @@ -1250,7 +1259,7 @@ def test_after_spam_is_spam_header( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "after_spam", }, ) @@ -1334,19 +1343,19 @@ def test_labels_uuids_collected(self): a = str(uuid.uuid4()) b = str(uuid.uuid4()) outcome = _classify_response_body( - json.dumps({"labels": [a, b]}).encode("utf-8") + json.dumps({"add_labels": [a, b]}).encode("utf-8") ) assert outcome.labels == {a, b} def test_labels_non_uuid_strings_skipped(self): good = str(uuid.uuid4()) outcome = _classify_response_body( - json.dumps({"labels": [good, "not-a-uuid", "", 42]}).encode("utf-8") + json.dumps({"add_labels": [good, "not-a-uuid", "", 42]}).encode("utf-8") ) assert outcome.labels == {good} def test_labels_non_list_ignored(self): - outcome = _classify_response_body(b'{"labels": "spam"}') + outcome = _classify_response_body(b'{"add_labels": "spam"}') assert outcome.labels == set() def test_combined_action_and_labels(self): @@ -1355,7 +1364,7 @@ def test_combined_action_and_labels(self): merge logic shouldn't lose them either).""" good = str(uuid.uuid4()) outcome = _classify_response_body( - json.dumps({"action": "drop", "labels": [good]}).encode("utf-8") + json.dumps({"action": "drop", "add_labels": [good]}).encode("utf-8") ) assert outcome.decision == Decision.DROP assert outcome.labels == {good} @@ -1495,7 +1504,7 @@ def test_oversize_arrays_are_capped(self): ] outcome = _classify_response_body( json.dumps( - {"labels": labels, "assign_to": emails, "add_event": events} + {"add_labels": labels, "assign_to": emails, "add_event": events} ).encode("utf-8") ) assert len(outcome.labels) == MAX_LABELS_PER_RESPONSE @@ -1540,7 +1549,7 @@ def test_blocking_drops_on_action_drop_body( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": True, }, ) @@ -1566,7 +1575,7 @@ def test_blocking_is_spam_override_continues( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": True, }, ) @@ -1596,7 +1605,7 @@ def test_non_blocking_ignores_action_body( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": False, }, ) @@ -1624,7 +1633,7 @@ def test_multi_webhook_drop_wins_and_short_circuits( mailbox=mailbox, settings={ "url": "https://hook.example.com/first", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": True, }, ) @@ -1633,7 +1642,7 @@ def test_multi_webhook_drop_wins_and_short_circuits( mailbox=mailbox, settings={ "url": "https://hook.example.com/second", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": True, }, ) @@ -1666,7 +1675,7 @@ def test_response_body_is_capped(self, mock_session, mailbox, parsed_email): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "blocking": True, }, ) @@ -1731,7 +1740,7 @@ def test_5xx_retries_and_keeps_inbound_message( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "before_spam", "blocking": True, }, @@ -1769,7 +1778,7 @@ def test_timeout_retries_and_keeps_inbound_message( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "before_spam", "blocking": True, }, @@ -1787,11 +1796,13 @@ def test_timeout_retries_and_keeps_inbound_message( @patch("core.mda.inbound_tasks._create_message_from_inbound") @patch("core.mda.inbound_pipeline._call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_retry_past_max_age_drops_with_loud_log( + def test_blocking_webhook_quarantine_delivers_flagged_after_window( self, mock_session, mock_check_spam, mock_create_message ): - """An InboundMessage held in retry for >7 days is dropped to - prevent a broken receiver from pinning the row forever.""" + """Past the 48h window a still-failing blocking webhook stops + holding: the message is delivered (never dropped), stamped + ``X-StMsg-Processing-Failed``, and forced to the inbox so the + warning banner is actually seen.""" mailbox = factories.MailboxFactory() raw_data = ( b"From: sender@example.com\r\n" @@ -1801,10 +1812,10 @@ def test_retry_past_max_age_drops_with_loud_log( inbound_message = models.InboundMessage.objects.create( mailbox=mailbox, raw_data=raw_data ) - # Backdate past the 7-day cap. + # Backdate past the quarantine window (auto_now_add → update()). models.InboundMessage.objects.filter(id=inbound_message.id).update( created_at=dj_timezone.now() - - RETRY_MAX_AGE + - QUARANTINE_AFTER - dj_timezone.timedelta(minutes=1) ) factories.ChannelFactory( @@ -1812,21 +1823,32 @@ def test_retry_past_max_age_drops_with_loud_log( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "before_spam", "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(503) + mock_create_message.return_value = True with patch.object(process_inbound_message_task, "update_state", Mock()): - result = process_inbound_message_task.run(str(inbound_message.id)) + process_inbound_message_task.run(str(inbound_message.id)) - assert result["error"] == "retry_exhausted" - # Row is gone — bounded retry budget. - assert not models.InboundMessage.objects.filter(id=inbound_message.id).exists() + # The pipeline aborts at the failing before_spam webhook (RETRY), + # so quarantine is decided at the task level — generic, not + # webhook-specific: rspamd is never reached this run. mock_check_spam.assert_not_called() - mock_create_message.assert_not_called() + # Delivered, not dropped. + mock_create_message.assert_called_once() + kwargs = mock_create_message.call_args.kwargs + # Stamped for the UI banner... + assert b"X-StMsg-Processing-Failed: true" in kwargs["raw_data"] + # ...and forced to the inbox (is_spam=False) so the banner is seen. + assert kwargs["is_spam"] is False + # Queue row consumed — not pinned, not dropped silently. + assert not models.InboundMessage.objects.filter( + id=inbound_message.id + ).exists() @pytest.mark.django_db @@ -1853,7 +1875,7 @@ def test_before_spam_is_spam_override_short_circuits_rspamd( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "before_spam", "blocking": True, }, @@ -1893,7 +1915,7 @@ def test_after_spam_is_spam_override_replaces_verdict( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "after_spam", "blocking": True, }, @@ -1940,7 +1962,7 @@ def test_webhook_label_attached_to_thread(self, mock_session, mock_check_spam): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "after_spam", "blocking": True, }, @@ -1950,7 +1972,7 @@ def test_webhook_label_attached_to_thread(self, mock_session, mock_check_spam): 200, body=json.dumps( { - "labels": [ + "add_labels": [ str(good_label.id), str(other_label.id), # wrong mailbox → skipped unknown_id, # unknown UUID → skipped @@ -2003,7 +2025,7 @@ def test_assign_to_resolves_email_and_attributes_channel( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "after_spam", "blocking": True, }, @@ -2079,7 +2101,7 @@ def test_assign_to_skips_unknown_ambiguous_and_viewer( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "after_spam", "blocking": True, }, @@ -2144,7 +2166,7 @@ def test_two_webhooks_each_produce_own_threadevent( mailbox=mailbox, settings={ "url": "https://hook.example.com/a", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "after_spam", "blocking": True, }, @@ -2154,7 +2176,7 @@ def test_two_webhooks_each_produce_own_threadevent( mailbox=mailbox, settings={ "url": "https://hook.example.com/b", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "after_spam", "blocking": True, }, @@ -2212,7 +2234,7 @@ def _send(self, mailbox, mime_id, action_body: bytes): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "after_spam", "blocking": True, }, @@ -2274,7 +2296,7 @@ def test_skip_autoreply_suppresses_autoreply_call( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "after_spam", "blocking": True, }, @@ -2315,7 +2337,7 @@ def test_add_event_im_creates_threadevent(self, mock_session, mock_rspamd): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "after_spam", "blocking": True, }, @@ -2389,7 +2411,7 @@ def test_reply_draft_creates_draft_with_template_body( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "after_spam", "blocking": True, }, @@ -2451,7 +2473,7 @@ def test_reply_draft_out_of_scope_template_skipped(self, mock_session, mock_rspa mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "after_spam", "blocking": True, }, @@ -2514,7 +2536,7 @@ def test_assign_failure_does_not_skip_labels( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.received"], + "events": ["message.inbound"], "phase": "after_spam", "blocking": True, }, @@ -2524,7 +2546,7 @@ def test_assign_failure_does_not_skip_labels( 200, body=json.dumps( { - "labels": [str(label.id)], + "add_labels": [str(label.id)], "assign_to": ["editor@example.org"], } ).encode("utf-8"), diff --git a/src/backend/core/tests/mda/test_spam_processing.py b/src/backend/core/tests/mda/test_spam_processing.py index e41850803..bba106cbd 100644 --- a/src/backend/core/tests/mda/test_spam_processing.py +++ b/src/backend/core/tests/mda/test_spam_processing.py @@ -11,7 +11,13 @@ from core import factories, models from core.mda.inbound import deliver_inbound_message -from core.mda.inbound_pipeline import _call_rspamd, _check_hardcoded_rules +from core.mda.inbound_pipeline import ( + Decision, + InboundContext, + _call_rspamd, + _check_hardcoded_rules, + _make_rspamd_step, +) from core.mda.inbound_tasks import ( process_inbound_message_task, process_inbound_messages_queue_task, @@ -854,3 +860,56 @@ def test_process_inbound_messages_queue_task(self, mock_task_delay): assert result["processed"] == 3 assert result["total"] == 3 assert mock_task_delay.call_count == 3 + + +@pytest.mark.django_db +class TestRspamdStepFailureHandling: + """rspamd never fails open: an error holds the message for retry + (feeding the quarantine path) rather than delivering it unchecked.""" + + def _ctx(self, spam_config): + mailbox = factories.MailboxFactory() + inbound = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=b"raw" + ) + return InboundContext( + mailbox=mailbox, + inbound_message=inbound, + recipient_email=str(mailbox), + raw_data=b"raw", + parsed_email={}, + spam_config=spam_config, + ) + + @patch("core.mda.inbound_pipeline._call_rspamd") + def test_error_holds_for_retry(self, mock_call): + spam_config = {"rspamd_url": "http://rspamd:11334"} + # _call_rspamd returns is_spam=False on error. + mock_call.return_value = (False, "connection refused", None) + + decision = _make_rspamd_step(spam_config)(self._ctx(spam_config)) + + # Never fail open — hold, don't deliver unchecked. + assert decision == Decision.RETRY + + @patch("core.mda.inbound_pipeline._call_rspamd") + def test_not_configured_continues(self, mock_call): + # rspamd absent is "no opinion", not an error → keep moving. + mock_call.return_value = (None, None, None) + ctx = self._ctx({}) + + decision = _make_rspamd_step({})(ctx) + + assert decision == Decision.CONTINUE + assert ctx.is_spam is None + + @patch("core.mda.inbound_pipeline._call_rspamd") + def test_success_sets_verdict(self, mock_call): + spam_config = {"rspamd_url": "http://rspamd:11334"} + mock_call.return_value = (True, None, {"action": "reject"}) + ctx = self._ctx(spam_config) + + decision = _make_rspamd_step(spam_config)(ctx) + + assert decision == Decision.CONTINUE + assert ctx.is_spam is True diff --git a/src/backend/messages/settings.py b/src/backend/messages/settings.py index 1df9efc9a..30a637158 100644 --- a/src/backend/messages/settings.py +++ b/src/backend/messages/settings.py @@ -1478,6 +1478,15 @@ class Test(Base): FEATURE_MAILBOX_ADMIN_CHANNELS = ["api_key", "widget", "webhook"] + # Tests must not depend on a reachable rspamd. With no rspamd_url the + # spam step is a no-op ("no opinion"), so inbound delivery tests + # deliver deterministically; spam-specific tests opt in explicitly via + # ``@override_settings(SPAM_CONFIG={"rspamd_url": ...})`` and mock the + # HTTP call. (A configured-but-unreachable rspamd now *holds* mail for + # retry instead of failing open — see ``_make_rspamd_step`` — so an + # inherited env SPAM_CONFIG would otherwise stall delivery in tests.) + SPAM_CONFIG = {} + SCHEMA_CUSTOM_ATTRIBUTES_USER = {} SCHEMA_CUSTOM_ATTRIBUTES_MAILDOMAIN = {} diff --git a/src/frontend/public/locales/common/en-US.json b/src/frontend/public/locales/common/en-US.json index a5412c72c..bf8a2bc0c 100755 --- a/src/frontend/public/locales/common/en-US.json +++ b/src/frontend/public/locales/common/en-US.json @@ -770,6 +770,7 @@ "This message has not been delivered. You cancelled the delivery.": "This message has not been delivered. You cancelled the delivery.", "This message has not yet been delivered to all recipients.": "This message has not yet been delivered to all recipients.", "This message is being delivered.": "This message is being delivered.", + "This message was delivered without our usual safety checks. Please review it with caution.": "This message was delivered without our usual safety checks. Please review it with caution.", "This name is for internal use only and will not be visible to users.": "This name is for internal use only and will not be visible to users.", "This signature is forced": "This signature is forced", "This thread has been reported as spam.": "This thread has been reported as spam.", diff --git a/src/frontend/public/locales/common/fr-FR.json b/src/frontend/public/locales/common/fr-FR.json index 4c6cecfbe..8203014aa 100755 --- a/src/frontend/public/locales/common/fr-FR.json +++ b/src/frontend/public/locales/common/fr-FR.json @@ -857,6 +857,7 @@ "This message has not been delivered. You cancelled the delivery.": "Ce message n'a pas pu être délivré. Vous avez annulé l'envoi.", "This message has not yet been delivered to all recipients.": "Ce message n'a pas encore été délivré à tous les destinataires.", "This message is being delivered.": "Ce message est en cours d'envoi.", + "This message was delivered without our usual safety checks. Please review it with caution.": "Ce message a été délivré sans nos vérifications de sécurité habituelles. Veuillez l'examiner avec prudence.", "This name is for internal use only and will not be visible to users.": "Ce nom est réservé à un usage interne et ne sera pas visible par les utilisateurs.", "This signature is forced": "Cette signature est forcée", "This thread has been reported as spam.": "Cette conversation a été signalée comme spam.", diff --git a/src/frontend/public/locales/common/nl-NL.json b/src/frontend/public/locales/common/nl-NL.json index 677de2755..82d1cc7cb 100644 --- a/src/frontend/public/locales/common/nl-NL.json +++ b/src/frontend/public/locales/common/nl-NL.json @@ -471,6 +471,7 @@ "This message has not been delivered. You cancelled the delivery.": "Dit bericht is niet verzonden. Je hebt de levering geannuleerd.", "This message has not yet been delivered to all recipients.": "Dit bericht is nog niet aan alle ontvangers afgeleverd.", "This message is being delivered.": "Dit bericht wordt verstuurd.", + "This message was delivered without our usual safety checks. Please review it with caution.": "Dit bericht is bezorgd zonder onze gebruikelijke veiligheidscontroles. Wees voorzichtig.", "This name is for internal use only and will not be visible to users.": "Deze naam is alleen voor intern gebruik en is niet zichtbaar voor gebruikers.", "This signature is forced": "Deze handtekening is geforceerd", "This thread has been reported as spam.": "Deze discussie is gerapporteerd als spam.", diff --git a/src/frontend/public/locales/common/ru-RU.json b/src/frontend/public/locales/common/ru-RU.json index 7ef431e95..8b042b2ce 100644 --- a/src/frontend/public/locales/common/ru-RU.json +++ b/src/frontend/public/locales/common/ru-RU.json @@ -736,6 +736,7 @@ "This message has not been delivered. You cancelled the delivery.": "Это сообщение не было доставлено. Вы отменили доставку.", "This message has not yet been delivered to all recipients.": "Это сообщение ещё не доставлено всем получателям.", "This message is being delivered.": "Это сообщение доставлено.", + "This message was delivered without our usual safety checks. Please review it with caution.": "Это сообщение было доставлено без наших обычных проверок безопасности. Будьте с ним осторожны.", "This name is for internal use only and will not be visible to users.": "Это имя только для внутреннего использования и не будет видно пользователям.", "This signature is forced": "Эта подпись обязательна", "This thread has been reported as spam.": "Обсуждение было помечено как спам.", diff --git a/src/frontend/public/locales/common/uk-UA.json b/src/frontend/public/locales/common/uk-UA.json index 9bfb7865d..e78507010 100644 --- a/src/frontend/public/locales/common/uk-UA.json +++ b/src/frontend/public/locales/common/uk-UA.json @@ -736,6 +736,7 @@ "This message has not been delivered. You cancelled the delivery.": "Це повідомлення не було доставлено. Ви скасували доставлення.", "This message has not yet been delivered to all recipients.": "Це повідомлення ще не було доставлено усім отримувачам.", "This message is being delivered.": "Це повідомлення доставлено.", + "This message was delivered without our usual safety checks. Please review it with caution.": "Це повідомлення було доставлено без наших звичайних перевірок безпеки. Будьте обачні.", "This name is for internal use only and will not be visible to users.": "Це ім'я тільки для внутрішнього використання та не буде видимим користувачам.", "This signature is forced": "Цей підпис є обов'язковим", "This thread has been reported as spam.": "Обговорення позначено як спам.", diff --git a/src/frontend/src/features/api/gen/models/channel_create_response.ts b/src/frontend/src/features/api/gen/models/channel_create_response.ts index b8abb14ff..c39e3bb16 100644 --- a/src/frontend/src/features/api/gen/models/channel_create_response.ts +++ b/src/frontend/src/features/api/gen/models/channel_create_response.ts @@ -54,12 +54,10 @@ export interface ChannelCreateResponse { readonly created_at: string; /** date and time at which a record was last updated */ readonly updated_at: string; - /** api_key channels only — the plaintext API key. */ + /** Plaintext API key — api_key channels and webhook channels with auth_method=api_key. */ api_key?: string; /** Plaintext password, when the channel type mints one. */ password?: string; /** webhook channels with auth_method=jwt — the HMAC/JWT signing secret. */ - webhook_secret?: string; - /** webhook channels with auth_method=api_key — the derived API key. */ - webhook_api_key?: string; + secret?: string; } diff --git a/src/frontend/src/features/api/gen/models/regenerated_secret_response.ts b/src/frontend/src/features/api/gen/models/regenerated_secret_response.ts index 6227a6703..ad427e93f 100644 --- a/src/frontend/src/features/api/gen/models/regenerated_secret_response.ts +++ b/src/frontend/src/features/api/gen/models/regenerated_secret_response.ts @@ -9,10 +9,8 @@ export interface RegeneratedSecretResponse { /** Channel id. */ id: string; - /** Present for ``api_key`` channels — the plaintext used in subsequent X-API-Key calls. Returned ONCE. */ + /** Present for ``api_key`` channels and webhook channels with ``auth_method='api_key'`` — the plaintext API key presented in a request header (X-API-Key / X-StMsg-Api-Key). Returned ONCE; for api_key webhooks it changes whenever the root rotates. */ api_key?: string; /** Present for webhook channels with ``auth_method='jwt'`` — the freshly minted root receivers use to verify the HMAC sig and JWT. */ - webhook_secret?: string; - /** Present for webhook channels with ``auth_method='api_key'`` — the HMAC-derived value sent as X-StMsg-Api-Key. Changes whenever the root rotates. */ - webhook_api_key?: string; + secret?: string; } diff --git a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx index 594a62965..04454155e 100644 --- a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx +++ b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx @@ -26,7 +26,7 @@ type WebhookChannelSettings = { url?: string; events?: string[]; phase?: "before_spam" | "after_spam"; - format?: "eml" | "jmap" | "jmap_without_body"; + format?: "eml" | "jmap" | "jmap_metadata"; blocking?: boolean; auth_method?: "jwt" | "api_key"; }; @@ -53,7 +53,7 @@ const createFormSchema = (t: (key: string) => string) => error: t("URL must start with http:// or https://"), }), phase: z.enum(["before_spam", "after_spam"]), - format: z.enum(["eml", "jmap", "jmap_without_body"]), + format: z.enum(["eml", "jmap", "jmap_metadata"]), blocking: z.boolean(), auth_method: z.enum(["jwt", "api_key"]), }); @@ -109,7 +109,7 @@ export const WebhookIntegrationForm = ({ const newSettings: WebhookChannelSettings = { url: data.url, - events: ["message.received"], + events: ["message.inbound"], phase: data.phase, format: data.format, blocking: data.blocking, @@ -151,22 +151,17 @@ export const WebhookIntegrationForm = ({ // Surface the freshly minted credential exactly // once — the receiver needs this value to verify // every webhook we send. The backend returns - // ``webhook_secret`` for auth_method=jwt and - // ``webhook_api_key`` for auth_method=api_key. - // The one-time ``webhook_secret`` / ``webhook_api_key`` - // are create-only response fields, not part of the + // ``secret`` for auth_method=jwt and ``api_key`` for + // auth_method=api_key. These one-time credentials are + // create-only response fields, not part of the // generated ``Channel`` type — read them off an // index-signature view. const payload = newChannel.data as unknown as Record< string, unknown >; - const secret = payload.webhook_secret as - | string - | undefined; - const apiKey = payload.webhook_api_key as - | string - | undefined; + const secret = payload.secret as string | undefined; + const apiKey = payload.api_key as string | undefined; if (secret) { setCreatedCredential({ label: t("Webhook signing secret"), @@ -307,8 +302,8 @@ export const WebhookIntegrationForm = ({ value: "jmap", }, { - label: t("JMAP Email JSON without body (notification only)"), - value: "jmap_without_body", + label: t("JMAP Email metadata (notification only)"), + value: "jmap_metadata", }, ]} text={t( diff --git a/src/frontend/src/features/layouts/components/thread-view/components/thread-message/thread-message-header.tsx b/src/frontend/src/features/layouts/components/thread-view/components/thread-message/thread-message-header.tsx index cfb56b8b5..9e8432da1 100644 --- a/src/frontend/src/features/layouts/components/thread-view/components/thread-message/thread-message-header.tsx +++ b/src/frontend/src/features/layouts/components/thread-view/components/thread-message/thread-message-header.tsx @@ -41,6 +41,11 @@ const ThreadMessageHeader = ({ const isUnverifiedSender = senderAuth === 'none'; const isForgedSender = senderAuth === 'fail'; + // A processing step (a blocking integration/webhook, spam check, …) + // failed repeatedly, so the message was delivered without it (see the + // quarantine path in the inbound pipeline). Warn prominently. + const processingFailed = Boolean(message.stmsg_headers?.['processing-failed']); + const isUserSender = useMemo(() => { if (!message.is_sender) return false; if (!message.sender_user) return true; @@ -162,6 +167,13 @@ const ThreadMessageHeader = ({ )} + {processingFailed && ( + +
+

{t("This message was delivered without our usual safety checks. Please review it with caution.")}

+
+ + )}
From c81b92bd49ec9c1177f632a5a4b9fb17be0a0dbd Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Fri, 19 Jun 2026 18:34:35 +0200 Subject: [PATCH 05/21] =?UTF-8?q?=F0=9F=94=A7(api)=20regenerate=20openapi?= =?UTF-8?q?=20schema=20and=20add=20missing=20test=20docstrings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/core/api/openapi.json | 24 +++++++------------ src/backend/core/mda/dispatch_webhooks.py | 16 ++++++++++--- .../core/tests/mda/test_spam_processing.py | 3 +++ 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/backend/core/api/openapi.json b/src/backend/core/api/openapi.json index ef445d433..18b676faa 100644 --- a/src/backend/core/api/openapi.json +++ b/src/backend/core/api/openapi.json @@ -2982,7 +2982,7 @@ } } }, - "description": "Channel created successfully. The response carries the one-time plaintext credentials (api_key / webhook_secret / webhook_api_key / password) which are never returned again." + "description": "Channel created successfully. The response carries the one-time plaintext credentials (api_key / secret / password) which are never returned again." }, "400": { "description": "Invalid input data" @@ -3244,7 +3244,7 @@ } } }, - "description": "Rotates the channel's secret. Single-active: the previous credential is invalidated immediately. The response carries exactly one of ``api_key`` / ``webhook_secret`` / ``webhook_api_key`` matching the channel's type (and, for webhooks, its current ``auth_method``)." + "description": "Rotates the channel's secret. Single-active: the previous credential is invalidated immediately. The response carries exactly one of ``api_key`` / ``secret`` matching the channel's type (and, for webhooks, its current ``auth_method``)." }, "400": { "description": "Channel type has no rotatable secret" @@ -6944,7 +6944,7 @@ } } }, - "description": "Channel created successfully. The response carries the one-time plaintext credentials (api_key / webhook_secret / webhook_api_key / password) which are never returned again." + "description": "Channel created successfully. The response carries the one-time plaintext credentials (api_key / secret / password) which are never returned again." }, "400": { "description": "Invalid input data" @@ -7161,7 +7161,7 @@ } } }, - "description": "Rotates the channel's secret. Single-active: the previous credential is invalidated immediately. The response carries exactly one of ``api_key`` / ``webhook_secret`` / ``webhook_api_key`` matching the channel's type (and, for webhooks, its current ``auth_method``)." + "description": "Rotates the channel's secret. Single-active: the previous credential is invalidated immediately. The response carries exactly one of ``api_key`` / ``secret`` matching the channel's type (and, for webhooks, its current ``auth_method``)." }, "400": { "description": "Channel type has no rotatable secret" @@ -7580,19 +7580,15 @@ }, "api_key": { "type": "string", - "description": "api_key channels only — the plaintext API key." + "description": "Plaintext API key — api_key channels and webhook channels with auth_method=api_key." }, "password": { "type": "string", "description": "Plaintext password, when the channel type mints one." }, - "webhook_secret": { + "secret": { "type": "string", "description": "webhook channels with auth_method=jwt — the HMAC/JWT signing secret." - }, - "webhook_api_key": { - "type": "string", - "description": "webhook channels with auth_method=api_key — the derived API key." } }, "required": [ @@ -9816,15 +9812,11 @@ }, "api_key": { "type": "string", - "description": "Present for ``api_key`` channels — the plaintext used in subsequent X-API-Key calls. Returned ONCE." + "description": "Present for ``api_key`` channels and webhook channels with ``auth_method='api_key'`` — the plaintext API key presented in a request header (X-API-Key / X-StMsg-Api-Key). Returned ONCE; for api_key webhooks it changes whenever the root rotates." }, - "webhook_secret": { + "secret": { "type": "string", "description": "Present for webhook channels with ``auth_method='jwt'`` — the freshly minted root receivers use to verify the HMAC sig and JWT." - }, - "webhook_api_key": { - "type": "string", - "description": "Present for webhook channels with ``auth_method='api_key'`` — the HMAC-derived value sent as X-StMsg-Api-Key. Changes whenever the root rotates." } }, "required": [ diff --git a/src/backend/core/mda/dispatch_webhooks.py b/src/backend/core/mda/dispatch_webhooks.py index 45f8483e5..71f599875 100644 --- a/src/backend/core/mda/dispatch_webhooks.py +++ b/src/backend/core/mda/dispatch_webhooks.py @@ -557,7 +557,7 @@ class UserWebhookStep: A webhook error never drops the user's email — only an explicit ``{"action": "drop"}`` does. Every failure is held for retry, - bounded by the pipeline's 7-day budget. + bounded by the pipeline's 48-hour quarantine window. """ def __init__(self, channel: models.Channel, phase: str): @@ -674,8 +674,8 @@ def _post( return _failure(blocking, Decision.RETRY) except Exception as exc: # Timeout, connection refused, DNS, unknown transport-level - # failure: all transient on the blocking path. The 7-day cap - # in the pipeline runner bounds the retry budget. + # failure: all transient on the blocking path. The 48-hour + # quarantine window in the pipeline runner bounds the retries. logger.exception( "Webhook channel %s network error for url=%s: %s", self.channel.id, @@ -781,6 +781,16 @@ def webhook_steps_for_mailbox(mailbox: models.Mailbox, *, phase: str) -> List[St if cfg.get("phase", PHASE_AFTER_SPAM) != phase: continue events = cfg.get("events") or [enums.WebhookEvents.MESSAGE_INBOUND.value] + if not isinstance(events, list): + # Validator guarantees a list on write; a non-list here is a + # misconfigured row. Fail closed — a bare string would make the + # ``in`` check below match substrings instead of members. + logger.warning( + "Webhook channel %s has non-list events=%r — skipping", + channel.id, + events, + ) + continue if enums.WebhookEvents.MESSAGE_INBOUND.value not in events: continue if not cfg.get("url"): diff --git a/src/backend/core/tests/mda/test_spam_processing.py b/src/backend/core/tests/mda/test_spam_processing.py index bba106cbd..7a62e0c63 100644 --- a/src/backend/core/tests/mda/test_spam_processing.py +++ b/src/backend/core/tests/mda/test_spam_processing.py @@ -883,6 +883,7 @@ def _ctx(self, spam_config): @patch("core.mda.inbound_pipeline._call_rspamd") def test_error_holds_for_retry(self, mock_call): + """On rspamd error, never fail open — hold the message for retry.""" spam_config = {"rspamd_url": "http://rspamd:11334"} # _call_rspamd returns is_spam=False on error. mock_call.return_value = (False, "connection refused", None) @@ -894,6 +895,7 @@ def test_error_holds_for_retry(self, mock_call): @patch("core.mda.inbound_pipeline._call_rspamd") def test_not_configured_continues(self, mock_call): + """When rspamd isn't configured, continue without a verdict.""" # rspamd absent is "no opinion", not an error → keep moving. mock_call.return_value = (None, None, None) ctx = self._ctx({}) @@ -905,6 +907,7 @@ def test_not_configured_continues(self, mock_call): @patch("core.mda.inbound_pipeline._call_rspamd") def test_success_sets_verdict(self, mock_call): + """A successful rspamd call sets the spam verdict and continues.""" spam_config = {"rspamd_url": "http://rspamd:11334"} mock_call.return_value = (True, None, {"action": "reject"}) ctx = self._ctx(spam_config) From e86646f7ffb660dbd7dc8a78d1f96aefd17a8427 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Mon, 22 Jun 2026 01:41:22 +0200 Subject: [PATCH 06/21] global review --- env.d/development/backend.defaults | 8 +++- src/backend/core/api/openapi.json | 6 +++ src/backend/core/api/serializers.py | 18 ++++++++ src/backend/core/enums.py | 6 ++- src/backend/core/mda/dispatch_webhooks.py | 33 +++++++++++---- src/backend/core/mda/inbound_create.py | 6 +++ src/backend/core/mda/inbound_pipeline.py | 42 ++++++++++++++----- src/frontend/public/locales/common/en-US.json | 35 ++++++++++++++++ src/frontend/public/locales/common/fr-FR.json | 35 ++++++++++++++++ src/frontend/public/locales/common/nl-NL.json | 35 ++++++++++++++++ .../features/api/gen/models/thread_event.ts | 2 + .../webhook-integration-form.tsx | 12 +++--- .../thread-event/assignment-message.test.ts | 1 + .../thread-event/assignment-message.ts | 9 +++- .../thread-event/group-system-events.test.ts | 2 + .../components/thread-event/index.tsx | 4 +- 16 files changed, 224 insertions(+), 30 deletions(-) diff --git a/env.d/development/backend.defaults b/env.d/development/backend.defaults index 77d0d6fb1..c43633b79 100644 --- a/env.d/development/backend.defaults +++ b/env.d/development/backend.defaults @@ -94,8 +94,12 @@ MTA_OUT_RELAY_HOST=mailcatcher:1025 MDA_API_SECRET=my-shared-secret-mda SALT_KEY=ThisIsAnExampleSaltForDevPurposeOnly -# Rspamd -SPAM_CONFIG={"rspamd_url": "http://mpa:8010/_api", "rspamd_auth": ""} +# Rspamd spam-check — disabled by default. The `mpa` service is not started by +# `make start`, and a configured-but-unreachable rspamd holds inbound mail for +# retry (fail-closed), which would stall the local pipeline. To enable spam +# checks locally, start `mpa` and uncomment the line below. +SPAM_CONFIG={} +# SPAM_CONFIG={"rspamd_url": "http://mpa:8010/_api", "rspamd_auth": ""} # AI AI_BASE_URL= diff --git a/src/backend/core/api/openapi.json b/src/backend/core/api/openapi.json index 18b676faa..e2da56ec1 100644 --- a/src/backend/core/api/openapi.json +++ b/src/backend/core/api/openapi.json @@ -10388,6 +10388,11 @@ ], "readOnly": true }, + "author_display": { + "type": "string", + "nullable": true, + "readOnly": true + }, "data": { "$ref": "#/components/schemas/ThreadEventData" }, @@ -10416,6 +10421,7 @@ }, "required": [ "author", + "author_display", "channel", "created_at", "data", diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index 17534dce7..b94741c94 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -1304,6 +1304,7 @@ class ThreadEventSerializer(CreateOnlyFieldsMixin, serializers.ModelSerializer): """Serialize thread event information.""" author = UserWithoutAbilitiesSerializer(read_only=True) + author_display = serializers.SerializerMethodField() data = ThreadEventDataField() has_unread_mention = serializers.SerializerMethodField() is_editable = serializers.SerializerMethodField() @@ -1317,6 +1318,7 @@ class Meta: "channel", "message", "author", + "author_display", "data", "has_unread_mention", "is_editable", @@ -1355,6 +1357,22 @@ def validate(self, attrs): raise serializers.ValidationError(exc.message_dict) from exc return attrs + @extend_schema_field(serializers.CharField(allow_null=True)) + def get_author_display(self, obj): + """Human-readable author label for the UI. + + Human-authored events resolve to the user's name (or email). Events + created by a webhook have no ``author`` but carry the firing + ``channel``; surface it as ``Webhook: {name}`` so the timeline shows + the integration instead of an anonymous "Unknown". Returns ``None`` + when neither is available, letting the client fall back. + """ + if obj.author_id: + return obj.author.full_name or obj.author.email + if obj.channel_id and obj.channel.name: + return f"Webhook: {obj.channel.name}" + return None + @extend_schema_field(serializers.BooleanField()) def get_has_unread_mention(self, obj): """Return whether the event has an unread mention for the current user.""" diff --git a/src/backend/core/enums.py b/src/backend/core/enums.py index cfa16b923..922351e5f 100644 --- a/src/backend/core/enums.py +++ b/src/backend/core/enums.py @@ -229,7 +229,11 @@ class WebhookEvents(StrEnum): """ MESSAGE_INBOUND = "message.inbound" - MESSAGE_OUTBOUND = "message.outbound" + # MESSAGE_OUTBOUND ("message.outbound") is not wired yet — the dispatcher + # only ever fires on inbound. Kept commented so the serializer rejects it + # at write time instead of accepting a webhook that silently never fires. + # Uncomment once outbound dispatch is implemented. + # MESSAGE_OUTBOUND = "message.outbound" class ChannelApiKeyScope(models.TextChoices): diff --git a/src/backend/core/mda/dispatch_webhooks.py b/src/backend/core/mda/dispatch_webhooks.py index 71f599875..ccfaf20d5 100644 --- a/src/backend/core/mda/dispatch_webhooks.py +++ b/src/backend/core/mda/dispatch_webhooks.py @@ -184,9 +184,12 @@ def _classify_response_body(body_bytes: bytes) -> _HttpResult: return _HttpResult() try: # ``json.loads`` accepts bytes natively and raises ValueError - # (incl. JSONDecodeError) on anything malformed. + # (incl. JSONDecodeError) on anything malformed. Deeply-nested + # JSON raises RecursionError (a RuntimeError, not a ValueError), + # so catch it too — a misbehaving receiver must never escape this + # parser and stall the message on an uncaught exception. payload = json.loads(body_bytes) - except ValueError: + except (ValueError, RecursionError): return _HttpResult() if not isinstance(payload, dict): return _HttpResult() @@ -676,11 +679,15 @@ def _post( # Timeout, connection refused, DNS, unknown transport-level # failure: all transient on the blocking path. The 48-hour # quarantine window in the pipeline runner bounds the retries. - logger.exception( - "Webhook channel %s network error for url=%s: %s", + # Log only the exception *type*, not its message or traceback: + # requests/urllib3 errors embed the full request URL (path + + # query), which is exactly where receivers carry secret tokens, + # so ``exc``/``exc_info`` would bypass ``_sanitize_url``. + logger.warning( + "Webhook channel %s network error (%s) for url=%s", self.channel.id, + type(exc).__name__, _sanitize_url(url), - exc, ) return _failure(blocking, Decision.RETRY) @@ -765,18 +772,30 @@ def _build_auth_headers( return None -def webhook_steps_for_mailbox(mailbox: models.Mailbox, *, phase: str) -> List[Step]: +def webhook_steps_for_mailbox( + mailbox: models.Mailbox, + *, + phase: str, + channels: Optional[List[models.Channel]] = None, +) -> List[Step]: """Build one ``UserWebhookStep`` per matching channel for the phase. Channels are filtered here (phase, events, url present, valid format) rather than at run time so the pipeline iterator sees a flat list of ready-to-call steps. + + ``channels`` may be passed in to reuse a single channel-set query + across both phases (the set is identical before- and after-spam); + when omitted it is fetched here. """ if phase not in VALID_PHASES: raise ValueError(f"Invalid webhook phase: {phase}") + if channels is None: + channels = find_webhook_channels_for_mailbox(mailbox) + steps: List[Step] = [] - for channel in find_webhook_channels_for_mailbox(mailbox): + for channel in channels: cfg = channel.settings or {} if cfg.get("phase", PHASE_AFTER_SPAM) != phase: continue diff --git a/src/backend/core/mda/inbound_create.py b/src/backend/core/mda/inbound_create.py index d38c11341..09bf83364 100644 --- a/src/backend/core/mda/inbound_create.py +++ b/src/backend/core/mda/inbound_create.py @@ -435,7 +435,13 @@ def _create_message_from_inbound( # pylint: disable=too-many-arguments is_draft=is_outbound, # Outbound: draft until prepare_outbound_message finalizes is_sender=is_sender, is_trashed=is_trashed, + # Keep timestamps in lockstep with the booleans, as the + # flag endpoint does — a NULL trashed_at/archived_at on a + # trashed/archived row breaks restore, ordering and any + # auto-purge that keys off the timestamp. + trashed_at=(timezone.now() if is_trashed else None), is_archived=is_archived, + archived_at=(timezone.now() if is_archived else None), is_spam=is_spam, has_attachments=len(parsed_email.get("attachments", [])) > 0, channel=channel, diff --git a/src/backend/core/mda/inbound_pipeline.py b/src/backend/core/mda/inbound_pipeline.py index 04df6d883..0afbfdee2 100644 --- a/src/backend/core/mda/inbound_pipeline.py +++ b/src/backend/core/mda/inbound_pipeline.py @@ -386,15 +386,24 @@ def build_inbound_pipeline(ctx: InboundContext) -> List[Step]: # cycle: webhook_steps_for_mailbox lives next to UserWebhookStep # because it instantiates one per matching channel. from core.mda.dispatch_webhooks import ( # pylint: disable=import-outside-toplevel + find_webhook_channels_for_mailbox, webhook_steps_for_mailbox, ) + # Fetch the channel set once and reuse it for both phases — it's + # identical before- and after-spam, so a second query is pure waste. + channels = find_webhook_channels_for_mailbox(ctx.mailbox) + return [ - *webhook_steps_for_mailbox(ctx.mailbox, phase="before_spam"), + *webhook_steps_for_mailbox( + ctx.mailbox, phase="before_spam", channels=channels + ), _make_hardcoded_rules_step(ctx.spam_config), _make_rspamd_step(ctx.spam_config), _make_inbound_auth_step(ctx.spam_config), - *webhook_steps_for_mailbox(ctx.mailbox, phase="after_spam"), + *webhook_steps_for_mailbox( + ctx.mailbox, phase="after_spam", channels=channels + ), ] @@ -561,6 +570,17 @@ def apply_thread_access_flags( access.save(update_fields=update_fields) +def _channels_by_id(channel_ids: List[Any]) -> Dict[Any, models.Channel]: + """Resolve a batch of pending-action channel ids in a single query. + + The deferred-apply helpers below each carry ``(channel_id, …)`` tuples; + resolving them one ``.get()`` at a time is an N+1 on the finalize hot + path. A channel absent from the returned map vanished mid-processing + (admin churn between dispatch and finalize) and the caller skips it. + """ + return {c.id: c for c in models.Channel.objects.filter(id__in=set(channel_ids))} + + def apply_pending_drafts( inbound_msg: models.Message, mailbox: models.Mailbox, @@ -584,6 +604,7 @@ def apply_pending_drafts( create_draft_reply_from_template, ) + channels = _channels_by_id([cid for cid, _ in pending]) for channel_id, template_id in pending: template = ( models.MessageTemplate.objects.filter( @@ -603,9 +624,8 @@ def apply_pending_drafts( mailbox.id, ) continue - try: - channel = models.Channel.objects.get(id=channel_id) - except models.Channel.DoesNotExist: + channel = channels.get(channel_id) + if channel is None: logger.warning( "Webhook channel %s vanished before reply_draft could land — skipping", channel_id, @@ -629,14 +649,14 @@ def apply_pending_events( (the classifier dropped unknown types); future types just need their dispatch case added without touching the contract. """ + channels = _channels_by_id([cid for cid, _ in pending]) for channel_id, event in pending: event_type = event.get("type") if event_type != enums.ThreadEventTypeChoices.IM: logger.warning("Unknown pending event type %r — skipping", event_type) continue - try: - channel = models.Channel.objects.get(id=channel_id) - except models.Channel.DoesNotExist: + channel = channels.get(channel_id) + if channel is None: logger.warning( "Webhook channel %s vanished before event could land — skipping", channel_id, @@ -665,13 +685,13 @@ def apply_pending_assigns( later webhook re-asking for an already-assigned user, so the first-to-ask is the canonical attribution. """ + channels = _channels_by_id([cid for cid, _ in pending]) for channel_id, emails in pending: assignees_data = _resolve_assignable_users(thread, emails) if not assignees_data: continue - try: - channel = models.Channel.objects.get(id=channel_id) - except models.Channel.DoesNotExist: + channel = channels.get(channel_id) + if channel is None: # The webhook channel was deleted between dispatch and # finalize (admin churn during processing). Skip the # assign rather than half-attribute it to a dead row. diff --git a/src/frontend/public/locales/common/en-US.json b/src/frontend/public/locales/common/en-US.json index bf8a2bc0c..114efa0ac 100755 --- a/src/frontend/public/locales/common/en-US.json +++ b/src/frontend/public/locales/common/en-US.json @@ -176,6 +176,7 @@ "Address shared between {{count}} members_other": "Address shared between {{count}} members", "Addresses": "Addresses", "After creating the widget, you will receive the installation code to add to your website.": "After creating the widget, you will receive the installation code to add to your website.", + "After spam check (recommended)": "After spam check (recommended)", "All messages": "All messages", "All settings": "All settings", "All users with access to the mailbox \"{{mailboxName}}\" will no longer see this thread.": "All users with access to the mailbox \"{{mailboxName}}\" will no longer see this thread.", @@ -195,6 +196,7 @@ "and {{count}} other users_one": "and 1 other user", "and {{count}} other users_other": "and {{count}} other users", "API Key": "API Key", + "API key in header — for receivers that can only check a static header value": "API key in header — for receivers that can only check a static header value", "Archive": "Archive", "Archives": "Archives", "Are you sure you want to close this dialog? Your upload will be aborted!": "Are you sure you want to close this dialog? Your upload will be aborted!", @@ -223,6 +225,7 @@ "Attachment size limit exceeded": "Attachment size limit exceeded", "Attachments": "Attachments", "Attachments must be less than {{size}}.": "Attachments must be less than {{size}}.", + "Authentication": "Authentication", "Authentication failed. Please check your credentials and ensure you have enabled IMAP connections in your account.": "Authentication failed. Please check your credentials and ensure you have enabled IMAP connections in your account.", "Auto-labeling": "Auto-labeling", "Auto-replies": "Auto-replies", @@ -236,7 +239,13 @@ "Back to your inbox": "Back to your inbox", "bcc": "bcc", "BCC: ": "BCC: ", + "Before spam check": "Before spam check", + "Behavior": "Behavior", "Blind copy: ": "Blind copy: ", + "Blocking — let this endpoint shape what happens to the message": "Blocking — let this endpoint shape what happens to the message", + "Blocking lets the receiver act on this single message": "Blocking lets the receiver act on this single message", + "Body posted to the endpoint. Envelope metadata is always sent as X-StMsg-* headers.": "Body posted to the endpoint. Envelope metadata is always sent as X-StMsg-* headers.", + "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.": "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.", "Calendar invite": "Calendar invite", "Calendar service unavailable": "Calendar service unavailable", "Cancel": "Cancel", @@ -292,6 +301,7 @@ "Create a new signature for {{domain}}": "Create a new signature for {{domain}}", "Create a new template": "Create a new template", "Create a simple redirect (Coming soon)": "Create a simple redirect (Coming soon)", + "Create a Webhook": "Create a Webhook", "Create a Widget": "Create a Widget", "Create integration": "Create integration", "Create one": "Create one", @@ -337,6 +347,7 @@ "Domain": "Domain", "Domain admin": "Domain admin", "Domain not found": "Domain not found", + "Done": "Done", "Download": "Download", "Download invitation": "Download invitation", "Download raw email": "Download raw email", @@ -348,11 +359,13 @@ "Drag and drop an archive here": "Drag and drop an archive here", "Drop your attachments here": "Drop your attachments here", "Duplicate": "Duplicate", + "Each incoming message will be POSTed to this URL in the format selected below.": "Each incoming message will be POSTed to this URL in the format selected below.", "Edit": "Edit", "Edit {{mailbox}} address": "Edit {{mailbox}} address", "Edit auto-reply \"{{autoreply}}\"": "Edit auto-reply \"{{autoreply}}\"", "Edit signature \"{{signature}}\"": "Edit signature \"{{signature}}\"", "Edit template \"{{template}}\"": "Edit template \"{{template}}\"", + "Edit Webhook": "Edit Webhook", "Edit Widget": "Edit Widget", "edited": "edited", "Editing message": "Editing message", @@ -364,6 +377,7 @@ "End day is required": "End day is required", "End time": "End time", "End time is required": "End time is required", + "Endpoint": "Endpoint", "Enter the email addresses of the recipients separated by commas": "Enter the email addresses of the recipients separated by commas", "Error while checking DNS records": "Error while checking DNS records", "Error while loading addresses": "Error while loading addresses", @@ -411,6 +425,7 @@ "Forced": "Forced", "Forced signature": "Forced signature", "Forward": "Forward", + "Forward every incoming message to a URL of your choice as a JSON POST.": "Forward every incoming message to a URL of your choice as a JSON POST.", "Forwarded message": "Forwarded message", "Friday": "Friday", "From": "From", @@ -424,6 +439,7 @@ "Grant editor access to the thread?": "Grant editor access to the thread?", "Help center & Support": "Help center & Support", "High failure rate": "High failure rate", + "How the receiver authenticates our requests. The secret is shown once at creation.": "How the receiver authenticates our requests. The secret is shown once at creation.", "How to allow IMAP connections from your account {{name}}?": "How to allow IMAP connections from your account {{name}}?", "I confirm that this address corresponds to the real identity of a colleague, and I commit to deactivating it when their position ends.": "I confirm that this address corresponds to the real identity of a colleague, and I commit to deactivating it when their position ends.", "I have an issue or a feature request": "I have an issue or a feature request", @@ -460,6 +476,8 @@ "Integration updated!": "Integration updated!", "Integrations": "Integrations", "Its actual content doesn't match its declared type. Open it only if you trust the sender.": "Its actual content doesn't match its declared type. Open it only if you trust the sender.", + "JMAP Email JSON (RFC 8621)": "JMAP Email JSON (RFC 8621)", + "JMAP Email metadata (notification only)": "JMAP Email metadata (notification only)", "just now": "just now", "Label \"{{label}}\" assigned and {{count}} threads archived._one": "Label \"{{label}}\" assigned and thread archived.", "Label \"{{label}}\" assigned and {{count}} threads archived._other": "Label \"{{label}}\" assigned and {{count}} threads archived.", @@ -583,6 +601,7 @@ "No thread could be starred.": "No thread could be starred.", "No threads": "No threads", "No threads match the active filters": "No threads match the active filters", + "Non-blocking (default) is the safe choice:": "Non-blocking (default) is the safe choice:", "OK": "OK", "Older": "Older", "On going": "On going", @@ -596,10 +615,12 @@ "or drag and drop some files": "or drag and drop some files", "Organizer": "Organizer", "Other services...": "Other services...", + "Outbound Webhook": "Outbound Webhook", "Outbox": "Outbox", "Password": "Password", "Password is required.": "Password is required.", "Password reset successfully!": "Password reset successfully!", + "Payload format": "Payload format", "Personal mailbox": "Personal mailbox", "Pick a calendar that matches one of the invitees to respond.": "Pick a calendar that matches one of the invitees to respond.", "Please enter a valid email address.": "Please enter a valid email address.", @@ -608,6 +629,7 @@ "Preview {{name}}": "Preview {{name}}", "Print": "Print", "Provenance": "Provenance", + "Raw .eml (message/rfc822)": "Raw .eml (message/rfc822)", "Read": "Read", "Read state": "Read state", "Read-only": "Read-only", @@ -640,6 +662,7 @@ "Save failed — retry": "Save failed — retry", "Save in {{driveAppName}}": "Save in {{driveAppName}}", "Save into your {{driveAppName}}'s workspace": "Save into your {{driveAppName}}'s workspace", + "Save this credential now": "Save this credential now", "Saving...": "Saving...", "Schedule": "Schedule", "Scheduled": "Scheduled", @@ -692,6 +715,7 @@ "Signature deleted!": "Signature deleted!", "Signature updated!": "Signature updated!", "Signatures": "Signatures", + "Signed (HMAC + JWT) — recommended for receivers that can verify a signature": "Signed (HMAC + JWT) — recommended for receivers that can verify a signature", "Simple and intuitive messaging": "Simple and intuitive messaging", "Simple redirect (Coming soon)": "Simple redirect (Coming soon)", "Skip to main content": "Skip to main content", @@ -775,6 +799,7 @@ "This signature is forced": "This signature is forced", "This thread has been reported as spam.": "This thread has been reported as spam.", "This thread has been reported as spam. For your security, previewing and downloading attachments has been disabled.": "This thread has been reported as spam. For your security, previewing and downloading attachments has been disabled.", + "This value is shown only once. Configure your receiver with it before closing — you can rotate it later if you need a new one.": "This value is shown only once. Configure your receiver with it before closing — you can rotate it later if you need a new one.", "This week": "This week", "This will move this message and all following messages to a new thread. Continue?": "This will move this message and all following messages to a new thread. Continue?", "Thread access removed": "Thread access removed", @@ -816,6 +841,9 @@ "Upload an archive": "Upload an archive", "Uploading your archive": "Uploading your archive", "Uploading... {{progress}}%": "Uploading... {{progress}}%", + "URL": "URL", + "URL is required.": "URL is required.", + "URL must start with http:// or https://": "URL must start with http:// or https://", "Use \"Send and archive\" by default": "Use \"Send and archive\" by default", "Use {referer_domain} to include the website domain in the subject.": "Use {referer_domain} to include the website domain in the subject.", "Use SSL": "Use SSL", @@ -824,9 +852,15 @@ "Variables": "Variables", "View full documentation": "View full documentation", "Visit the Help center": "Visit the Help center", + "we POST and ignore the response. Receivers cannot affect the message.": "we POST and ignore the response. Receivers cannot affect the message.", + "Webhook": "Webhook", + "Webhook API key": "Webhook API key", + "Webhook signing secret": "Webhook signing secret", "Website Widget": "Website Widget", "Wednesday": "Wednesday", "Weekly": "Weekly", + "When to fire": "When to fire", + "Whether the webhook fires before or after the message is checked for spam.": "Whether the webhook fires before or after the message is checked for spam.", "While the auto-reply is disabled, it will not be sent.": "While the auto-reply is disabled, it will not be sent.", "While the signature is disabled, it will not be available to the users.": "While the signature is disabled, it will not be available to the users.", "Widget": "Widget", @@ -863,5 +897,6 @@ "You were unassigned": "You were unassigned", "You will no longer have access to the mailbox \"{{mailboxName}}\".": "You will no longer have access to the mailbox \"{{mailboxName}}\".", "Your email...": "Your email...", + "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).": "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).", "Your session has expired. Please log in again.": "Your session has expired. Please log in again." } diff --git a/src/frontend/public/locales/common/fr-FR.json b/src/frontend/public/locales/common/fr-FR.json index 8203014aa..793a44616 100755 --- a/src/frontend/public/locales/common/fr-FR.json +++ b/src/frontend/public/locales/common/fr-FR.json @@ -246,6 +246,7 @@ "Address shared between {{count}} members_other": "Adresse partagée entre {{count}} membres", "Addresses": "Adresses", "After creating the widget, you will receive the installation code to add to your website.": "Après avoir créé le widget, vous recevrez le code d'installation à ajouter à votre site web.", + "After spam check (recommended)": "Après le contrôle anti-spam (recommandé)", "All messages": "Tous les messages", "All settings": "Tous les paramètres", "All users with access to the mailbox \"{{mailboxName}}\" will no longer see this thread.": "Tous les utilisateurs avec un accès à la boîte \"{{mailboxName}}\" ne verront plus ce thread.", @@ -266,6 +267,7 @@ "and {{count}} other users_many": "et {{count}} autres utilisateurs", "and {{count}} other users_other": "et {{count}} autres utilisateurs", "API Key": "Clé API", + "API key in header — for receivers that can only check a static header value": "Clé API dans l'en-tête — pour les destinataires qui ne peuvent vérifier qu'une valeur d'en-tête statique", "Archive": "Archiver", "Archives": "Archives", "Are you sure you want to close this dialog? Your upload will be aborted!": "Êtes-vous sûr de vouloir fermer cette boîte de dialogue ? Votre téléversement sera annulé !", @@ -295,6 +297,7 @@ "Attachment size limit exceeded": "La taille des pièces jointes excède la limite autorisée.", "Attachments": "Pièces jointes", "Attachments must be less than {{size}}.": "Les pièces jointes ne peuvent pas excéder {{size}}.", + "Authentication": "Authentification", "Authentication failed. Please check your credentials and ensure you have enabled IMAP connections in your account.": "L'authentification a échoué. Veuillez vérifier vos identifiants et assurez-vous d'avoir autorisé les connexions IMAP dans votre compte.", "Auto-labeling": "Labellisation automatique", "Auto-replies": "Réponses automatiques", @@ -308,7 +311,13 @@ "Back to your inbox": "Retour à votre messagerie", "bcc": "cci", "BCC: ": "CCI : ", + "Before spam check": "Avant le contrôle anti-spam", + "Behavior": "Comportement", "Blind copy: ": "Copie cachée : ", + "Blocking — let this endpoint shape what happens to the message": "Bloquant — laissez ce point de terminaison déterminer le sort du message", + "Blocking lets the receiver act on this single message": "Le mode bloquant permet au destinataire d'agir sur ce message précis", + "Body posted to the endpoint. Envelope metadata is always sent as X-StMsg-* headers.": "Corps envoyé au point de terminaison. Les métadonnées d'enveloppe sont toujours transmises sous forme d'en-têtes X-StMsg-*.", + "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.": "en renvoyant un corps JSON. Actions disponibles (toutes limitées au message en cours de réception) : abandonner / réessayer la distribution, remplacer le verdict anti-spam, ajouter des libellés, assigner des utilisateurs par e-mail, marquer comme suivi / lu / supprimé / archivé, désactiver la réponse automatique, ajouter un commentaire interne au fil de discussion et créer un brouillon de réponse à partir d'un modèle. N'utilisez le mode bloquant qu'avec des destinataires de confiance.", "Calendar invite": "Invitation calendrier", "Calendar service unavailable": "Service d'agenda indisponible", "Cancel": "Annuler", @@ -364,6 +373,7 @@ "Create a new signature for {{domain}}": "Création d'une nouvelle signature pour {{domain}}", "Create a new template": "Créer un nouveau modèle", "Create a simple redirect (Coming soon)": "Créer une simple redirection (Bientôt disponible)", + "Create a Webhook": "Créer un Webhook", "Create a Widget": "Créer un Widget", "Create integration": "Créer l'intégration", "Create one": "En créer un", @@ -409,6 +419,7 @@ "Domain": "Domaine", "Domain admin": "Gestion des domaines", "Domain not found": "Domaine introuvable", + "Done": "Terminé", "Download": "Télécharger", "Download invitation": "Télécharger l'invitation", "Download raw email": "Télécharger l'email brut", @@ -420,11 +431,13 @@ "Drag and drop an archive here": "Glissez-déposez une archive ici", "Drop your attachments here": "Déposez vos pièces jointes ici", "Duplicate": "Dupliqué", + "Each incoming message will be POSTed to this URL in the format selected below.": "Chaque message entrant sera envoyé via POST à cette URL au format sélectionné ci-dessous.", "Edit": "Modifier", "Edit {{mailbox}} address": "Modifier l'adresse {{mailbox}}", "Edit auto-reply \"{{autoreply}}\"": "Modifier la réponse automatique \"{{autoreply}}\"", "Edit signature \"{{signature}}\"": "Modifier la signature \"{{signature}}\"", "Edit template \"{{template}}\"": "Modifier le modèle \"{{template}}\"", + "Edit Webhook": "Modifier le Webhook", "Edit Widget": "Modifier le Widget", "edited": "modifié", "Editing message": "Modification du message", @@ -436,6 +449,7 @@ "End day is required": "Le jour de fin est requis", "End time": "Heure de fin", "End time is required": "L'heure de fin est requise", + "Endpoint": "Point de terminaison", "Enter the email addresses of the recipients separated by commas": "Entrez les adresses e-mail des destinataires séparés par des virgules", "Error while checking DNS records": "Erreur lors de la vérification des enregistrements DNS", "Error while loading addresses": "Erreur lors du chargement des adresses", @@ -488,6 +502,7 @@ "Forced": "Forcée", "Forced signature": "Signature forcée", "Forward": "Transférer", + "Forward every incoming message to a URL of your choice as a JSON POST.": "Transférez chaque message entrant vers une URL de votre choix via un POST JSON.", "Forwarded message": "Message transféré", "Friday": "Vendredi", "From": "De", @@ -501,6 +516,7 @@ "Grant editor access to the thread?": "Accorder l'accès en édition à la conversation ?", "Help center & Support": "Centre d'aide et Support", "High failure rate": "Taux d'échec élevé", + "How the receiver authenticates our requests. The secret is shown once at creation.": "Comment le destinataire authentifie nos requêtes. Le secret est affiché une seule fois lors de la création.", "How to allow IMAP connections from your account {{name}}?": "Comment autoriser les connexions IMAP depuis votre compte {{name}} ?", "I confirm that this address corresponds to the real identity of a colleague, and I commit to deactivating it when their position ends.": "Je confirme que cette adresse correspond à l'identité d'une personne physique travaillant avec moi, et m'engage à la désactiver quand son poste prendra fin.", "I have an issue or a feature request": "J'ai un problème ou une demande d'amélioration", @@ -538,6 +554,8 @@ "Integration updated!": "Intégration mise à jour !", "Integrations": "Intégrations", "Its actual content doesn't match its declared type. Open it only if you trust the sender.": "Son contenu réel ne correspond pas au type déclaré. Ne l'ouvrez que si vous avez confiance en l'expéditeur.", + "JMAP Email JSON (RFC 8621)": "JMAP Email JSON (RFC 8621)", + "JMAP Email metadata (notification only)": "Métadonnées JMAP Email (notification uniquement)", "just now": "à l'instant", "Label \"{{label}}\" assigned and {{count}} threads archived._one": "Libellé \"{{label}}\" assigné et conversation archivée.", "Label \"{{label}}\" assigned and {{count}} threads archived._many": "Libellé \"{{label}}\" assigné et {{count}} conversations archivées.", @@ -665,6 +683,7 @@ "No thread could be starred.": "Aucune conversation n'a pu être suivie.", "No threads": "Aucune conversation", "No threads match the active filters": "Aucune conversation ne correspond aux filtres actifs", + "Non-blocking (default) is the safe choice:": "Le mode non bloquant (par défaut) est le choix sûr :", "OK": "OK", "Older": "Plus ancien", "On going": "En cours", @@ -678,10 +697,12 @@ "or drag and drop some files": "ou glissez-déposez des fichiers", "Organizer": "Organisateur", "Other services...": "Autres services...", + "Outbound Webhook": "Webhook sortant", "Outbox": "Boîte d'envoi", "Password": "Mot de passe", "Password is required.": "Le mot de passe est requis.", "Password reset successfully!": "Mot de passe réinitialisé avec succès !", + "Payload format": "Format de la charge utile", "Personal mailbox": "Boîte personnelle", "Pick a calendar that matches one of the invitees to respond.": "Sélectionnez un calendrier qui correspond à l'un des invités à répondre.", "Please enter a valid email address.": "Veuillez saisir une adresse email valide.", @@ -690,6 +711,7 @@ "Preview {{name}}": "Aperçu de {{name}}", "Print": "Imprimer", "Provenance": "Provenance", + "Raw .eml (message/rfc822)": ".eml brut (message/rfc822)", "Read": "Lu", "Read state": "État de lecture", "Read-only": "Lecture seule", @@ -722,6 +744,7 @@ "Save failed — retry": "Échec de l'enregistrement - réessayez", "Save in {{driveAppName}}": "Enregistrer dans {{driveAppName}}", "Save into your {{driveAppName}}'s workspace": "Enregistrer dans votre espace de travail {{driveAppName}}", + "Save this credential now": "Enregistrez cet identifiant maintenant", "Saving...": "Enregistrement en cours...", "Schedule": "Planification", "Scheduled": "Planifiée", @@ -777,6 +800,7 @@ "Signature deleted!": "Signature supprimée !", "Signature updated!": "Signature mise à jour !", "Signatures": "Signatures", + "Signed (HMAC + JWT) — recommended for receivers that can verify a signature": "Signé (HMAC + JWT) — recommandé pour les destinataires capables de vérifier une signature", "Simple and intuitive messaging": "Une messagerie simple et intuitive", "Simple redirect (Coming soon)": "Créer une simple redirection (Bientôt disponible)", "Skip to main content": "Aller au contenu principal", @@ -862,6 +886,7 @@ "This signature is forced": "Cette signature est forcée", "This thread has been reported as spam.": "Cette conversation a été signalée comme spam.", "This thread has been reported as spam. For your security, previewing and downloading attachments has been disabled.": "Cette conversation a été signalée comme spam. Pour votre sécurité, la prévisualisation et le téléchargement des pièces jointes ont été désactivés.", + "This value is shown only once. Configure your receiver with it before closing — you can rotate it later if you need a new one.": "Cette valeur n'est affichée qu'une seule fois. Configurez votre destinataire avec celle-ci avant de fermer — vous pourrez la renouveler plus tard si vous avez besoin d'une nouvelle.", "This week": "Cette semaine", "This will move this message and all following messages to a new thread. Continue?": "Cela déplacera ce message et tous les messages suivants dans une nouvelle conversation. Continuer ?", "Thread access removed": "Accès à la conversation supprimé", @@ -903,6 +928,9 @@ "Upload an archive": "Téléversez une archive", "Uploading your archive": "Téléversement de votre archive", "Uploading... {{progress}}%": "Téléversement en cours... {{progress}}%", + "URL": "URL", + "URL is required.": "L'URL est obligatoire.", + "URL must start with http:// or https://": "L'URL doit commencer par http:// ou https://", "Use \"Send and archive\" by default": "Utiliser \"Envoyer et archiver\" par défaut", "Use {referer_domain} to include the website domain in the subject.": "Utilisez {referer_domain} pour inclure le domaine du site web dans l'objet.", "Use SSL": "Utiliser SSL", @@ -911,9 +939,15 @@ "Variables": "Variables", "View full documentation": "Voir la documentation complète", "Visit the Help center": "Consulter le centre d'aide", + "we POST and ignore the response. Receivers cannot affect the message.": "nous envoyons un POST et ignorons la réponse. Les destinataires ne peuvent pas agir sur le message.", + "Webhook": "Webhook", + "Webhook API key": "Clé API du Webhook", + "Webhook signing secret": "Secret de signature du Webhook", "Website Widget": "Widget de site web", "Wednesday": "Mercredi", "Weekly": "Hebdomadaire", + "When to fire": "Quand déclencher", + "Whether the webhook fires before or after the message is checked for spam.": "Indique si le Webhook se déclenche avant ou après le contrôle anti-spam du message.", "While the auto-reply is disabled, it will not be sent.": "Tant que la réponse automatique est désactivée, elle ne sera pas envoyée.", "While the signature is disabled, it will not be available to the users.": "Tant que la signature est désactivée, elle ne sera pas disponible pour les utilisateurs.", "Widget": "Widget", @@ -956,5 +990,6 @@ "You were unassigned": "Vous avez été désassigné·e", "You will no longer have access to the mailbox \"{{mailboxName}}\".": "Vous n'aurez plus accès à la boîte aux lettres « {{mailboxName}} ».", "Your email...": "Renseigner votre email...", + "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).": "Votre point de terminaison recevra une charge utile JSON contenant le message analysé (expéditeur, destinataire, objet, corps, en-têtes, …).", "Your session has expired. Please log in again.": "Votre session a expiré. Veuillez vous reconnecter." } diff --git a/src/frontend/public/locales/common/nl-NL.json b/src/frontend/public/locales/common/nl-NL.json index 82d1cc7cb..c714628e6 100644 --- a/src/frontend/public/locales/common/nl-NL.json +++ b/src/frontend/public/locales/common/nl-NL.json @@ -1,4 +1,39 @@ { + "After spam check (recommended)": "Na spamcontrole (aanbevolen)", + "API key in header — for receivers that can only check a static header value": "API-sleutel in header — voor ontvangers die alleen een statische headerwaarde kunnen controleren", + "Authentication": "Authenticatie", + "Before spam check": "Voor spamcontrole", + "Behavior": "Gedrag", + "Blocking lets the receiver act on this single message": "Blokkerend laat de ontvanger ingrijpen op dit ene bericht", + "Blocking — let this endpoint shape what happens to the message": "Blokkerend — laat dit endpoint bepalen wat er met het bericht gebeurt", + "Body posted to the endpoint. Envelope metadata is always sent as X-StMsg-* headers.": "Body die naar het endpoint wordt verzonden. Envelopmetadata wordt altijd verzonden als X-StMsg-* headers.", + "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.": "door een JSON-body terug te sturen. Beschikbare acties (allemaal beperkt tot het ontvangen bericht): de bezorging weigeren / opnieuw proberen, het spamoordeel overschrijven, labels toevoegen, gebruikers toewijzen op e-mailadres, markeren als met ster / gelezen / verwijderd / gearchiveerd, het automatisch antwoord onderdrukken, een interne opmerking aan het gesprek toevoegen en een conceptantwoord vanuit een sjabloon maken. Gebruik blokkerend alleen bij ontvangers die je vertrouwt.", + "Create a Webhook": "Een Webhook maken", + "Done": "Klaar", + "Each incoming message will be POSTed to this URL in the format selected below.": "Elk binnenkomend bericht wordt via POST naar deze URL verzonden in het hieronder geselecteerde formaat.", + "Edit Webhook": "Webhook bewerken", + "Endpoint": "Endpoint", + "Forward every incoming message to a URL of your choice as a JSON POST.": "Stuur elk binnenkomend bericht door naar een URL naar keuze als een JSON POST.", + "How the receiver authenticates our requests. The secret is shown once at creation.": "Hoe de ontvanger onze verzoeken authenticeert. Het geheim wordt eenmalig getoond bij het aanmaken.", + "JMAP Email JSON (RFC 8621)": "JMAP Email JSON (RFC 8621)", + "JMAP Email metadata (notification only)": "JMAP Email-metadata (alleen melding)", + "Non-blocking (default) is the safe choice:": "Niet-blokkerend (standaard) is de veilige keuze:", + "Outbound Webhook": "Uitgaande webhook", + "Payload format": "Payloadformaat", + "Raw .eml (message/rfc822)": "Onbewerkte .eml (message/rfc822)", + "Save this credential now": "Sla deze inloggegevens nu op", + "Signed (HMAC + JWT) — recommended for receivers that can verify a signature": "Ondertekend (HMAC + JWT) — aanbevolen voor ontvangers die een handtekening kunnen verifiëren", + "This value is shown only once. Configure your receiver with it before closing — you can rotate it later if you need a new one.": "Deze waarde wordt slechts eenmaal getoond. Configureer je ontvanger ermee voordat je sluit — je kunt deze later roteren als je een nieuwe nodig hebt.", + "URL": "URL", + "URL is required.": "URL is vereist.", + "URL must start with http:// or https://": "URL moet beginnen met http:// of https://", + "we POST and ignore the response. Receivers cannot affect the message.": "verzenden we via POST en negeren we het antwoord. Ontvangers kunnen het bericht niet beïnvloeden.", + "Webhook": "Webhook", + "Webhook API key": "Webhook API-sleutel", + "Webhook signing secret": "Webhook-ondertekeningsgeheim", + "When to fire": "Wanneer activeren", + "Whether the webhook fires before or after the message is checked for spam.": "Of de webhook wordt geactiveerd voordat of nadat het bericht op spam is gecontroleerd.", + "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).": "Je endpoint ontvangt een JSON-payload met het geparseerde bericht (from, to, subject, body, headers, …).", "{{count}} attachments_one": "{{count}} bijlage", "{{count}} attachments_other": "{{count}} bijlagen", "{{count}} days ago_one": "{{count}} dag geleden", diff --git a/src/frontend/src/features/api/gen/models/thread_event.ts b/src/frontend/src/features/api/gen/models/thread_event.ts index 612133881..792f3b585 100644 --- a/src/frontend/src/features/api/gen/models/thread_event.ts +++ b/src/frontend/src/features/api/gen/models/thread_event.ts @@ -29,6 +29,8 @@ export interface ThreadEvent { */ message?: string | null; readonly author: UserWithoutAbilities; + /** @nullable */ + readonly author_display: string | null; data: ThreadEventData; readonly has_unread_mention: boolean; readonly is_editable: boolean; diff --git a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx index 04454155e..8c68b92cb 100644 --- a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx +++ b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx @@ -1,4 +1,4 @@ -import { Button, Input } from "@gouvfr-lasuite/cunningham-react"; +import { Button } from "@gouvfr-lasuite/cunningham-react"; import { Icon, IconType } from "@gouvfr-lasuite/ui-kit"; import { useTranslation } from "react-i18next"; import { useForm, FormProvider } from "react-hook-form"; @@ -20,6 +20,7 @@ import { } from "@/features/forms/components/react-hook-form"; import { addToast, ToasterItem } from "@/features/ui/components/toaster"; import { Banner } from "@/features/ui/components/banner"; +import { CopyableInput } from "@/features/ui/components/copyable-input"; import { handle } from "@/features/utils/errors"; type WebhookChannelSettings = { @@ -193,11 +194,12 @@ export const WebhookIntegrationForm = ({ "This value is shown only once. Configure your receiver with it before closing — you can rotate it later if you need a new one.", )} - + {createdCredential.label} + +
diff --git a/src/frontend/src/features/layouts/components/thread-view/components/thread-event/assignment-message.test.ts b/src/frontend/src/features/layouts/components/thread-view/components/thread-event/assignment-message.test.ts index e5f9be8e8..45ad78752 100644 --- a/src/frontend/src/features/layouts/components/thread-view/components/thread-event/assignment-message.test.ts +++ b/src/frontend/src/features/layouts/components/thread-view/components/thread-event/assignment-message.test.ts @@ -27,6 +27,7 @@ const makeEvent = ( author: authorId === null ? (null as unknown as ThreadEvent['author']) : ({ id: authorId, full_name: `User ${authorId}`, email: `${authorId}@ex.com` } as ThreadEvent['author']), + author_display: authorId === null ? null : `User ${authorId}`, data: { assignees }, has_unread_mention: false, is_editable: false, diff --git a/src/frontend/src/features/layouts/components/thread-view/components/thread-event/assignment-message.ts b/src/frontend/src/features/layouts/components/thread-view/components/thread-event/assignment-message.ts index 15f77bfe2..81003b445 100644 --- a/src/frontend/src/features/layouts/components/thread-view/components/thread-event/assignment-message.ts +++ b/src/frontend/src/features/layouts/components/thread-view/components/thread-event/assignment-message.ts @@ -40,9 +40,14 @@ export const buildAssignmentMessage = ( const isAssign = event.type === ThreadEventTypeEnum.assign; const assignees: Assignee[] = event.data.assignees ?? []; const authorId = event.author?.id; - const authorName = event.author?.full_name || event.author?.email || t("Unknown"); + // Prefer the human author's name/email; for channel-driven events (no + // author) fall back to ``author_display`` ("Webhook: {name}"), then Unknown. + const authorName = event.author?.full_name || event.author?.email || event.author_display || t("Unknown"); const isAuthorSelf = !!currentUserId && authorId === currentUserId; - const isSystem = event.author === null; + // A true system event has no actor at all. A webhook event also has a null + // ``author`` but carries ``author_display`` — treat it as a named third + // party, not as the passive "was unassigned" system wording. + const isSystem = event.author === null && !event.author_display; const selfInAssignees = !!currentUserId && assignees.some((a) => a.id === currentUserId); // Author appears among assignees AND the viewer is not the author — that's diff --git a/src/frontend/src/features/layouts/components/thread-view/components/thread-event/group-system-events.test.ts b/src/frontend/src/features/layouts/components/thread-view/components/thread-event/group-system-events.test.ts index fcf15ccd8..824a1b6e7 100644 --- a/src/frontend/src/features/layouts/components/thread-view/components/thread-event/group-system-events.test.ts +++ b/src/frontend/src/features/layouts/components/thread-view/components/thread-event/group-system-events.test.ts @@ -17,6 +17,7 @@ const makeAssignEvent = ( author: authorId ? ({ id: authorId, full_name: `User ${authorId}`, email: `${authorId}@example.com` } as ThreadEvent['author']) : (null as unknown as ThreadEvent['author']), + author_display: authorId ? `User ${authorId}` : null, data: { assignees }, has_unread_mention: false, is_editable: false, @@ -30,6 +31,7 @@ const makeIMEvent = (id: string, authorId: string, createdAt = '2026-01-01T10:00 type: ThreadEventTypeEnum.im, channel: null, author: { id: authorId, full_name: `User ${authorId}`, email: `${authorId}@example.com` } as ThreadEvent['author'], + author_display: `User ${authorId}`, data: { content: 'hello', mentions: [] }, has_unread_mention: false, is_editable: false, diff --git a/src/frontend/src/features/layouts/components/thread-view/components/thread-event/index.tsx b/src/frontend/src/features/layouts/components/thread-view/components/thread-event/index.tsx index 3c3b79a76..6abf788c4 100755 --- a/src/frontend/src/features/layouts/components/thread-view/components/thread-event/index.tsx +++ b/src/frontend/src/features/layouts/components/thread-view/components/thread-event/index.tsx @@ -255,8 +255,8 @@ export const ThreadEvent = ({ event, isCondensed = false, onEdit, onDelete, ment {!isCondensed && (
- - {event.author?.full_name || event.author?.email || t("Unknown")} + + {event.author_display || t("Unknown")} {t('{{date}} at {{time}}', { From c2606320e14caf070acb68d9e6e6980568f1c8a6 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Sun, 28 Jun 2026 00:25:29 +0200 Subject: [PATCH 07/21] review fixes --- docs/tiered-storage.md | 1 + docs/webhooks.md | 8 + src/backend/core/admin.py | 26 +- src/backend/core/api/serializers.py | 5 + src/backend/core/mda/dispatch_webhooks.py | 479 ++++++++++++------ src/backend/core/mda/inbound.py | 50 +- src/backend/core/mda/inbound_pipeline.py | 25 +- src/backend/core/mda/inbound_tasks.py | 87 +++- src/backend/core/mda/outbound.py | 9 +- ...lob_inboundmessage_is_internal_and_more.py | 33 ++ src/backend/core/models.py | 61 ++- src/backend/core/services/exporter/tasks.py | 3 +- src/backend/core/signals.py | 11 + src/backend/core/tasks.py | 1 + .../admin/core/channel/change_form.html | 2 +- ...d_api_key.html => regenerated_secret.html} | 0 .../core/tests/mda/test_dispatch_webhooks.py | 334 +++++++++++- src/backend/core/tests/mda/test_inbound.py | 69 ++- .../tests/mda/test_inbound_spoofed_sender.py | 2 +- src/backend/core/tests/models/test_blob.py | 48 +- src/frontend/public/locales/common/en-US.json | 5 +- src/frontend/public/locales/common/fr-FR.json | 5 +- src/frontend/public/locales/common/nl-NL.json | 5 +- src/frontend/public/locales/common/ru-RU.json | 40 +- src/frontend/public/locales/common/uk-UA.json | 40 +- .../webhook-integration-form.tsx | 128 +++-- 26 files changed, 1184 insertions(+), 293 deletions(-) create mode 100644 src/backend/core/migrations/0032_inboundmessage_blob_inboundmessage_is_internal_and_more.py rename src/backend/core/templates/admin/core/channel/{regenerated_api_key.html => regenerated_secret.html} (100%) diff --git a/docs/tiered-storage.md b/docs/tiered-storage.md index 83571aa37..60160c7e8 100644 --- a/docs/tiered-storage.md +++ b/docs/tiered-storage.md @@ -52,6 +52,7 @@ A Blob is alive as long as any of these references it: (the body of a draft being composed) - ``Attachment.blob`` (per-attachment during draft composition) - ``MessageTemplate.blob`` (signatures, autoreply bodies) +- ``InboundMessage.blob`` (in-flight internal message) Plus a short-lived **upload reservation** in the form of a ``MailboxBlob`` row carrying an explicit ``expires_at`` timestamp. The diff --git a/docs/webhooks.md b/docs/webhooks.md index 6eefa66ea..2926c60c2 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -193,6 +193,14 @@ The mechanism is generic: a blocking webhook is the trigger today, but any step that returns `RETRY` (e.g. a persistently-unreachable spam checker) is quarantined the same way. +> **Delivery is at-least-once — make your receiver idempotent.** A +> `RETRY` re-fires the webhook on the next sweep, and a worker crash +> after we POST but before we record success can re-deliver the same +> message. The `Message` itself is created exactly once (deduplicated by +> `Message-ID`), but your endpoint may legitimately see the *same* +> message more than once. Key on the `Message-ID` (or the +> `X-StMsg-*` envelope headers) and treat repeats as no-ops. + (The marker rides in the stored MIME as an `X-StMsg-*` header. Sender- supplied `X-StMsg-*` headers are stripped at ingest, so receivers can't forge it.) diff --git a/src/backend/core/admin.py b/src/backend/core/admin.py index d9355556d..8ab8a317e 100644 --- a/src/backend/core/admin.py +++ b/src/backend/core/admin.py @@ -574,13 +574,13 @@ def get_urls(self): custom_urls = [ path( "/regenerate-secret/", - self.admin_site.admin_view(self.regenerate_api_key_view), - name="core_channel_regenerate_api_key", + self.admin_site.admin_view(self.regenerate_secret_view), + name="core_channel_regenerate_secret", ), ] return custom_urls + urls - def regenerate_api_key_view(self, request, object_id): + def regenerate_secret_view(self, request, object_id): """Regenerate the secret on an api_key or webhook channel. Both channel types authenticate with a single root secret @@ -609,17 +609,31 @@ def regenerate_api_key_view(self, request, object_id): ) return redirect("..") - plaintext = channel.rotate_secret() + root = channel.rotate_secret() + + # Show the credential the *receiver* actually presents, matching + # the DRF ``_attach_credential`` flow. ``rotate_secret`` always + # returns the root signing secret, but an ``api_key``-auth webhook + # authenticates with the derived ``whk_…`` value (the root never + # touches the wire) — showing the root here would hand the operator + # a credential the receiver never sees. + if ( + channel.type == ChannelTypes.WEBHOOK + and (channel.settings or {}).get("auth_method") == "api_key" + ): + display_secret = channel.get_webhook_api_key() + else: + display_secret = root context = { **self.admin_site.each_context(request), "opts": self.model._meta, # noqa: SLF001 "original": channel, "title": "New secret generated", - "secret": plaintext, + "secret": display_secret, } return TemplateResponse( - request, "admin/core/channel/regenerated_api_key.html", context + request, "admin/core/channel/regenerated_secret.html", context ) diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index b94741c94..e8f3aa849 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -1366,6 +1366,11 @@ def get_author_display(self, obj): ``channel``; surface it as ``Webhook: {name}`` so the timeline shows the integration instead of an anonymous "Unknown". Returns ``None`` when neither is available, letting the client fall back. + + The ``Webhook:`` prefix is intentionally a fixed, server-side label + and not localized: it is a trust/provenance marker that prevents a + channel from choosing a ``name`` that impersonates a human author + in the timeline. This might get moved to the frontend in the future. """ if obj.author_id: return obj.author.full_name or obj.author.email diff --git a/src/backend/core/mda/dispatch_webhooks.py b/src/backend/core/mda/dispatch_webhooks.py index ccfaf20d5..094cd58e8 100644 --- a/src/backend/core/mda/dispatch_webhooks.py +++ b/src/backend/core/mda/dispatch_webhooks.py @@ -31,6 +31,7 @@ from __future__ import annotations +import base64 import hashlib import hmac import json @@ -52,9 +53,17 @@ from core.mda.inbound_pipeline import Decision, InboundContext, Step from core.services.ssrf import SSRFSafeSession, SSRFValidationError +from messages.celery_app import app as celery_app + logger = logging.getLogger(__name__) +# Total wall-clock budget for one webhook delivery (connect + send + +# read the capped response body). Enforced as a hard deadline across the +# streamed body read too, so a receiver that drip-feeds bytes just under +# the per-read timeout can't pin a worker indefinitely. WEBHOOK_TIMEOUT = 30 # seconds +# Separate, tight cap on just the TCP/TLS connect phase. +WEBHOOK_CONNECT_TIMEOUT = 5 # seconds # Hard cap on the receiver response body we parse for the action JSON. # The contract body is tiny (action / is_spam / labels = a few hundred @@ -111,7 +120,7 @@ class _HttpResult: # pylint: disable=too-many-instance-attributes reply_draft_template_id: Optional[str] = None -def _read_capped_body(response) -> bytes: +def _read_capped_body(response, deadline: Optional[float] = None) -> bytes: """Read at most ``MAX_RESPONSE_BODY`` bytes from a streaming response. The action body contract is tiny (a few hundred bytes). Reading @@ -119,11 +128,19 @@ def _read_capped_body(response) -> bytes: receiver returns a huge payload we keep what we have and ignore the rest. Network errors mid-stream get logged and the caller treats the partial body as if the receiver had returned no body. + + ``deadline`` (a ``time.monotonic()`` value) bounds total read time: + a receiver dribbling bytes just under the per-read socket timeout + would otherwise hold the worker far past ``WEBHOOK_TIMEOUT``. When + the deadline is crossed we raise ``TimeoutError`` so the caller + treats it as a transport failure (RETRY), not an empty body. """ chunks: List[bytes] = [] received = 0 try: for chunk in response.iter_content(chunk_size=8192, decode_unicode=False): + if deadline is not None and time.monotonic() > deadline: + raise TimeoutError("webhook response read exceeded time budget") if not chunk: continue remaining = MAX_RESPONSE_BODY - received @@ -135,6 +152,8 @@ def _read_capped_body(response) -> bytes: break chunks.append(chunk) received += len(chunk) + except TimeoutError: + raise except Exception as exc: logger.warning("Truncated response body read failed: %s", exc) return b"".join(chunks) @@ -321,20 +340,19 @@ def _resolve_body( body_format: str, raw_data: bytes, parsed_email: JmapEmail, -) -> Tuple[str, bytes, Any]: - """Compute (Content-Type, raw bytes to sign, request kwarg). +) -> Tuple[str, bytes]: + """Compute (Content-Type, raw bytes to sign and POST). - The dispatcher needs: - - the Content-Type to send, - - the exact byte string the signature is computed over, - - the kwarg (``data=`` or ``json=``) to hand off to ``requests``. + The dispatcher needs the Content-Type to send and the exact byte + string the signature is computed over — which is also the byte + string we POST verbatim via ``data=``. JSON is serialised here once so the signature and the wire bytes cannot drift (``requests`` would otherwise re-serialise with different separators/ordering). """ if body_format == FORMAT_EML: - return "message/rfc822", raw_data, {"data": raw_data} + return "message/rfc822", raw_data include_body = body_format == FORMAT_JMAP payload = build_jmap_email(parsed_email, include_body=include_body) # ``separators=(",", ":")`` produces the compact bytes we sign. @@ -343,7 +361,7 @@ def _resolve_body( body_bytes = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode( "utf-8" ) - return "application/json", body_bytes, {"data": body_bytes} + return "application/json", body_bytes def _sign(secret: str, timestamp: str, body_bytes: bytes) -> str: @@ -572,23 +590,43 @@ def __init__(self, channel: models.Channel, phase: str): def __call__(self, ctx: InboundContext) -> Decision: cfg = self.channel.settings or {} - url = cfg["url"] # validated at channel-create time body_format = cfg.get("format", DEFAULT_FORMAT) blocking = bool(cfg.get("blocking", False)) - headers = _envelope_headers( + content_type, body_bytes = _resolve_body( + body_format, ctx.raw_data, ctx.parsed_email + ) + + if not blocking: + # Non-blocking webhooks can't influence delivery, so we never + # run their network I/O on the inbound worker — a slow/hostile + # receiver (the mailbox owner's own endpoint) would otherwise + # stall mail processing for everyone sharing the queue. Snapshot + # the phase-correct request and hand the POST to a Celery task + # on the default queue, then continue immediately. Fires via + # ``.delay`` (not ``on_commit``) to match the previous inline + # semantics: a non-blocking webhook fires regardless of whether + # the message is later dropped or its delivery rolls back. + dispatch_webhook_task.delay( + str(self.channel.id), + str(ctx.mailbox.id), + self.phase, + ctx.is_spam, + ctx.recipient_email, + content_type, + base64.b64encode(body_bytes).decode("ascii"), + ) + return Decision.CONTINUE + + result = _dispatch_webhook( channel=self.channel, - phase=self.phase, mailbox=ctx.mailbox, - recipient_email=ctx.recipient_email, + phase=self.phase, is_spam=ctx.is_spam, - ) - result = self._post( - url=url, - body_format=body_format, - headers=headers, - ctx=ctx, - blocking=blocking, + recipient_email=ctx.recipient_email, + content_type=content_type, + body_bytes=body_bytes, + blocking=True, ) if result.is_spam_override is not None: ctx.is_spam = result.is_spam_override @@ -616,160 +654,263 @@ def __call__(self, ctx: InboundContext) -> Decision: ctx.pending_drafts.append((self.channel.id, result.reply_draft_template_id)) return result.decision - def _post( - self, - *, - url: str, - body_format: str, - headers: Dict[str, str], - ctx: InboundContext, - blocking: bool, - ) -> _HttpResult: - content_type, body_bytes, body_kwargs = _resolve_body( - body_format, ctx.raw_data, ctx.parsed_email - ) - secret = (self.channel.encrypted_settings or {}).get("secret") - if not secret: - # The create path always mints a secret; a row without one - # is misconfigured. We can't sign the POST, so we hold for - # RETRY rather than drop the user's mail — re-minting the - # secret lets the next sweep deliver. A webhook failure must - # never silently discard the email (only an explicit - # ``{"action": "drop"}`` on a 2xx does that). - logger.warning( - "Webhook channel %s has no secret — holding for retry", - self.channel.id, - ) - return _failure(blocking, Decision.RETRY) - - auth_headers = self._build_auth_headers(secret, body_bytes, ctx.mailbox) - if auth_headers is None: - # Unknown/misconfigured auth_method — same reasoning: hold - # for retry, never drop the email on our config error. - return _failure(blocking, Decision.RETRY) - - signed_headers = { - **headers, - "Content-Type": content_type, - **auth_headers, +def _build_auth_headers( + channel: models.Channel, + secret: str, + body_bytes: bytes, + mailbox: models.Mailbox, +) -> Optional[Dict[str, str]]: + """Return the auth headers for the channel's ``auth_method``, or + ``None`` when the channel is misconfigured (caller fails closed).""" + auth_method = (channel.settings or {}).get("auth_method") + + if auth_method == "api_key": + # Derived from the root secret via HMAC. The raw root never + # touches the wire — a receiver-side log leak of this value + # reveals nothing about the root, so HMAC/JWT verification + # remains unforgeable. + return {"X-StMsg-Api-Key": channel.get_webhook_api_key()} + + if auth_method == "jwt": + # HMAC signature over the body + short-TTL HS256 JWT, both keyed + # by the root secret. Signed at send time (here / in the task), + # so the JWT TTL is measured from the actual POST, not enqueue. + now = int(time.time()) + timestamp = str(now) + signature = _sign(secret, timestamp, body_bytes) + bearer = _sign_jwt( + secret, + channel=channel, + mailbox=mailbox, + body_bytes=body_bytes, + issued_at=now, + ) + return { + "X-StMsg-Webhook-Timestamp": timestamp, + "X-StMsg-Webhook-Signature": f"{SIGNATURE_SCHEME}={signature}", + "Authorization": f"Bearer {bearer}", } - kwargs = {"headers": signed_headers, **body_kwargs} - - # ``stream=True`` lets us cap the response body we actually - # read — a misconfigured receiver returning a multi-GB error - # page must not OOM the worker. - try: - response = SSRFSafeSession().post( - url, timeout=WEBHOOK_TIMEOUT, stream=True, **kwargs - ) - except SSRFValidationError as exc: - # SSRF block is a config error on our side (the URL points at a - # disallowed address). Hold for RETRY rather than drop — fixing - # the URL lets the next sweep deliver. We never discard the - # user's mail because of a webhook/config failure. - logger.warning( - "Webhook channel %s rejected by SSRF for url=%s: %s", - self.channel.id, - _sanitize_url(url), - exc, - ) - return _failure(blocking, Decision.RETRY) - except Exception as exc: - # Timeout, connection refused, DNS, unknown transport-level - # failure: all transient on the blocking path. The 48-hour - # quarantine window in the pipeline runner bounds the retries. - # Log only the exception *type*, not its message or traceback: - # requests/urllib3 errors embed the full request URL (path + - # query), which is exactly where receivers carry secret tokens, - # so ``exc``/``exc_info`` would bypass ``_sanitize_url``. - logger.warning( - "Webhook channel %s network error (%s) for url=%s", - self.channel.id, - type(exc).__name__, - _sanitize_url(url), - ) - return _failure(blocking, Decision.RETRY) - - try: - status = response.status_code - if 200 <= status < 300: - if not blocking: - # Non-blocking webhooks never influence delivery — - # ignore the body entirely. Avoids surprises if a - # receiver accidentally returns {"action":"drop"}. - return _HttpResult() - body_bytes_response = _read_capped_body(response) - result = _classify_response_body(body_bytes_response) - if result.decision == Decision.DROP: - logger.info( - "Webhook channel %s requested DROP via response body for url=%s", - self.channel.id, - _sanitize_url(url), - ) - return result - logger.info( - "Webhook channel %s returned status %s for url=%s", - self.channel.id, - status, - _sanitize_url(url), - ) - # Any non-2xx status is a transient failure → RETRY. A - # blocking webhook DROPs an email *only* when it explicitly - # returns ``{"action": "drop"}`` with a 2xx (handled above). - # A receiver bug that answers 4xx must never cost the user - # their mail — the 7-day retry budget bounds the hold. - return _failure(blocking, Decision.RETRY) - finally: - response.close() - - def _build_auth_headers( - self, - secret: str, - body_bytes: bytes, - mailbox: models.Mailbox, - ) -> Optional[Dict[str, str]]: - """Return the auth headers for the channel's ``auth_method``, - or ``None`` when the channel is misconfigured (caller fails - closed).""" - auth_method = (self.channel.settings or {}).get("auth_method") - - if auth_method == "api_key": - # Derived from the root secret via HMAC. The raw root - # never touches the wire — a receiver-side log leak of - # this value reveals nothing about the root, so HMAC/JWT - # verification remains unforgeable. - return {"X-StMsg-Api-Key": self.channel.get_webhook_api_key()} - - if auth_method == "jwt": - # HMAC signature over the body + short-TTL HS256 JWT, - # both keyed by the root secret. - now = int(time.time()) - timestamp = str(now) - signature = _sign(secret, timestamp, body_bytes) - bearer = _sign_jwt( - secret, - channel=self.channel, - mailbox=mailbox, - body_bytes=body_bytes, - issued_at=now, - ) - return { - "X-StMsg-Webhook-Timestamp": timestamp, - "X-StMsg-Webhook-Signature": f"{SIGNATURE_SCHEME}={signature}", - "Authorization": f"Bearer {bearer}", - } - - # Settings validator forbids creating a webhook channel without - # a valid auth_method; an existing row with a missing/unknown - # value is misconfigured. + # Settings validator forbids creating a webhook channel without a + # valid auth_method; an existing row with a missing/unknown value is + # misconfigured. + logger.warning( + "Webhook channel %s has missing/unknown auth_method=%r — skipping", + channel.id, + auth_method, + ) + return None + + +def _deliver_signed_webhook( + *, + channel: models.Channel, + mailbox: models.Mailbox, + url: str, + content_type: str, + body_bytes: bytes, + envelope_headers: Dict[str, str], + blocking: bool, +) -> _HttpResult: + """Sign and POST one webhook, returning the classified ``_HttpResult``. + + The single network path shared by the inline blocking step and the + out-of-band non-blocking task, so signing/SSRF/timeout/response + handling can never drift between them. + """ + secret = (channel.encrypted_settings or {}).get("secret") + if not secret: + # The create path always mints a secret; a row without one is + # misconfigured. We can't sign the POST, so we hold for RETRY + # rather than drop the user's mail — re-minting the secret lets + # the next sweep deliver. A webhook failure must never silently + # discard the email (only an explicit ``{"action": "drop"}`` on + # a 2xx does that). logger.warning( - "Webhook channel %s has missing/unknown auth_method=%r — skipping", - self.channel.id, - auth_method, + "Webhook channel %s has no secret — holding for retry", + channel.id, + ) + return _failure(blocking, Decision.RETRY) + + auth_headers = _build_auth_headers(channel, secret, body_bytes, mailbox) + if auth_headers is None: + # Unknown/misconfigured auth_method — same reasoning: hold for + # retry, never drop the email on our config error. + return _failure(blocking, Decision.RETRY) + + signed_headers = { + **envelope_headers, + "Content-Type": content_type, + **auth_headers, + } + # ``stream=True`` lets us cap the response body we actually read — a + # misconfigured receiver returning a multi-GB error page must not OOM + # the worker. The ``(connect, read)`` tuple bounds the connect phase + # tightly and each socket read; the ``deadline`` below bounds the + # *total* exchange against slow drip. + deadline = time.monotonic() + WEBHOOK_TIMEOUT + try: + response = SSRFSafeSession().post( + url, + timeout=(WEBHOOK_CONNECT_TIMEOUT, WEBHOOK_TIMEOUT), + stream=True, + headers=signed_headers, + data=body_bytes, + ) + except SSRFValidationError as exc: + # SSRF block is a config error on our side (the URL points at a + # disallowed address). Hold for RETRY rather than drop — fixing + # the URL lets the next sweep deliver. We never discard the user's + # mail because of a webhook/config failure. + logger.warning( + "Webhook channel %s rejected by SSRF for url=%s: %s", + channel.id, + _sanitize_url(url), + exc, + ) + return _failure(blocking, Decision.RETRY) + except Exception as exc: + # Timeout, connection refused, DNS, unknown transport-level + # failure: all transient. The 48-hour quarantine window in the + # pipeline runner bounds the retries. Log only the exception + # *type*, not its message or traceback: requests/urllib3 errors + # embed the full request URL (path + query), which is exactly + # where receivers carry secret tokens, so ``exc``/``exc_info`` + # would bypass ``_sanitize_url``. + logger.warning( + "Webhook channel %s network error (%s) for url=%s", + channel.id, + type(exc).__name__, + _sanitize_url(url), + ) + return _failure(blocking, Decision.RETRY) + + try: + status = response.status_code + if 200 <= status < 300: + if not blocking: + # Non-blocking webhooks never influence delivery — ignore + # the body entirely. Avoids surprises if a receiver + # accidentally returns {"action":"drop"}. + return _HttpResult() + try: + body_bytes_response = _read_capped_body(response, deadline=deadline) + except TimeoutError: + logger.warning( + "Webhook channel %s exceeded %ss budget reading response " + "for url=%s — holding for retry", + channel.id, + WEBHOOK_TIMEOUT, + _sanitize_url(url), + ) + return _failure(blocking, Decision.RETRY) + result = _classify_response_body(body_bytes_response) + if result.decision == Decision.DROP: + logger.info( + "Webhook channel %s requested DROP via response body for url=%s", + channel.id, + _sanitize_url(url), + ) + return result + + logger.info( + "Webhook channel %s returned status %s for url=%s", + channel.id, + status, + _sanitize_url(url), + ) + # Any non-2xx status is a transient failure → RETRY. A blocking + # webhook DROPs an email *only* when it explicitly returns + # ``{"action": "drop"}`` with a 2xx (handled above). A receiver + # bug that answers 4xx must never cost the user their mail — the + # 48-hour quarantine window bounds the hold. + return _failure(blocking, Decision.RETRY) + finally: + response.close() + + +def _dispatch_webhook( + *, + channel: models.Channel, + mailbox: models.Mailbox, + phase: str, + is_spam: Optional[bool], + recipient_email: str, + content_type: str, + body_bytes: bytes, + blocking: bool, +) -> _HttpResult: + """Build the envelope headers and deliver one webhook. + + The shared entry point above ``_deliver_signed_webhook``: both the + inline blocking step and the out-of-band non-blocking task land here, + so the URL lookup and header-building can't drift between them. + """ + url = (channel.settings or {}).get("url") + if not url: + # The serializer guarantees a url on create; a row without one is + # misconfigured. Hold for retry rather than drop (blocking); for + # the non-blocking task this collapses to a no-op CONTINUE. + logger.warning("Webhook channel %s has no url — skipping", channel.id) + return _failure(blocking, Decision.RETRY) + envelope_headers = _envelope_headers( + channel=channel, + phase=phase, + mailbox=mailbox, + recipient_email=recipient_email, + is_spam=is_spam, + ) + return _deliver_signed_webhook( + channel=channel, + mailbox=mailbox, + url=url, + content_type=content_type, + body_bytes=body_bytes, + envelope_headers=envelope_headers, + blocking=blocking, + ) + + +@celery_app.task +def dispatch_webhook_task( + channel_id: str, + mailbox_id: str, + phase: str, + is_spam: Optional[bool], + recipient_email: str, + content_type: str, + body_b64: str, +) -> None: + """Deliver one non-blocking webhook off the inbound worker. + + Non-blocking webhooks can't influence delivery, so their network I/O + runs here (default queue) instead of pinning the time-sensitive + inbound pipeline worker. Best-effort and at-least-once: the message + is already handled, so any failure is logged and swallowed (matching + the previous inline non-blocking contract). The request is re-signed + here at send time, so the root secret never travels through the + broker and the JWT TTL is measured from the actual POST. + """ + try: + channel = models.Channel.objects.filter(id=channel_id).first() + mailbox = models.Mailbox.objects.filter(id=mailbox_id).first() + if channel is None or mailbox is None: + return + _dispatch_webhook( + channel=channel, + mailbox=mailbox, + phase=phase, + is_spam=is_spam, + recipient_email=recipient_email, + content_type=content_type, + body_bytes=base64.b64decode(body_b64), + blocking=False, + ) + except Exception: # pylint: disable=broad-exception-caught + logger.exception( + "Non-blocking webhook dispatch failed (channel=%s)", channel_id ) - return None def webhook_steps_for_mailbox( diff --git a/src/backend/core/mda/inbound.py b/src/backend/core/mda/inbound.py index ee5cc1ffc..fb9ee781f 100644 --- a/src/backend/core/mda/inbound.py +++ b/src/backend/core/mda/inbound.py @@ -139,13 +139,22 @@ def deliver_inbound_message( imap_labels: list[str] | None = None, imap_flags: list[str] | None = None, channel: models.Channel | None = None, - skip_inbound_queue: bool = False, + is_internal: bool = False, + blob: "models.Blob | None" = None, ) -> bool: # Return True on success, False on failure """Deliver a parsed inbound email message. - For imports (is_import=True) or when skip_inbound_queue=True, messages are created - directly without spam checking. For regular messages, they are queued for spam - processing via rspamd. Warning: messages imported here could be is_sender=True. + Imports (``is_import=True``) bypass the queue and create the message + directly — historical bulk data, no spam check, no user webhooks. + Warning: messages imported here could be is_sender=True. + + Everything else is queued for the inbound pipeline via + ``process_inbound_message_task`` (spam steps + user webhooks). + Internal mailbox-to-mailbox delivery sets ``is_internal=True`` so the + pipeline skips spam checking while still firing webhooks, and passes + the sender's already-committed ``blob`` so the queue row references + the encrypted bytes instead of copying them. ``raw_data`` is stored + inline only when no ``blob`` is supplied (the external MTA path). raw_data is not parsed again, just stored as is. """ @@ -181,9 +190,10 @@ def deliver_inbound_message( ) return True # Return success since we handled the duplicate gracefully - # --- 3. Handle imports and internal messages directly, queue others for spam processing --- # - if is_import or skip_inbound_queue: - # Imports and internal messages bypass spam checking and create messages directly + # --- 3. Imports bypass the queue; everything else runs the pipeline --- # + if is_import: + # Historical bulk import: create the message directly, no spam + # check and no user webhooks (autoreply is suppressed too). result = _create_message_from_inbound( recipient_email=recipient_email, parsed_email=parsed_email, @@ -196,28 +206,30 @@ def deliver_inbound_message( channel=channel, is_spam=False, # Bypassed messages are never marked as spam ) - - # Send autoreply for internal messages (not imports, which are historical) - if not is_import and isinstance(result, models.Message): - from core.mda.autoreply import ( # pylint: disable=import-outside-toplevel - try_send_autoreply, - ) - - try_send_autoreply(mailbox, parsed_email, result) - return bool(result) - # Regular messages: queue for spam processing + # Internal mail is expected to reference the sender's already-committed + # blob — that's the whole point (no second plaintext copy). Enforce the + # contract so a future caller can't silently fall back to inline bytes. + if is_internal and blob is None: + raise ValueError("internal delivery requires a blob") + + # External and internal messages: queue for the inbound pipeline. + # Internal mail references the sender's existing blob (no second + # plaintext copy); external mail stores its MTA bytes inline. try: inbound_message = models.InboundMessage.objects.create( mailbox=mailbox, - raw_data=raw_data, + raw_data=None if blob is not None else raw_data, + blob=blob, channel=channel, + is_internal=is_internal, ) logger.info( - "Queued inbound message %s (recipient: %s)", + "Queued inbound message %s (recipient: %s, internal: %s)", inbound_message.id, recipient_email, + is_internal, ) # Queue the task immediately for processing (no lag) process_inbound_message_task.delay(str(inbound_message.id)) diff --git a/src/backend/core/mda/inbound_pipeline.py b/src/backend/core/mda/inbound_pipeline.py index 0afbfdee2..01fa8b079 100644 --- a/src/backend/core/mda/inbound_pipeline.py +++ b/src/backend/core/mda/inbound_pipeline.py @@ -394,16 +394,29 @@ def build_inbound_pipeline(ctx: InboundContext) -> List[Step]: # identical before- and after-spam, so a second query is pure waste. channels = find_webhook_channels_for_mailbox(ctx.mailbox) + # Internal mailbox-to-mailbox mail is trusted and not externally + # authenticated: run only the user-webhook steps. The spam steps + # would no-op anyway (the task pre-sets is_spam=False), and the auth + # step would prepend a meaningless X-StMsg-Sender-Auth banner — which + # also mutates the bytes and defeats blob dedup with the sender — plus + # do needless DNS/rspamd work. Webhooks still fire on both phases so + # internal mail is indistinguishable from external to a consumer. + if ctx.inbound_message.is_internal: + return [ + *webhook_steps_for_mailbox( + ctx.mailbox, phase="before_spam", channels=channels + ), + *webhook_steps_for_mailbox( + ctx.mailbox, phase="after_spam", channels=channels + ), + ] + return [ - *webhook_steps_for_mailbox( - ctx.mailbox, phase="before_spam", channels=channels - ), + *webhook_steps_for_mailbox(ctx.mailbox, phase="before_spam", channels=channels), _make_hardcoded_rules_step(ctx.spam_config), _make_rspamd_step(ctx.spam_config), _make_inbound_auth_step(ctx.spam_config), - *webhook_steps_for_mailbox( - ctx.mailbox, phase="after_spam", channels=channels - ), + *webhook_steps_for_mailbox(ctx.mailbox, phase="after_spam", channels=channels), ] diff --git a/src/backend/core/mda/inbound_tasks.py b/src/backend/core/mda/inbound_tasks.py index 3c2424bc8..00a751042 100644 --- a/src/backend/core/mda/inbound_tasks.py +++ b/src/backend/core/mda/inbound_tasks.py @@ -112,6 +112,43 @@ def _handle_retry( } +def _retry_or_abandon( + inbound_message: models.InboundMessage, reason: str +) -> Dict[str, Any]: + """Bounded handling for a message that failed to be created/processed. + + Within ``QUARANTINE_AFTER`` the row is kept so the 5-min sweep retries + it (a transient DB error or constraint hiccup clears on its own). Past + the window the attempt is abandoned and the row deleted: otherwise a + poison message (one that parses but can never be created) loops + forever, re-running the whole pipeline — and re-firing every user + webhook — on each sweep. + """ + age = timezone.now() - inbound_message.created_at + if age <= QUARANTINE_AFTER: + inbound_message.error_message = reason + inbound_message.save(update_fields=["error_message"]) + return { + "success": False, + "inbound_message_id": str(inbound_message.id), + "error": "retry", + "reason": reason, + } + logger.error( + "Inbound message %s abandoned after persistent failure (age=%s): %s", + inbound_message.id, + age, + reason, + ) + inbound_message.delete() + return { + "success": False, + "inbound_message_id": str(inbound_message.id), + "error": "abandoned", + "reason": reason, + } + + def _stamp_processing_failed(ctx: InboundContext) -> None: """Prepend the ``X-StMsg-Processing-Failed`` marker to the message. @@ -162,7 +199,7 @@ def process_inbound_message_task(self, inbound_message_id: str): logger.error(error_msg) return {"success": False, "error": error_msg} - raw_data_bytes = bytes(inbound_message.raw_data) + raw_data_bytes = inbound_message.get_raw_bytes() parsed_email = parse_email(raw_data_bytes) if parsed_email is None: error_msg = "Failed to parse email message" @@ -181,13 +218,17 @@ def process_inbound_message_task(self, inbound_message_id: str): parsed_email=parsed_email, spam_config=mailbox.domain.get_spam_config(), ) - if _is_selfcheck(parsed_email, recipient_email): - # System self-probe: short-circuit the spam check before - # the pipeline runs. The hardcoded-rules + rspamd steps - # both no-op when ctx.is_spam is already set. + if inbound_message.is_internal or _is_selfcheck(parsed_email, recipient_email): + # Internal mailbox-to-mailbox mail is trusted, and the system + # self-probe must never be junked: short-circuit the spam + # check before the pipeline runs. The hardcoded-rules + rspamd + # steps both no-op when ctx.is_spam is already set, but the + # user-webhook steps still fire — so internal mail looks + # identical to external mail to a webhook consumer. ctx.is_spam = False logger.debug( - "Selfcheck probe — pre-setting is_spam=False for %s", + "Skipping spam check (internal=%s) for %s", + inbound_message.is_internal, inbound_message_id, ) @@ -297,9 +338,22 @@ def process_inbound_message_task(self, inbound_message_id: str): # The ``skip_autoreply`` flag wraps the same gate from # the outside so a non-spam message can also opt out # (e.g. when the webhook itself replies). - try_send_autoreply( - mailbox, ctx.parsed_email, inbound_msg, is_spam=bool(ctx.is_spam) - ) + # + # Best-effort: the message has already landed (and the + # InboundMessage row is already deleted). A send failure + # here must not bubble to the outer ``except`` — that + # would try to retry/abandon an already-deleted row. + try: + try_send_autoreply( + mailbox, + ctx.parsed_email, + inbound_msg, + is_spam=bool(ctx.is_spam), + ) + except Exception: # pylint: disable=broad-exception-caught + logger.exception( + "Autoreply failed for inbound message %s", inbound_message_id + ) logger.info( "Successfully processed inbound message %s (is_spam=%s)", @@ -313,19 +367,20 @@ def process_inbound_message_task(self, inbound_message_id: str): "is_spam": ctx.is_spam, } - error_msg = "Failed to create message from inbound message" - inbound_message.error_message = error_msg - inbound_message.save(update_fields=["error_message"]) - # Keep the message for retry - return {"success": False, "error": error_msg} + # Creation failed (transient DB error, constraint, …). Hold for a + # bounded retry rather than keeping the row forever. + return _retry_or_abandon( + inbound_message, "Failed to create message from inbound message" + ) except Exception as e: logger.exception( "Error processing inbound message %s: %s", inbound_message_id, e ) if inbound_message: - inbound_message.error_message = str(e) - inbound_message.save(update_fields=["error_message"]) + # Same bounded-retry policy as a failed creation: a persistent + # error must not pin the row (and re-fire webhooks) forever. + return _retry_or_abandon(inbound_message, str(e)) return {"success": False, "error": str(e)} finally: # Always release the lock diff --git a/src/backend/core/mda/outbound.py b/src/backend/core/mda/outbound.py index 51be79cfb..adac5a848 100644 --- a/src/backend/core/mda/outbound.py +++ b/src/backend/core/mda/outbound.py @@ -645,11 +645,18 @@ def _mark_delivered( and not force_mta_out ): try: + # Reference the sender's already-committed blob so + # the queue row carries the encrypted bytes, not a + # second plaintext copy. ``delivered`` here means + # "handed off to the recipient's inbound queue" — + # the recipient's webhook outcome plays out on their + # side and never feeds back into the sender's status. delivered = deliver_inbound_message( recipient_email, parsed_email, blob_content, - skip_inbound_queue=True, + is_internal=True, + blob=message.blob, ) _mark_delivered(recipient_email, delivered, True) except Exception as e: diff --git a/src/backend/core/migrations/0032_inboundmessage_blob_inboundmessage_is_internal_and_more.py b/src/backend/core/migrations/0032_inboundmessage_blob_inboundmessage_is_internal_and_more.py new file mode 100644 index 000000000..5db9b6967 --- /dev/null +++ b/src/backend/core/migrations/0032_inboundmessage_blob_inboundmessage_is_internal_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 5.2.11 on 2026-06-27 17:18 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0031_message_mime_id_index'), + ] + + operations = [ + migrations.AddField( + model_name='inboundmessage', + name='blob', + field=models.ForeignKey(blank=True, help_text='Existing blob holding the message bytes (internal mail)', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='inbound_messages', to='core.blob'), + ), + migrations.AddField( + model_name='inboundmessage', + name='is_internal', + field=models.BooleanField(default=False, help_text='Internal mailbox-to-mailbox delivery (skips spam checking)', verbose_name='is internal'), + ), + migrations.AlterField( + model_name='inboundmessage', + name='raw_data', + field=models.BinaryField(blank=True, help_text='Raw email message bytes (external mail; null when backed by blob)', null=True, verbose_name='raw data'), + ), + migrations.AddConstraint( + model_name='inboundmessage', + constraint=models.CheckConstraint(condition=models.Q(models.Q(('blob__isnull', True), ('raw_data__isnull', False)), models.Q(('blob__isnull', False), ('raw_data__isnull', True)), _connector='OR'), name='inboundmessage_exactly_one_source'), + ), + ] diff --git a/src/backend/core/models.py b/src/backend/core/models.py index 9eddda8d8..5b8ddf02c 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -2206,7 +2206,41 @@ class InboundMessage(BaseModel): on_delete=models.CASCADE, related_name="inbound_messages", ) - raw_data = models.BinaryField("raw data", help_text="Raw email message bytes") + # Two mutually-exclusive sources for the message bytes (enforced by + # the ``inboundmessage_exactly_one_source`` constraint below): + # - ``raw_data``: external mail arrives from the MTA as loose + # bytes with no Blob yet, so we store them inline (plaintext, + # transient — deleted once the message is created). + # - ``blob``: internal mail already has a committed, encrypted, + # deduped Blob (the sender's ``Message.blob``). We reference it + # instead of copying the bytes — no second plaintext copy, and + # ``get_content()`` reads it back transparently. + raw_data = models.BinaryField( + "raw data", + null=True, + blank=True, + help_text="Raw email message bytes (external mail; null when backed by blob)", + ) + # PROTECT mirrors ``Message.blob``: the GC sweep is the only + # authorised deleter and clears references first. ``is_referenced`` + # counts this FK, so a referenced blob is never collected. + blob = models.ForeignKey( + "Blob", + on_delete=models.PROTECT, + null=True, + blank=True, + related_name="inbound_messages", + help_text="Existing blob holding the message bytes (internal mail)", + ) + # Internal mail (mailbox-to-mailbox) is trusted: the task pre-sets + # ``is_spam=False`` so the spam steps no-op while the user-webhook + # steps still run. Kept explicit rather than inferred from ``blob`` + # so storage mechanism stays decoupled from trust semantics. + is_internal = models.BooleanField( + "is internal", + default=False, + help_text="Internal mailbox-to-mailbox delivery (skips spam checking)", + ) channel = models.ForeignKey( "Channel", on_delete=models.SET_NULL, @@ -2228,10 +2262,31 @@ class Meta: indexes = [ models.Index(fields=["created_at"]), ] + constraints = [ + # Exactly one byte source: inline raw_data XOR a blob FK. + models.CheckConstraint( + check=( + models.Q(raw_data__isnull=False, blob__isnull=True) + | models.Q(raw_data__isnull=True, blob__isnull=False) + ), + name="inboundmessage_exactly_one_source", + ), + ] def __str__(self): return f"InboundMessage {self.id} - {self.mailbox}" + def get_raw_bytes(self) -> bytes: + """Return the message bytes regardless of storage backing. + + Internal mail references an existing (encrypted, deduped) blob; + external mail stores the bytes inline. ``Blob.get_content()`` + transparently decrypts and handles tiered storage. + """ + if self.blob_id is not None: + return self.blob.get_content() + return bytes(self.raw_data) + class BlobManager(models.Manager): """Custom Manager for Blob model.""" @@ -2381,6 +2436,10 @@ def is_referenced(self, blob_id) -> bool: ).exists() or Attachment.objects.filter(blob_id=blob_id).exists() or MessageTemplate.objects.filter(blob_id=blob_id).exists() + # Internal mail in flight references the sender's blob from + # its transient queue row; collecting it here would strip the + # bytes out from under the recipient pipeline mid-delivery. + or InboundMessage.objects.filter(blob_id=blob_id).exists() ) def user_can_access(self, user, blob_id) -> bool: diff --git a/src/backend/core/services/exporter/tasks.py b/src/backend/core/services/exporter/tasks.py index dfa48a8c5..8918d36fb 100644 --- a/src/backend/core/services/exporter/tasks.py +++ b/src/backend/core/services/exporter/tasks.py @@ -713,6 +713,5 @@ def _create_notification_message( recipient_email=mailbox_email, parsed_email=parsed_email, raw_data=raw_data, - is_import=True, # Skip spam checking - skip_inbound_queue=True, + is_import=True, # Skip spam checking; bypass queue and webhooks ) diff --git a/src/backend/core/signals.py b/src/backend/core/signals.py index fb38324f4..7f852aaa3 100644 --- a/src/backend/core/signals.py +++ b/src/backend/core/signals.py @@ -165,6 +165,17 @@ def schedule_template_blob_for_gc(sender, instance, **kwargs): schedule_for_gc(instance.blob_id) +@receiver(post_delete, sender=models.InboundMessage) +def schedule_inbound_message_blob_for_gc(sender, instance, **kwargs): + """Push ``InboundMessage.blob`` id into the GC set. + + Internal mail references the sender's blob while in flight; once the + task deletes the queue row the blob may have become collectable + (no-op when ``blob_id`` is None, i.e. external inline-bytes rows). + """ + schedule_for_gc(instance.blob_id) + + @receiver(post_delete, sender=models.Message) def delete_message_from_index(sender, instance, **kwargs): """Enqueue a targeted OpenSearch delete for the message child document. diff --git a/src/backend/core/tasks.py b/src/backend/core/tasks.py index 3b1d626cb..892dfa0a9 100644 --- a/src/backend/core/tasks.py +++ b/src/backend/core/tasks.py @@ -1,6 +1,7 @@ # pylint: disable=wildcard-import, unused-wildcard-import """Register all tasks here so that Celery autodiscovery can find them.""" +from core.mda.dispatch_webhooks import dispatch_webhook_task # noqa: F401 from core.mda.inbound_tasks import * # noqa: F403 from core.mda.outbound_tasks import * # noqa: F403 from core.services.blob_gc import * # noqa: F403 diff --git a/src/backend/core/templates/admin/core/channel/change_form.html b/src/backend/core/templates/admin/core/channel/change_form.html index 20d4cace7..b4709beff 100644 --- a/src/backend/core/templates/admin/core/channel/change_form.html +++ b/src/backend/core/templates/admin/core/channel/change_form.html @@ -10,7 +10,7 @@ {% if original and original.type == "api_key" %}
  • {% csrf_token %} diff --git a/src/backend/core/templates/admin/core/channel/regenerated_api_key.html b/src/backend/core/templates/admin/core/channel/regenerated_secret.html similarity index 100% rename from src/backend/core/templates/admin/core/channel/regenerated_api_key.html rename to src/backend/core/templates/admin/core/channel/regenerated_secret.html diff --git a/src/backend/core/tests/mda/test_dispatch_webhooks.py b/src/backend/core/tests/mda/test_dispatch_webhooks.py index 2f1a6a907..467b325e3 100644 --- a/src/backend/core/tests/mda/test_dispatch_webhooks.py +++ b/src/backend/core/tests/mda/test_dispatch_webhooks.py @@ -4,6 +4,7 @@ # pylint: disable=missing-class-docstring,too-many-lines,too-many-public-methods # pylint: disable=use-implicit-booleaness-not-comparison +import base64 import hashlib import hmac import json @@ -18,6 +19,7 @@ import requests as requests_lib from core import enums, factories, models +from core.mda import outbound from core.mda.dispatch_webhooks import ( DEFAULT_FORMAT, FORMAT_EML, @@ -26,6 +28,7 @@ PHASE_BEFORE_SPAM, _classify_response_body, build_jmap_email, + dispatch_webhook_task, find_webhook_channels_for_mailbox, webhook_steps_for_mailbox, ) @@ -635,8 +638,9 @@ def test_blocking_retries_on_connection_error( def test_blocking_retries_on_unknown_exception( self, mock_session, mailbox, parsed_email ): - """Unknown transport-level errors land as RETRY — the 7-day cap - bounds how long we'll keep trying a busted receiver.""" + """Unknown transport-level errors land as RETRY — the 48-hour + quarantine window bounds how long we'll keep trying a busted + receiver.""" factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -749,9 +753,7 @@ def test_jmap_format_sends_jmap_email_json( assert kwargs["headers"]["Content-Type"] == "application/json" @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_jmap_metadata_skips_body_parts( - self, mock_session, mailbox, parsed_email - ): + def test_jmap_metadata_skips_body_parts(self, mock_session, mailbox, parsed_email): """Notification variant: no textBody/htmlBody/bodyValues/attachments.""" factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, @@ -1275,6 +1277,296 @@ def test_after_spam_is_spam_header( assert headers["X-StMsg-Is-Spam"] == "true" assert headers["X-StMsg-Phase"] == "after_spam" + @patch("core.mda.inbound_tasks._create_message_from_inbound") + @patch("core.mda.inbound_pipeline._call_rspamd") + def test_creation_failure_retries_then_abandons(self, mock_rspamd, mock_create): + """A message that parses but can never be created is held for a + bounded retry, then abandoned — it must not loop (re-firing the + pipeline + webhooks) forever.""" + mock_rspamd.return_value = (False, None, None) + mock_create.return_value = None # creation always fails + mailbox = factories.MailboxFactory() + raw_data = ( + b"From: s@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: t\r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + + # Within the quarantine window → held for retry, row kept. + with patch.object(process_inbound_message_task, "update_state", Mock()): + result = process_inbound_message_task.run(str(inbound_message.id)) + assert result["error"] == "retry" + assert models.InboundMessage.objects.filter(id=inbound_message.id).exists() + + # Aged past the window → abandoned, row deleted (loop stops). + models.InboundMessage.objects.filter(id=inbound_message.id).update( + created_at=dj_timezone.now() - QUARANTINE_AFTER - QUARANTINE_AFTER + ) + with patch.object(process_inbound_message_task, "update_state", Mock()): + result = process_inbound_message_task.run(str(inbound_message.id)) + assert result["error"] == "abandoned" + assert not models.InboundMessage.objects.filter(id=inbound_message.id).exists() + + +# --- non-blocking dispatch isolation --- # + + +@pytest.mark.django_db +class TestNonBlockingDispatch: + """Non-blocking webhooks must hand the POST to a Celery task (default + queue) rather than do network I/O on the inbound worker — that's the + worker-isolation guarantee.""" + + def _ctx(self, mailbox, parsed_email, is_spam=False): + return InboundContext( + mailbox=mailbox, + inbound_message=Mock(id="im", created_at=dj_timezone.now()), + recipient_email=str(mailbox), + raw_data=b"From: s@example.com\r\nTo: x\r\nSubject: t\r\n\r\nbody", + parsed_email=parsed_email, + spam_config={}, + is_spam=is_spam, + ) + + @patch("core.mda.dispatch_webhooks.dispatch_webhook_task") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_non_blocking_enqueues_and_skips_inline_post( + self, mock_session, mock_task, mailbox, parsed_email + ): + channel = factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.inbound"], + "blocking": False, + "auth_method": "jwt", + }, + ) + ctx = self._ctx(mailbox, parsed_email, is_spam=False) + + for step in webhook_steps_for_mailbox(mailbox, phase=PHASE_AFTER_SPAM): + assert step(ctx) == Decision.CONTINUE + + # Enqueued exactly once with the phase-correct snapshot... + mock_task.delay.assert_called_once() + args = mock_task.delay.call_args[0] + assert args[0] == str(channel.id) + assert args[1] == str(mailbox.id) + assert args[2] == PHASE_AFTER_SPAM + assert args[3] is False # is_spam + # ...and NO network I/O happened on the inbound worker. + mock_session.return_value.post.assert_not_called() + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_dispatch_webhook_task_posts_signed(self, mock_session, mailbox): + channel = factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.inbound"], + "blocking": False, + "auth_method": "jwt", + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + + dispatch_webhook_task( + str(channel.id), + str(mailbox.id), + PHASE_AFTER_SPAM, + False, + str(mailbox), + "message/rfc822", + base64.b64encode(b"raw mime").decode("ascii"), + ) + + mock_session.return_value.post.assert_called_once() + headers = mock_session.return_value.post.call_args.kwargs["headers"] + assert headers["X-StMsg-Phase"] == "after_spam" + assert headers["X-StMsg-Event"] == enums.WebhookEvents.MESSAGE_INBOUND.value + # Signed at send time (jwt auth_method). + assert "X-StMsg-Webhook-Signature" in headers + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_dispatch_webhook_task_no_ops_when_channel_gone( + self, mock_session, mailbox + ): + # Channel id that doesn't exist → best-effort no-op, no POST, no raise. + dispatch_webhook_task( + str(uuid.uuid4()), + str(mailbox.id), + PHASE_AFTER_SPAM, + False, + str(mailbox), + "message/rfc822", + base64.b64encode(b"raw").decode("ascii"), + ) + mock_session.return_value.post.assert_not_called() + + +# --- internal (mailbox-to-mailbox) delivery --- # + + +@pytest.mark.django_db +class TestInternalDeliveryWebhooks: + """Internal mail (one local mailbox to another) must fire the + recipient's ``message.inbound`` webhook exactly like external mail. + + To a webhook consumer an internal email should be indistinguishable + from an external one — same event, same envelope headers. The + sender and recipient may be fully unrelated tenants (different + domains on the same instance), so the recipient's webhook outcome + (drop / retry / failure) must NOT leak back into the sender's + delivery status: the sender sees ``INTERNAL`` the moment the message + is handed off, and the webhook plays out on the recipient's side. + """ + + def _build_internal_message(self, sender_mailbox, recipient_email): + thread = factories.ThreadFactory() + factories.ThreadAccessFactory( + mailbox=sender_mailbox, + thread=thread, + role=enums.ThreadAccessRoleChoices.EDITOR, + ) + sender_contact = factories.ContactFactory(mailbox=sender_mailbox) + message = factories.MessageFactory( + thread=thread, + sender=sender_contact, + is_draft=False, + is_sender=True, + subject="Internal hello", + ) + raw_mime = ( + f"From: {sender_contact.email}\r\n" + f"To: {recipient_email}\r\n" + "Subject: Internal hello\r\n" + "Message-ID: \r\n\r\nbody" + ).encode() + message.blob = factories.BlobFactory( + mailbox=sender_mailbox, + content=raw_mime, + content_type="message/rfc822", + ) + message.save() + recipient_contact = factories.ContactFactory( + mailbox=sender_mailbox, email=recipient_email + ) + factories.MessageRecipientFactory( + message=message, + contact=recipient_contact, + type=models.MessageRecipientTypeChoices.TO, + ) + return message + + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_internal_delivery_fires_recipient_webhook(self, mock_session, mock_rspamd): + # Sender and recipient live on *different* domains — unrelated + # tenants that happen to share the instance. + sender_mailbox = factories.MailboxFactory() + recipient_mailbox = factories.MailboxFactory() + recipient_email = str(recipient_mailbox) + + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=recipient_mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.inbound"], + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + + message = self._build_internal_message(sender_mailbox, recipient_email) + outbound.send_message(message) + + # The recipient's webhook fired with the inbound event, addressed + # to the recipient mailbox. + assert mock_session.return_value.post.called + headers = mock_session.return_value.post.call_args.kwargs["headers"] + assert headers["X-StMsg-Event"] == enums.WebhookEvents.MESSAGE_INBOUND.value + assert headers["X-StMsg-Recipient"] == recipient_email + + # Sender's view is decoupled from the recipient's webhook: it sees + # a clean internal handoff regardless of what the webhook returns. + recipient = message.recipients.first() + recipient.refresh_from_db() + assert recipient.delivery_status == enums.MessageDeliveryStatusChoices.INTERNAL + + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_recipient_webhook_failure_does_not_affect_sender( + self, mock_session, mock_rspamd + ): + """A failing/blocking recipient webhook is the recipient tenant's + problem: it holds *their* queue row for retry but never feeds back + into the (possibly unrelated) sender's delivery status.""" + sender_mailbox = factories.MailboxFactory() + recipient_mailbox = factories.MailboxFactory() + recipient_email = str(recipient_mailbox) + + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=recipient_mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.inbound"], + "phase": "before_spam", + "blocking": True, + }, + ) + # 4xx is a webhook error → the recipient pipeline holds for RETRY. + mock_session.return_value.post.return_value = _make_response(403) + + message = self._build_internal_message(sender_mailbox, recipient_email) + outbound.send_message(message) + + # Sender still sees a clean handoff. + recipient = message.recipients.first() + recipient.refresh_from_db() + assert recipient.delivery_status == enums.MessageDeliveryStatusChoices.INTERNAL + + # The recipient's queue row is held (not lost, not delivered), + # and no message has landed in their mailbox yet. + assert models.InboundMessage.objects.filter( + mailbox=recipient_mailbox, is_internal=True + ).exists() + assert not models.Message.objects.filter( + thread__accesses__mailbox=recipient_mailbox + ).exists() + + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_internal_delivery_skips_spam_scan(self, mock_session, mock_rspamd): + """Internal mail is trusted: the spam steps are skipped (rspamd is + never consulted) while the message still lands for the recipient.""" + sender_mailbox = factories.MailboxFactory() + recipient_mailbox = factories.MailboxFactory() + recipient_email = str(recipient_mailbox) + + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=recipient_mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.inbound"], + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + + message = self._build_internal_message(sender_mailbox, recipient_email) + outbound.send_message(message) + + mock_rspamd.assert_not_called() + assert models.Message.objects.filter( + thread__accesses__mailbox=recipient_mailbox + ).exists() + # Keep dj_timezone import used to silence "imported but unused" if the # linter wakes up after edits; it's referenced from fixtures via factories. @@ -1284,6 +1576,34 @@ def test_after_spam_is_spam_header( # --- response body parsing --- # +class TestReadCappedBody: + """``_read_capped_body`` bounds memory (size cap) and time (deadline) + so a hostile/slow receiver can't OOM or pin a worker.""" + + class _FakeResponse: + def __init__(self, chunks): + self._chunks = chunks + + def iter_content(self, chunk_size, decode_unicode): + yield from self._chunks + + def test_size_cap(self): + from core.mda.dispatch_webhooks import MAX_RESPONSE_BODY, _read_capped_body + + resp = self._FakeResponse([b"x" * (MAX_RESPONSE_BODY + 1000)]) + assert len(_read_capped_body(resp)) == MAX_RESPONSE_BODY + + def test_deadline_exceeded_raises(self): + import time + + from core.mda.dispatch_webhooks import _read_capped_body + + resp = self._FakeResponse([b"a", b"b"]) + # Deadline already in the past → first chunk trips the guard. + with pytest.raises(TimeoutError): + _read_capped_body(resp, deadline=time.monotonic() - 1) + + class TestClassifyResponseBody: """``_classify_response_body`` is the only thing that lets a receiver shape delivery beyond accept/drop. Cover the JSON contract @@ -1846,9 +2166,7 @@ def test_blocking_webhook_quarantine_delivers_flagged_after_window( # ...and forced to the inbox (is_spam=False) so the banner is seen. assert kwargs["is_spam"] is False # Queue row consumed — not pinned, not dropped silently. - assert not models.InboundMessage.objects.filter( - id=inbound_message.id - ).exists() + assert not models.InboundMessage.objects.filter(id=inbound_message.id).exists() @pytest.mark.django_db diff --git a/src/backend/core/tests/mda/test_inbound.py b/src/backend/core/tests/mda/test_inbound.py index 7195a03f3..ea5ae51a9 100644 --- a/src/backend/core/tests/mda/test_inbound.py +++ b/src/backend/core/tests/mda/test_inbound.py @@ -6,6 +6,7 @@ from django.utils import timezone import pytest +from jmap_email import parse_email from core import enums, factories, models from core.mda.inbound import deliver_inbound_message @@ -257,7 +258,7 @@ def test_basic_delivery_new_thread( assert models.Message.objects.count() == 0 success = deliver_inbound_message( - recipient_addr, sample_parsed_email, raw_email_data, skip_inbound_queue=True + recipient_addr, sample_parsed_email, raw_email_data, is_import=True ) assert success is True @@ -323,7 +324,7 @@ def test_basic_delivery_existing_thread( assert models.Message.objects.count() == 0 success = deliver_inbound_message( - recipient_addr, sample_parsed_email, raw_email_data, skip_inbound_queue=True + recipient_addr, sample_parsed_email, raw_email_data, is_import=True ) assert success is True @@ -346,7 +347,7 @@ def test_mailbox_creation_enabled(self, sample_parsed_email, raw_email_data): ).exists() success = deliver_inbound_message( - recipient_addr, sample_parsed_email, raw_email_data, skip_inbound_queue=True + recipient_addr, sample_parsed_email, raw_email_data, is_import=True ) assert success is True @@ -366,7 +367,7 @@ def test_mailbox_creation_disabled(self, sample_parsed_email, raw_email_data): ).exists() success = deliver_inbound_message( - recipient_addr, sample_parsed_email, raw_email_data, skip_inbound_queue=True + recipient_addr, sample_parsed_email, raw_email_data, is_import=True ) assert success is False @@ -395,7 +396,7 @@ def test_contact_creation( ).exists() success = deliver_inbound_message( - recipient_addr, sample_parsed_email, raw_email_data, skip_inbound_queue=True + recipient_addr, sample_parsed_email, raw_email_data, is_import=True ) assert success is True @@ -420,7 +421,7 @@ def test_invalid_sender_email_validation( ] success = deliver_inbound_message( - recipient_addr, sample_parsed_email, raw_email_data, skip_inbound_queue=True + recipient_addr, sample_parsed_email, raw_email_data, is_import=True ) assert success is True # Should still succeed using fallback @@ -439,7 +440,7 @@ def test_no_sender_email(self, target_mailbox, sample_parsed_email, raw_email_da del sample_parsed_email["from"] # Remove From header success = deliver_inbound_message( - recipient_addr, sample_parsed_email, raw_email_data, skip_inbound_queue=True + recipient_addr, sample_parsed_email, raw_email_data, is_import=True ) assert success is True @@ -466,7 +467,7 @@ def test_invalid_recipient_email_skipped( ] success = deliver_inbound_message( - recipient_addr, sample_parsed_email, raw_email_data, skip_inbound_queue=True + recipient_addr, sample_parsed_email, raw_email_data, is_import=True ) assert success is True # Delivery succeeds overall @@ -501,7 +502,7 @@ def test_email_exchange_single_thread(self): raw_email_1 = b"Raw for message 1" success1 = deliver_inbound_message( - addr2, parsed_email_1, raw_email_1, skip_inbound_queue=True + addr2, parsed_email_1, raw_email_1, is_import=True ) assert success1 is True assert models.Thread.objects.filter(accesses__mailbox=mailbox1).count() == 0 @@ -530,7 +531,7 @@ def test_email_exchange_single_thread(self): raw_email_2 = b"Raw for message 2" success2 = deliver_inbound_message( - addr1, parsed_email_2, raw_email_2, skip_inbound_queue=True + addr1, parsed_email_2, raw_email_2, is_import=True ) assert success2 is True assert models.Thread.objects.filter(accesses__mailbox=mailbox1).count() == 1 @@ -559,7 +560,7 @@ def test_email_exchange_single_thread(self): raw_email_3 = b"Raw for message 3" success3 = deliver_inbound_message( - addr2, parsed_email_3, raw_email_3, skip_inbound_queue=True + addr2, parsed_email_3, raw_email_3, is_import=True ) assert success3 is True # Counts should remain 1 thread per mailbox @@ -595,7 +596,7 @@ def test_deliver_message_with_empty_subject(self, target_mailbox, raw_email_data recipient_addr, parsed_email_empty_subject, raw_email_data, - skip_inbound_queue=True, + is_import=True, ) assert success is True @@ -632,7 +633,7 @@ def test_deliver_message_with_null_subject(self, target_mailbox, raw_email_data) recipient_addr, parsed_email_null_subject, raw_email_data, - skip_inbound_queue=True, + is_import=True, ) assert success is True @@ -671,7 +672,7 @@ def test_deliver_message_without_subject_field( recipient_addr, parsed_email_no_subject, raw_email_data, - skip_inbound_queue=True, + is_import=True, ) assert success is True @@ -712,7 +713,7 @@ def test_deliver_message_with_very_long_subject( recipient_addr, parsed_email_long_subject, raw_email_data, - skip_inbound_queue=True, + is_import=True, ) assert success is True assert models.Message.objects.count() == 1 @@ -747,7 +748,7 @@ def test_thread_subject_consistency_with_empty_subject( } success1 = deliver_inbound_message( - recipient_addr, parsed_email_1, raw_email_data, skip_inbound_queue=True + recipient_addr, parsed_email_1, raw_email_data, is_import=True ) assert success1 is True @@ -763,7 +764,7 @@ def test_thread_subject_consistency_with_empty_subject( } success2 = deliver_inbound_message( - recipient_addr, parsed_email_2, raw_email_data, skip_inbound_queue=True + recipient_addr, parsed_email_2, raw_email_data, is_import=True ) assert success2 is True @@ -818,7 +819,7 @@ def test_duplicate_recipients_handled_gracefully( recipient_addr, parsed_email_with_duplicates, raw_email_data, - skip_inbound_queue=True, + is_import=True, ) assert success is True @@ -890,24 +891,40 @@ def sample_parsed_email(self): } @patch("core.mda.autoreply.try_send_autoreply") - def test_autoreply_called_on_direct_delivery( - self, mock_try_autoreply, target_mailbox, sample_parsed_email + def test_autoreply_called_on_internal_delivery( + self, mock_try_autoreply, target_mailbox ): - """try_send_autoreply is called for skip_inbound_queue deliveries.""" + """try_send_autoreply fires for internal deliveries. + + Internal mail now runs through the inbound pipeline task, which + re-parses the raw bytes, so we feed real MIME (not a placeholder). + The autoreply is dispatched from the task after message creation. + """ recipient_addr = f"{target_mailbox.local_part}@{target_mailbox.domain.name}" + raw = ( + b"From: sender@test.com\r\n" + b"To: " + recipient_addr.encode() + b"\r\n" + b"Subject: Autoreply Test\r\n" + b"Message-ID: \r\n\r\nHello" + ) + # Internal delivery references the sender's committed blob, just + # like outbound.send_message does. + blob = models.Blob.objects.create_blob( + content=raw, content_type="message/rfc822" + ) result = deliver_inbound_message( recipient_addr, - sample_parsed_email, - b"raw data", - skip_inbound_queue=True, + parse_email(raw), + raw, + is_internal=True, + blob=blob, ) assert result is True mock_try_autoreply.assert_called_once() args = mock_try_autoreply.call_args[0] assert args[0] == target_mailbox - assert args[1] == sample_parsed_email assert isinstance(args[2], models.Message) @patch("core.mda.autoreply.try_send_autoreply") @@ -937,7 +954,7 @@ def test_autoreply_not_called_on_failed_delivery( "nonexistent@nonexistent-domain.invalid", sample_parsed_email, b"raw data", - skip_inbound_queue=True, + is_import=True, ) assert result is False diff --git a/src/backend/core/tests/mda/test_inbound_spoofed_sender.py b/src/backend/core/tests/mda/test_inbound_spoofed_sender.py index 0a7b7e7db..4baacc5d8 100644 --- a/src/backend/core/tests/mda/test_inbound_spoofed_sender.py +++ b/src/backend/core/tests/mda/test_inbound_spoofed_sender.py @@ -178,7 +178,7 @@ def test_inbound_spoofed_sender_legit_self_send_dedupes(self, victim_mailbox): assert ( deliver_inbound_message( - recipient, parsed_email, b"raw", skip_inbound_queue=True + recipient, parsed_email, b"raw", is_import=True ) is True ) diff --git a/src/backend/core/tests/models/test_blob.py b/src/backend/core/tests/models/test_blob.py index 9a5ec9656..cbead3ba2 100644 --- a/src/backend/core/tests/models/test_blob.py +++ b/src/backend/core/tests/models/test_blob.py @@ -5,7 +5,7 @@ import pytest -from core import enums, factories +from core import enums, factories, models @pytest.mark.django_db @@ -75,3 +75,49 @@ def test_blob_large_content_compression(self): compression_ratio < 0.1 ) # Should compress to less than 10% of original size assert blob.get_content() == content # Verify data integrity + + +@pytest.mark.django_db +class TestInboundMessageBlobReference: + """Internal mail parks the sender's blob on a transient InboundMessage + while the recipient pipeline runs. The GC must treat that as a live + reference, or it could reap the bytes out from under delivery.""" + + def test_inbound_message_counts_as_a_blob_reference(self): + """A blob referenced only by an InboundMessage survives GC, and + becomes collectable once that row is gone.""" + # pylint: disable-next=import-outside-toplevel + from core.services.blob_gc import gc_orphan_blobs_task + + mailbox = factories.MailboxFactory() + blob = models.Blob.objects.create_blob( + content=b"internal mime bytes", content_type="message/rfc822" + ) + inbound = models.InboundMessage.objects.create( + mailbox=mailbox, blob=blob, is_internal=True + ) + + # Referenced solely by the in-flight queue row → still alive. + assert models.Blob.objects.is_referenced(blob.id) is True + gc_orphan_blobs_task(mode="full") + assert models.Blob.objects.filter(id=blob.id).exists() + + # Queue row gone, nothing else references it → collectable. + inbound.delete() + assert models.Blob.objects.is_referenced(blob.id) is False + gc_orphan_blobs_task(mode="full") + assert not models.Blob.objects.filter(id=blob.id).exists() + + def test_exactly_one_source_constraint(self): + """An InboundMessage must carry raw_data XOR a blob, never both.""" + mailbox = factories.MailboxFactory() + blob = models.Blob.objects.create_blob( + content=b"bytes", content_type="message/rfc822" + ) + + # Both set → rejected (full_clean surfaces the check constraint + # as a ValidationError before the INSERT reaches the DB). + with pytest.raises(ValidationError): + models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=b"bytes", blob=blob + ) diff --git a/src/frontend/public/locales/common/en-US.json b/src/frontend/public/locales/common/en-US.json index 114efa0ac..9a15ae8bc 100755 --- a/src/frontend/public/locales/common/en-US.json +++ b/src/frontend/public/locales/common/en-US.json @@ -898,5 +898,8 @@ "You will no longer have access to the mailbox \"{{mailboxName}}\".": "You will no longer have access to the mailbox \"{{mailboxName}}\".", "Your email...": "Your email...", "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).": "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).", - "Your session has expired. Please log in again.": "Your session has expired. Please log in again." + "Your session has expired. Please log in again.": "Your session has expired. Please log in again.", + "URL must start with https://": "URL must start with https://", + "Regenerate secret": "Regenerate secret", + "Regenerating the secret invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Regenerating the secret invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again." } diff --git a/src/frontend/public/locales/common/fr-FR.json b/src/frontend/public/locales/common/fr-FR.json index 793a44616..60231b2e7 100755 --- a/src/frontend/public/locales/common/fr-FR.json +++ b/src/frontend/public/locales/common/fr-FR.json @@ -991,5 +991,8 @@ "You will no longer have access to the mailbox \"{{mailboxName}}\".": "Vous n'aurez plus accès à la boîte aux lettres « {{mailboxName}} ».", "Your email...": "Renseigner votre email...", "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).": "Votre point de terminaison recevra une charge utile JSON contenant le message analysé (expéditeur, destinataire, objet, corps, en-têtes, …).", - "Your session has expired. Please log in again.": "Votre session a expiré. Veuillez vous reconnecter." + "Your session has expired. Please log in again.": "Votre session a expiré. Veuillez vous reconnecter.", + "URL must start with https://": "L'URL doit commencer par https://", + "Regenerate secret": "Régénérer le secret", + "Regenerating the secret invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "La régénération du secret invalide immédiatement l'ancien. Le récepteur doit être mis à jour avec la nouvelle valeur avant de pouvoir vérifier à nouveau les webhooks." } diff --git a/src/frontend/public/locales/common/nl-NL.json b/src/frontend/public/locales/common/nl-NL.json index c714628e6..44c586383 100644 --- a/src/frontend/public/locales/common/nl-NL.json +++ b/src/frontend/public/locales/common/nl-NL.json @@ -564,5 +564,8 @@ "You have unsaved changes. Are you sure you want to close?": "Je hebt niet-opgeslagen wijzigingen. Weet je zeker dat je wil annuleren?", "You must confirm this statement.": "U moet deze verklaring bevestigen.", "Your email...": "Jouw email...", - "Your session has expired. Please log in again.": "Je sessie is verlopen. Log opnieuw in." + "Your session has expired. Please log in again.": "Je sessie is verlopen. Log opnieuw in.", + "URL must start with https://": "URL moet beginnen met https://", + "Regenerate secret": "Geheim opnieuw genereren", + "Regenerating the secret invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Het opnieuw genereren van het geheim maakt het oude direct ongeldig. De ontvanger moet met de nieuwe waarde worden bijgewerkt voordat webhooks weer geverifieerd kunnen worden." } diff --git a/src/frontend/public/locales/common/ru-RU.json b/src/frontend/public/locales/common/ru-RU.json index 8b042b2ce..13b6d8bd5 100644 --- a/src/frontend/public/locales/common/ru-RU.json +++ b/src/frontend/public/locales/common/ru-RU.json @@ -828,5 +828,43 @@ "You unassigned yourself": "Вы исключили себя из участников", "You were unassigned": "Вы исключены из числа участников", "Your email...": "Ваш e-mail...", - "Your session has expired. Please log in again.": "Время сеанса истекло. Пожалуйста, войдите снова." + "Your session has expired. Please log in again.": "Время сеанса истекло. Пожалуйста, войдите снова.", + "After spam check (recommended)": "После проверки на спам (рекомендуется)", + "API key in header — for receivers that can only check a static header value": "API-ключ в заголовке — для получателей, которые могут проверять только статическое значение заголовка", + "Authentication": "Аутентификация", + "Before spam check": "До проверки на спам", + "Behavior": "Поведение", + "Blocking — let this endpoint shape what happens to the message": "Блокирующий — позволить этому эндпоинту влиять на судьбу сообщения", + "Blocking lets the receiver act on this single message": "Блокирующий режим позволяет получателю воздействовать на это конкретное сообщение", + "Body posted to the endpoint. Envelope metadata is always sent as X-StMsg-* headers.": "Тело запроса, отправляемое на эндпоинт. Метаданные конверта всегда передаются в заголовках X-StMsg-*.", + "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.": "вернув тело JSON. Доступные действия (все применяются только к получаемому сообщению): отклонить / повторить доставку, переопределить вердикт о спаме, добавить метки, назначить пользователей по email, пометить как избранное / прочитанное / удалённое / архивированное, отключить автоответ, добавить внутренний комментарий к цепочке и создать черновик ответа из шаблона. Используйте блокирующий режим только с доверенными получателями.", + "Create a Webhook": "Создать вебхук", + "Done": "Готово", + "Each incoming message will be POSTed to this URL in the format selected below.": "Каждое входящее сообщение будет отправлено POST-запросом на этот URL в выбранном ниже формате.", + "Edit Webhook": "Редактировать вебхук", + "Endpoint": "Эндпоинт", + "Forward every incoming message to a URL of your choice as a JSON POST.": "Пересылайте каждое входящее сообщение на выбранный вами URL в виде JSON POST-запроса.", + "How the receiver authenticates our requests. The secret is shown once at creation.": "Как получатель аутентифицирует наши запросы. Секрет показывается один раз при создании.", + "JMAP Email JSON (RFC 8621)": "JMAP Email JSON (RFC 8621)", + "JMAP Email metadata (notification only)": "Метаданные JMAP Email (только уведомление)", + "Non-blocking (default) is the safe choice:": "Неблокирующий (по умолчанию) — безопасный выбор:", + "Outbound Webhook": "Исходящий вебхук", + "Payload format": "Формат полезной нагрузки", + "Raw .eml (message/rfc822)": "Исходный .eml (message/rfc822)", + "Save this credential now": "Сохраните эти учётные данные сейчас", + "Signed (HMAC + JWT) — recommended for receivers that can verify a signature": "Подписанный (HMAC + JWT) — рекомендуется для получателей, способных проверять подпись", + "This value is shown only once. Configure your receiver with it before closing — you can rotate it later if you need a new one.": "Это значение показывается только один раз. Настройте получателя с ним перед закрытием — позже вы сможете сгенерировать новое, если потребуется.", + "URL": "URL", + "URL is required.": "URL обязателен.", + "URL must start with http:// or https://": "URL должен начинаться с http:// или https://", + "we POST and ignore the response. Receivers cannot affect the message.": "мы отправляем POST-запрос и игнорируем ответ. Получатели не могут повлиять на сообщение.", + "Webhook": "Вебхук", + "Webhook API key": "API-ключ вебхука", + "Webhook signing secret": "Секрет подписи вебхука", + "When to fire": "Когда срабатывать", + "Whether the webhook fires before or after the message is checked for spam.": "Срабатывает ли вебхук до или после проверки сообщения на спам.", + "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).": "Ваш эндпоинт получит полезную нагрузку JSON с разобранным сообщением (отправитель, получатель, тема, тело, заголовки, …).", + "URL must start with https://": "URL должен начинаться с https://", + "Regenerate secret": "Сгенерировать новый секрет", + "Regenerating the secret invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Генерация нового секрета немедленно делает старый недействительным. Получателя необходимо обновить новым значением, прежде чем он снова сможет проверять вебхуки." } diff --git a/src/frontend/public/locales/common/uk-UA.json b/src/frontend/public/locales/common/uk-UA.json index e78507010..2eadc09cb 100644 --- a/src/frontend/public/locales/common/uk-UA.json +++ b/src/frontend/public/locales/common/uk-UA.json @@ -828,5 +828,43 @@ "You unassigned yourself": "Ви виключили себе з учасників", "You were unassigned": "Ви виключені з учасників", "Your email...": "Ваша ел. адреса...", - "Your session has expired. Please log in again.": "Ваш сеанс закінчився. Будь ласка, увійдіть знову." + "Your session has expired. Please log in again.": "Ваш сеанс закінчився. Будь ласка, увійдіть знову.", + "After spam check (recommended)": "Після перевірки на спам (рекомендовано)", + "API key in header — for receivers that can only check a static header value": "API-ключ у заголовку — для отримувачів, які можуть перевіряти лише статичне значення заголовка", + "Authentication": "Автентифікація", + "Before spam check": "До перевірки на спам", + "Behavior": "Поведінка", + "Blocking — let this endpoint shape what happens to the message": "Блокувальний — дозволити цьому ендпоінту впливати на долю повідомлення", + "Blocking lets the receiver act on this single message": "Блокувальний режим дозволяє отримувачу впливати на це конкретне повідомлення", + "Body posted to the endpoint. Envelope metadata is always sent as X-StMsg-* headers.": "Тіло запиту, що надсилається на ендпоінт. Метадані конверта завжди передаються в заголовках X-StMsg-*.", + "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.": "повернувши тіло JSON. Доступні дії (усі застосовуються лише до отримуваного повідомлення): відхилити / повторити доставку, перевизначити вердикт щодо спаму, додати мітки, призначити користувачів за email, позначити як обране / прочитане / видалене / архівоване, вимкнути автовідповідь, додати внутрішній коментар до ланцюжка та створити чернетку відповіді з шаблону. Використовуйте блокувальний режим лише з довіреними отримувачами.", + "Create a Webhook": "Створити вебхук", + "Done": "Готово", + "Each incoming message will be POSTed to this URL in the format selected below.": "Кожне вхідне повідомлення буде надіслано POST-запитом на цей URL у вибраному нижче форматі.", + "Edit Webhook": "Редагувати вебхук", + "Endpoint": "Ендпоінт", + "Forward every incoming message to a URL of your choice as a JSON POST.": "Пересилайте кожне вхідне повідомлення на вибраний вами URL у вигляді JSON POST-запиту.", + "How the receiver authenticates our requests. The secret is shown once at creation.": "Як отримувач автентифікує наші запити. Секрет показується один раз під час створення.", + "JMAP Email JSON (RFC 8621)": "JMAP Email JSON (RFC 8621)", + "JMAP Email metadata (notification only)": "Метадані JMAP Email (лише сповіщення)", + "Non-blocking (default) is the safe choice:": "Неблокувальний (за замовчуванням) — безпечний вибір:", + "Outbound Webhook": "Вихідний вебхук", + "Payload format": "Формат корисного навантаження", + "Raw .eml (message/rfc822)": "Необроблений .eml (message/rfc822)", + "Save this credential now": "Збережіть ці облікові дані зараз", + "Signed (HMAC + JWT) — recommended for receivers that can verify a signature": "Підписаний (HMAC + JWT) — рекомендовано для отримувачів, які можуть перевіряти підпис", + "This value is shown only once. Configure your receiver with it before closing — you can rotate it later if you need a new one.": "Це значення показується лише один раз. Налаштуйте отримувача з ним перед закриттям — пізніше ви зможете згенерувати нове, якщо буде потрібно.", + "URL": "URL", + "URL is required.": "URL є обов'язковим.", + "URL must start with http:// or https://": "URL має починатися з http:// або https://", + "we POST and ignore the response. Receivers cannot affect the message.": "ми надсилаємо POST-запит та ігноруємо відповідь. Отримувачі не можуть вплинути на повідомлення.", + "Webhook": "Вебхук", + "Webhook API key": "API-ключ вебхука", + "Webhook signing secret": "Секрет підпису вебхука", + "When to fire": "Коли спрацьовувати", + "Whether the webhook fires before or after the message is checked for spam.": "Чи спрацьовує вебхук до або після перевірки повідомлення на спам.", + "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).": "Ваш ендпоінт отримає корисне навантаження JSON з розібраним повідомленням (відправник, отримувач, тема, тіло, заголовки, …).", + "URL must start with https://": "URL має починатися з https://", + "Regenerate secret": "Згенерувати новий секрет", + "Regenerating the secret invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Генерація нового секрета негайно робить старий недійсним. Отримувача потрібно оновити новим значенням, перш ніж він знову зможе перевіряти вебхуки." } diff --git a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx index 8c68b92cb..0916883fa 100644 --- a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx +++ b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx @@ -8,9 +8,12 @@ import { useState, useMemo } from "react"; import { useQueryClient } from "@tanstack/react-query"; import { Channel, + ChannelCreateResponse, Mailbox, + RegeneratedSecretResponse, useMailboxesChannelsCreate, useMailboxesChannelsPartialUpdate, + useMailboxesChannelsRegenerateSecretCreate, getMailboxesChannelsListUrl, } from "@/features/api/gen"; import { @@ -21,8 +24,13 @@ import { import { addToast, ToasterItem } from "@/features/ui/components/toaster"; import { Banner } from "@/features/ui/components/banner"; import { CopyableInput } from "@/features/ui/components/copyable-input"; +import { useConfig } from "@/features/providers/config"; import { handle } from "@/features/utils/errors"; +// Environments where the backend (DEBUG=True) accepts http:// webhook +// URLs; everywhere else https is required, so mirror that client-side. +const DEV_ENVIRONMENTS = ["development", "developmentminimal", "e2e"]; + type WebhookChannelSettings = { url?: string; events?: string[]; @@ -44,14 +52,19 @@ type WebhookIntegrationFormProps = { onClose: () => void; }; -const createFormSchema = (t: (key: string) => string) => +const createFormSchema = ( + t: (key: string) => string, + allowInsecureUrl: boolean, +) => z.object({ name: z.string().min(1, { error: t("Name is required.") }), url: z .string() .min(1, { error: t("URL is required.") }) - .regex(/^https?:\/\//i, { - error: t("URL must start with http:// or https://"), + .regex(allowInsecureUrl ? /^https?:\/\//i : /^https:\/\//i, { + error: allowInsecureUrl + ? t("URL must start with http:// or https://") + : t("URL must start with https://"), }), phase: z.enum(["before_spam", "after_spam"]), format: z.enum(["eml", "jmap", "jmap_metadata"]), @@ -68,15 +81,21 @@ export const WebhookIntegrationForm = ({ onClose, }: WebhookIntegrationFormProps) => { const { t } = useTranslation(); + const config = useConfig(); const queryClient = useQueryClient(); const [error, setError] = useState(null); const settings = channel?.settings as WebhookChannelSettings | undefined; const isEditing = !!channel; + const allowInsecureUrl = DEV_ENVIRONMENTS.includes(config.ENVIRONMENT); const createMutation = useMailboxesChannelsCreate(); const updateMutation = useMailboxesChannelsPartialUpdate(); + const regenerateMutation = useMailboxesChannelsRegenerateSecretCreate(); - const formSchema = useMemo(() => createFormSchema(t), [t]); + const formSchema = useMemo( + () => createFormSchema(t, allowInsecureUrl), + [t, allowInsecureUrl], + ); const form = useForm({ resolver: zodResolver(formSchema), @@ -105,6 +124,46 @@ export const WebhookIntegrationForm = ({ }); }; + // Surface a freshly minted credential exactly once — the receiver + // needs it to verify every webhook we send. The backend returns + // ``secret`` for auth_method=jwt and ``api_key`` for api_key. Returns + // true when a credential was shown (so callers know not to close yet). + const showCredential = ( + source: { secret?: string; api_key?: string }, + ): boolean => { + if (source.secret) { + setCreatedCredential({ + label: t("Webhook signing secret"), + value: source.secret, + }); + return true; + } + if (source.api_key) { + setCreatedCredential({ + label: t("Webhook API key"), + value: source.api_key, + }); + return true; + } + return false; + }; + + const onRegenerate = async () => { + if (!channel) return; + setError(null); + try { + const response = await regenerateMutation.mutateAsync({ + mailboxId: mailbox.id, + id: channel.id, + }); + const data = response.data as RegeneratedSecretResponse; + showCredential(data); + } catch (err) { + handle(err); + setError(t("An error occurred while saving the integration.")); + } + }; + const onSubmit = async (data: FormFields) => { setError(null); @@ -149,31 +208,13 @@ export const WebhookIntegrationForm = ({ ); await invalidateChannels(); if (newChannel.status === 201) { - // Surface the freshly minted credential exactly - // once — the receiver needs this value to verify - // every webhook we send. The backend returns - // ``secret`` for auth_method=jwt and ``api_key`` for - // auth_method=api_key. These one-time credentials are - // create-only response fields, not part of the - // generated ``Channel`` type — read them off an - // index-signature view. - const payload = newChannel.data as unknown as Record< - string, - unknown - >; - const secret = payload.secret as string | undefined; - const apiKey = payload.api_key as string | undefined; - if (secret) { - setCreatedCredential({ - label: t("Webhook signing secret"), - value: secret, - }); - } else if (apiKey) { - setCreatedCredential({ - label: t("Webhook API key"), - value: apiKey, - }); - } else { + // The one-time credentials are create-only response + // fields surfaced via the generated + // ``ChannelCreateResponse`` view (the create hook types + // ``data`` as the plain ``Channel``, which omits them). + const payload = + newChannel.data as unknown as ChannelCreateResponse; + if (!showCredential(payload)) { onSuccess(newChannel.data); } } @@ -184,7 +225,9 @@ export const WebhookIntegrationForm = ({ } }; - if (createdCredential && !isEditing) { + // Shown after create OR after regenerating in edit mode — the new + // credential is only ever returned once. + if (createdCredential) { return (
    @@ -194,10 +237,14 @@ export const WebhookIntegrationForm = ({ "This value is shown only once. Configure your receiver with it before closing — you can rotate it later if you need a new one.", )} -
    + {isEditing && ( +
    +

    {t("Authentication")}

    + + {t( + "Regenerating the secret invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.", + )} + + +
    + )} + {error && {error}}
    From 19f0798ff1b4862b5b76ba7ad10ac31da3f8e257 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Sun, 28 Jun 2026 01:29:02 +0200 Subject: [PATCH 08/21] fix lint --- src/backend/core/factories.py | 6 +----- src/backend/core/tasks.py | 4 +++- src/backend/core/tests/api/test_channel_scope_level.py | 3 ++- src/backend/core/tests/mda/test_dispatch_webhooks.py | 8 +++++--- src/backend/core/tests/mda/test_inbound.py | 2 ++ src/backend/core/tests/mda/test_inbound_spoofed_sender.py | 4 +--- src/backend/core/tests/mda/test_spam_processing.py | 4 +--- 7 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/backend/core/factories.py b/src/backend/core/factories.py index d20ebd39f..9bd670101 100644 --- a/src/backend/core/factories.py +++ b/src/backend/core/factories.py @@ -363,11 +363,7 @@ class Meta: # fails-closed without one. Non-webhook channels keep an empty # encrypted_settings (matching production behaviour). encrypted_settings = factory.LazyAttribute( - lambda o: ( - {"secret": "whsec_factory_test"} - if o.type == "webhook" - else {} - ) + lambda o: {"secret": "whsec_factory_test"} if o.type == "webhook" else {} ) mailbox = factory.SubFactory(MailboxFactory) maildomain = None diff --git a/src/backend/core/tasks.py b/src/backend/core/tasks.py index 892dfa0a9..8a0be7b57 100644 --- a/src/backend/core/tasks.py +++ b/src/backend/core/tasks.py @@ -1,7 +1,9 @@ # pylint: disable=wildcard-import, unused-wildcard-import """Register all tasks here so that Celery autodiscovery can find them.""" -from core.mda.dispatch_webhooks import dispatch_webhook_task # noqa: F401 +from core.mda.dispatch_webhooks import ( # noqa: F401 # pylint: disable=unused-import + dispatch_webhook_task, +) from core.mda.inbound_tasks import * # noqa: F403 from core.mda.outbound_tasks import * # noqa: F403 from core.services.blob_gc import * # noqa: F403 diff --git a/src/backend/core/tests/api/test_channel_scope_level.py b/src/backend/core/tests/api/test_channel_scope_level.py index 1e8473bf7..8788705ee 100644 --- a/src/backend/core/tests/api/test_channel_scope_level.py +++ b/src/backend/core/tests/api/test_channel_scope_level.py @@ -1025,7 +1025,8 @@ def test_personal_webhook_channel(self, api_client): "type": "webhook", "settings": { "url": "https://hook.example.com/me", - "events": ["message.inbound"], "auth_method": "jwt", + "events": ["message.inbound"], + "auth_method": "jwt", }, }, format="json", diff --git a/src/backend/core/tests/mda/test_dispatch_webhooks.py b/src/backend/core/tests/mda/test_dispatch_webhooks.py index 467b325e3..1da9907a1 100644 --- a/src/backend/core/tests/mda/test_dispatch_webhooks.py +++ b/src/backend/core/tests/mda/test_dispatch_webhooks.py @@ -1465,7 +1465,9 @@ def _build_internal_message(self, sender_mailbox, recipient_email): @patch("core.mda.inbound_pipeline._call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_internal_delivery_fires_recipient_webhook(self, mock_session, mock_rspamd): + def test_internal_delivery_fires_recipient_webhook( + self, mock_session, _mock_rspamd + ): # Sender and recipient live on *different* domains — unrelated # tenants that happen to share the instance. sender_mailbox = factories.MailboxFactory() @@ -1501,7 +1503,7 @@ def test_internal_delivery_fires_recipient_webhook(self, mock_session, mock_rspa @patch("core.mda.inbound_pipeline._call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_recipient_webhook_failure_does_not_affect_sender( - self, mock_session, mock_rspamd + self, mock_session, _mock_rspamd ): """A failing/blocking recipient webhook is the recipient tenant's problem: it holds *their* queue row for retry but never feeds back @@ -1584,7 +1586,7 @@ class _FakeResponse: def __init__(self, chunks): self._chunks = chunks - def iter_content(self, chunk_size, decode_unicode): + def iter_content(self, chunk_size, decode_unicode): # pylint: disable=unused-argument yield from self._chunks def test_size_cap(self): diff --git a/src/backend/core/tests/mda/test_inbound.py b/src/backend/core/tests/mda/test_inbound.py index ea5ae51a9..9d78e67f1 100644 --- a/src/backend/core/tests/mda/test_inbound.py +++ b/src/backend/core/tests/mda/test_inbound.py @@ -1,5 +1,7 @@ """Tests for the core.mda.inbound module.""" +# pylint: disable=too-many-lines + from unittest.mock import patch from django.test import override_settings diff --git a/src/backend/core/tests/mda/test_inbound_spoofed_sender.py b/src/backend/core/tests/mda/test_inbound_spoofed_sender.py index 4baacc5d8..31e926553 100644 --- a/src/backend/core/tests/mda/test_inbound_spoofed_sender.py +++ b/src/backend/core/tests/mda/test_inbound_spoofed_sender.py @@ -177,9 +177,7 @@ def test_inbound_spoofed_sender_legit_self_send_dedupes(self, victim_mailbox): } assert ( - deliver_inbound_message( - recipient, parsed_email, b"raw", is_import=True - ) + deliver_inbound_message(recipient, parsed_email, b"raw", is_import=True) is True ) diff --git a/src/backend/core/tests/mda/test_spam_processing.py b/src/backend/core/tests/mda/test_spam_processing.py index 7a62e0c63..17869a3ac 100644 --- a/src/backend/core/tests/mda/test_spam_processing.py +++ b/src/backend/core/tests/mda/test_spam_processing.py @@ -869,9 +869,7 @@ class TestRspamdStepFailureHandling: def _ctx(self, spam_config): mailbox = factories.MailboxFactory() - inbound = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=b"raw" - ) + inbound = models.InboundMessage.objects.create(mailbox=mailbox, raw_data=b"raw") return InboundContext( mailbox=mailbox, inbound_message=inbound, From ac2bdc7db3024c456e88601cab49ba1144dc8259 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Sun, 28 Jun 2026 10:59:50 +0200 Subject: [PATCH 09/21] improve nonblocking webhooks --- docs/webhooks.md | 35 +++- src/backend/core/mda/dispatch_webhooks.py | 142 +++++++++---- src/backend/core/mda/inbound_pipeline.py | 16 +- src/backend/core/mda/inbound_tasks.py | 13 ++ .../core/tests/mda/test_dispatch_webhooks.py | 188 ++++++++++++------ .../tests/mda/test_inbound_spoofed_sender.py | 2 +- src/frontend/public/locales/common/en-US.json | 8 +- src/frontend/public/locales/common/fr-FR.json | 8 +- src/frontend/public/locales/common/nl-NL.json | 8 +- src/frontend/public/locales/common/ru-RU.json | 8 +- src/frontend/public/locales/common/uk-UA.json | 8 +- .../modal-compose-integration/index.tsx | 2 +- .../webhook-integration-form.tsx | 6 +- .../thread-event/assignment-message.test.ts | 29 ++- .../components/thread-event/index.tsx | 5 +- 15 files changed, 348 insertions(+), 130 deletions(-) diff --git a/docs/webhooks.md b/docs/webhooks.md index 2926c60c2..9d3e27124 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -26,9 +26,23 @@ A channel may also be **blocking** (`settings.blocking: true`). A blocking webhook gets to shape delivery: it can drop the message, ask to be retried later, or return a small JSON body that overrides the spam verdict and/or attaches labels to the resulting thread (see -[Response contract](#response-contract) below). Non-blocking webhooks -are fire-and-forget — failures are logged and the pipeline continues -unchanged. +[Response contract](#response-contract) below). Blocking webhooks run +**inline** at their phase, on the pipeline worker. + +Non-blocking webhooks are fire-and-forget — failures are logged and the +pipeline continues unchanged. Because they can't influence delivery, +they don't run on the pipeline worker at all: at their phase the channel +is just **recorded** (capturing the phase and the phase-time `is_spam`), +and the actual POST is handed to a background task **after the `Message` +is persisted**. The task renders the body from the stored message, so +nothing is copied or sent through the broker. Two consequences: + +* A non-blocking webhook fires only for messages that become a + `Message` — not for one a blocking webhook later **drops**. (Spam is + *not* a drop: it still lands in the spam folder, so it still fires, + with `X-StMsg-Is-Spam: true`.) +* The POST body is the canonical stored message regardless of phase; the + `phase`/`is_spam` context lives in the `X-StMsg-*` headers as before. ## Channel scopes @@ -62,7 +76,7 @@ A webhook channel stores its configuration in `Channel.settings` | Key | Type | Default | Description | | ------------- | -------- | -------------- | --------------------------------------------------------------------------- | -| `url` | string | **required** | `http://` or `https://` endpoint. Validated by the SSRF guard at each call. | +| `url` | string | **required** | `https://` endpoint, validated by the SSRF guard at each call. `http://` is accepted only when Django `DEBUG` is on (the local-dev escape hatch). | | `events` | string[] | **required** | Currently only `message.inbound` is implemented. | | `phase` | string | `after_spam` | `before_spam` or `after_spam`. | | `format` | string | `eml` | `eml`, `jmap`, or `jmap_metadata` (see [Payload formats](#payload-formats)). | @@ -136,10 +150,17 @@ message lands. | `X-StMsg-Mailbox` | Destination mailbox address | | `X-StMsg-Recipient` | Envelope `RCPT TO` (usually the same as `X-StMsg-Mailbox`) | | `X-StMsg-Is-Spam` | `true`, `false`, or `unknown` (`unknown` in the `before_spam` phase) | +| `X-StMsg-Message-Id` | UUID of the stored `Message` — **non-blocking only** (see note) | +| `X-StMsg-Thread-Id` | UUID of the `Message`'s `Thread` — **non-blocking only** | -The message-id is **not** sent as a header — every body format already -carries it (`messageId` in the JMAP variants, the raw `Message-ID:` -header in `eml`). +The MIME message-id is **not** sent as a header — every body format +already carries it (`messageId` in the JMAP variants, the raw +`Message-ID:` header in `eml`). + +`X-StMsg-Message-Id` / `X-StMsg-Thread-Id` are the platform's own ids +(for calling back into the API). They're only present on **non-blocking** +webhooks, which fire after the `Message` is persisted; blocking webhooks +run before it exists, so they can't carry them. ### Response contract diff --git a/src/backend/core/mda/dispatch_webhooks.py b/src/backend/core/mda/dispatch_webhooks.py index 094cd58e8..f8f6f8269 100644 --- a/src/backend/core/mda/dispatch_webhooks.py +++ b/src/backend/core/mda/dispatch_webhooks.py @@ -31,7 +31,6 @@ from __future__ import annotations -import base64 import hashlib import hmac import json @@ -47,7 +46,7 @@ from django.db.models import Q import jwt -from jmap_email import JmapEmail +from jmap_email import JmapEmail, parse_email from core import enums, models from core.mda.inbound_pipeline import Decision, InboundContext, Step @@ -155,7 +154,9 @@ def _read_capped_body(response, deadline: Optional[float] = None) -> bytes: except TimeoutError: raise except Exception as exc: - logger.warning("Truncated response body read failed: %s", exc) + # Don't interpolate ``exc`` — its text can echo the request URL or + # body and leak receiver secrets into logs. The type name is enough. + logger.warning("Truncated response body read failed (%s)", type(exc).__name__) return b"".join(chunks) @@ -530,20 +531,26 @@ def _envelope_headers( mailbox: models.Mailbox, recipient_email: str, is_spam: Optional[bool], + message: Optional[models.Message] = None, ) -> Dict[str, str]: """Build the ``X-StMsg-*`` envelope headers attached to every webhook POST regardless of body format. Same shape for ``eml`` and ``jmap``. - The message-id is intentionally *not* a header: every body format - already carries it (``messageId`` in the jmap variants, the raw - ``Message-ID:`` header in ``eml``), so a header would only duplicate - it. + The *MIME* message-id is intentionally *not* a header: every body + format already carries it (``messageId`` in the jmap variants, the + raw ``Message-ID:`` header in ``eml``), so a header would only + duplicate it. + + When ``message`` is supplied (the non-blocking path fires after the + ``Message`` is persisted) we add the platform's own ``Message`` / + ``Thread`` ids so a receiver can call back into our API. Blocking + webhooks fire before the row exists, so they don't carry them. """ if is_spam is None: spam_value = "unknown" else: spam_value = "true" if is_spam else "false" - return { + headers = { "User-Agent": USER_AGENT, "X-StMsg-Event": enums.WebhookEvents.MESSAGE_INBOUND.value, "X-StMsg-Phase": phase, @@ -552,6 +559,10 @@ def _envelope_headers( "X-StMsg-Recipient": recipient_email, "X-StMsg-Is-Spam": spam_value, } + if message is not None: + headers["X-StMsg-Message-Id"] = str(message.id) + headers["X-StMsg-Thread-Id"] = str(message.thread_id) + return headers # --- dispatch --- # @@ -590,34 +601,30 @@ def __init__(self, channel: models.Channel, phase: str): def __call__(self, ctx: InboundContext) -> Decision: cfg = self.channel.settings or {} - body_format = cfg.get("format", DEFAULT_FORMAT) blocking = bool(cfg.get("blocking", False)) + if not blocking: + # Non-blocking webhooks can't influence delivery, so they don't + # run network I/O on the inbound worker. We also don't render or + # snapshot the body here: we record the channel (with the + # phase-time ``is_spam``) and fire it AFTER the Message is + # created — see ``dispatch_recorded_webhooks``. The task then + # renders the payload from the durable ``Message.blob``, so the + # email bytes never get copied or pushed through the broker. + # + # Consequence (intended): a non-blocking webhook fires only for + # messages that actually become a Message — not for ones a + # blocking webhook later DROPs. The X-StMsg-* headers still carry + # the phase/spam context, and the body is the canonical stored + # MIME regardless of phase. + ctx.pending_webhooks.append((self.channel.id, self.phase, ctx.is_spam)) + return Decision.CONTINUE + + body_format = cfg.get("format", DEFAULT_FORMAT) content_type, body_bytes = _resolve_body( body_format, ctx.raw_data, ctx.parsed_email ) - if not blocking: - # Non-blocking webhooks can't influence delivery, so we never - # run their network I/O on the inbound worker — a slow/hostile - # receiver (the mailbox owner's own endpoint) would otherwise - # stall mail processing for everyone sharing the queue. Snapshot - # the phase-correct request and hand the POST to a Celery task - # on the default queue, then continue immediately. Fires via - # ``.delay`` (not ``on_commit``) to match the previous inline - # semantics: a non-blocking webhook fires regardless of whether - # the message is later dropped or its delivery rolls back. - dispatch_webhook_task.delay( - str(self.channel.id), - str(ctx.mailbox.id), - self.phase, - ctx.is_spam, - ctx.recipient_email, - content_type, - base64.b64encode(body_bytes).decode("ascii"), - ) - return Decision.CONTINUE - result = _dispatch_webhook( channel=self.channel, mailbox=ctx.mailbox, @@ -840,12 +847,15 @@ def _dispatch_webhook( content_type: str, body_bytes: bytes, blocking: bool, + message: Optional[models.Message] = None, ) -> _HttpResult: """Build the envelope headers and deliver one webhook. The shared entry point above ``_deliver_signed_webhook``: both the inline blocking step and the out-of-band non-blocking task land here, so the URL lookup and header-building can't drift between them. + ``message`` is set on the non-blocking path (fired post-persist) so + its id / thread id ride along as headers. """ url = (channel.settings or {}).get("url") if not url: @@ -860,6 +870,7 @@ def _dispatch_webhook( mailbox=mailbox, recipient_email=recipient_email, is_spam=is_spam, + message=message, ) return _deliver_signed_webhook( channel=channel, @@ -872,15 +883,52 @@ def _dispatch_webhook( ) +def _resolve_body_from_message( + body_format: str, message: models.Message +) -> Tuple[str, bytes]: + """Render the webhook body from a durable ``Message``. + + The non-blocking dispatch path sources its bytes from the stored + ``Message.blob`` (re-parsed the same way the pipeline parsed them) + instead of a transient snapshot, so there's no second copy of the + email. Mirrors ``_resolve_body``'s output contract. + """ + raw_data = message.blob.get_content() + parsed_email = parse_email(raw_data) or {} + return _resolve_body(body_format, raw_data, parsed_email) + + +def dispatch_recorded_webhooks( + message: models.Message, + mailbox: models.Mailbox, + pending: List[Tuple[Any, str, Optional[bool]]], +) -> None: + """Fire the non-blocking webhooks recorded during the pipeline. + + Called from the inbound finalizer once the ``Message`` exists and is + committed (the inbound task runs in autocommit). Each task receives + only ids — it re-fetches the message and renders the body from + ``Message.blob`` at run time, so nothing large rides the broker and + there's no payload snapshot to keep alive. If the message somehow + isn't there when the task runs, the task skips rather than guessing. + """ + if not pending: + return + message_id = str(message.id) + mailbox_id = str(mailbox.id) + for channel_id, phase, is_spam in pending: + dispatch_webhook_task.delay( + message_id, str(channel_id), mailbox_id, phase, is_spam + ) + + @celery_app.task def dispatch_webhook_task( + message_id: str, channel_id: str, mailbox_id: str, phase: str, is_spam: Optional[bool], - recipient_email: str, - content_type: str, - body_b64: str, ) -> None: """Deliver one non-blocking webhook off the inbound worker. @@ -891,21 +939,38 @@ def dispatch_webhook_task( the previous inline non-blocking contract). The request is re-signed here at send time, so the root secret never travels through the broker and the JWT TTL is measured from the actual POST. + + The payload never travels through the broker — only the ``message_id`` + does. We re-fetch (and so re-validate) the source ``Message`` at task + init and render the body from its blob; if the message is already gone + (e.g. deleted before the task ran) the dispatch is skipped rather than + guessed at. """ try: channel = models.Channel.objects.filter(id=channel_id).first() mailbox = models.Mailbox.objects.filter(id=mailbox_id).first() if channel is None or mailbox is None: return + message = models.Message.objects.filter(id=message_id).first() + if message is None or message.blob_id is None: + logger.warning( + "Webhook source message %s missing — skipping dispatch (channel=%s)", + message_id, + channel_id, + ) + return + body_format = (channel.settings or {}).get("format", DEFAULT_FORMAT) + content_type, body_bytes = _resolve_body_from_message(body_format, message) _dispatch_webhook( channel=channel, mailbox=mailbox, phase=phase, is_spam=is_spam, - recipient_email=recipient_email, + recipient_email=str(mailbox), content_type=content_type, - body_bytes=base64.b64decode(body_b64), + body_bytes=body_bytes, blocking=False, + message=message, ) except Exception: # pylint: disable=broad-exception-caught logger.exception( @@ -940,7 +1005,12 @@ def webhook_steps_for_mailbox( cfg = channel.settings or {} if cfg.get("phase", PHASE_AFTER_SPAM) != phase: continue - events = cfg.get("events") or [enums.WebhookEvents.MESSAGE_INBOUND.value] + events = cfg.get("events") + if events is None: + # Only a missing/null key falls back to the default. An explicit + # empty list means "subscribed to nothing" and must stay empty so + # the membership check below correctly skips the channel. + events = [enums.WebhookEvents.MESSAGE_INBOUND.value] if not isinstance(events, list): # Validator guarantees a list on write; a non-list here is a # misconfigured row. Fail closed — a bare string would make the diff --git a/src/backend/core/mda/inbound_pipeline.py b/src/backend/core/mda/inbound_pipeline.py index 01fa8b079..1f32813da 100644 --- a/src/backend/core/mda/inbound_pipeline.py +++ b/src/backend/core/mda/inbound_pipeline.py @@ -113,6 +113,16 @@ class InboundContext: # pylint: disable=too-many-instance-attributes # the autoreply path's shared record helper. pending_drafts: List[Tuple[Any, str]] = field(default_factory=list) + # Deferred non-blocking webhook dispatches. Each entry is + # ``(channel_id, phase, is_spam_at_phase)`` — recorded when the webhook + # step runs and fired AFTER the Message exists, so the task renders the + # payload from the durable ``Message`` (no transient snapshot, nothing + # large on the broker). Same record-now / fire-after-Message pattern + # push notifications use, so the two slot together cleanly. + pending_webhooks: List[Tuple[Any, str, Optional[bool]]] = field( + default_factory=list + ) + # Blocking-webhook flag actions (OR-merged across webhooks). All # default to False and are only ever flipped to True by a # receiver explicitly opting in via the JSON action body. The @@ -725,8 +735,10 @@ def apply_pending_assigns( # pre-filtered, so this shouldn't fire — but if a race # invalidated the rights between filter and service call, # don't blow up delivery over it. + # Log only the exception type — the message can embed assignee + # emails or names and leak user-identifying details into logs. logger.warning( - "assign_users skipped %d assignee(s) due to race: %s", + "assign_users skipped %d assignee(s) due to race (%s)", len(assignees_data), - exc, + type(exc).__name__, ) diff --git a/src/backend/core/mda/inbound_tasks.py b/src/backend/core/mda/inbound_tasks.py index 00a751042..b7cd4f375 100644 --- a/src/backend/core/mda/inbound_tasks.py +++ b/src/backend/core/mda/inbound_tasks.py @@ -18,6 +18,7 @@ from jmap_email import JmapEmail, first_address_email, parse_email from core import models +from core.mda.dispatch_webhooks import dispatch_recorded_webhooks from core.mda.inbound_create import _create_message_from_inbound from core.mda.inbound_pipeline import ( QUARANTINE_AFTER, @@ -266,6 +267,10 @@ def process_inbound_message_task(self, inbound_message_id: str): ) _stamp_processing_failed(ctx) quarantined = True + # The message is being forced to the inbox, so it is no longer + # treated as spam. Normalize ctx.is_spam so downstream consumers + # (autoreply gate, task result) agree with where it actually lands. + ctx.is_spam = False inbound_msg = _create_message_from_inbound( recipient_email=ctx.recipient_email, @@ -328,6 +333,14 @@ def process_inbound_message_task(self, inbound_message_id: str): mark_read=ctx.mark_read, ), ) + _safe_finalize( + "webhooks", + inbound_message_id, + ctx.pending_webhooks, + lambda: dispatch_recorded_webhooks( + inbound_msg, mailbox, ctx.pending_webhooks + ), + ) if isinstance(inbound_msg, models.Message) and not ctx.skip_autoreply: from core.mda.autoreply import ( # pylint: disable=import-outside-toplevel diff --git a/src/backend/core/tests/mda/test_dispatch_webhooks.py b/src/backend/core/tests/mda/test_dispatch_webhooks.py index 1da9907a1..8a206f458 100644 --- a/src/backend/core/tests/mda/test_dispatch_webhooks.py +++ b/src/backend/core/tests/mda/test_dispatch_webhooks.py @@ -4,12 +4,12 @@ # pylint: disable=missing-class-docstring,too-many-lines,too-many-public-methods # pylint: disable=use-implicit-booleaness-not-comparison -import base64 import hashlib import hmac import json import uuid from dataclasses import dataclass, field +from datetime import timedelta from typing import Optional, Set from unittest.mock import Mock, patch @@ -418,8 +418,10 @@ def test_skips_channel_without_matching_event( mock_session.assert_not_called() @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_non_blocking_continues_on_5xx(self, mock_session, mailbox, parsed_email): - """Non-blocking webhooks never influence delivery, even on 5xx.""" + def test_non_blocking_does_no_inline_io(self, mock_session, mailbox, parsed_email): + """A non-blocking webhook never influences delivery and never + touches the network on the inbound worker — it's recorded during + the pipeline and fired from a task after the Message exists.""" factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -429,7 +431,6 @@ def test_non_blocking_continues_on_5xx(self, mock_session, mailbox, parsed_email "blocking": False, }, ) - mock_session.return_value.post.return_value = _make_response(500) outcome = dispatch_webhooks( phase=PHASE_AFTER_SPAM, mailbox=mailbox, @@ -439,7 +440,7 @@ def test_non_blocking_continues_on_5xx(self, mock_session, mailbox, parsed_email is_spam=False, ) assert outcome.decision == Decision.CONTINUE - mock_session.return_value.post.assert_called_once() + mock_session.return_value.post.assert_not_called() @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_blocking_retries_on_5xx(self, mock_session, mailbox, parsed_email): @@ -560,30 +561,6 @@ def test_blocking_retries_on_ssrf_rejection( ) assert outcome.decision == Decision.RETRY - @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_non_blocking_continues_on_ssrf_rejection( - self, mock_session, mailbox, parsed_email - ): - factories.ChannelFactory( - type=enums.ChannelTypes.WEBHOOK, - mailbox=mailbox, - settings={ - "url": "https://internal.example.com", - "events": ["message.inbound"], - "blocking": False, - }, - ) - mock_session.return_value.post.side_effect = SSRFValidationError("blocked") - outcome = dispatch_webhooks( - phase=PHASE_AFTER_SPAM, - mailbox=mailbox, - recipient_email=str(mailbox), - parsed_email=parsed_email, - raw_data=b"raw", - is_spam=False, - ) - assert outcome.decision == Decision.CONTINUE - @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_blocking_retries_on_timeout(self, mock_session, mailbox, parsed_email): """A connection timeout is transient: retry rather than lose the message.""" @@ -672,6 +649,7 @@ def test_phase_filtering_dispatches_only_matching( "url": "https://hook.example.com/before", "events": ["message.inbound"], "phase": "before_spam", + "blocking": True, }, ) factories.ChannelFactory( @@ -681,6 +659,7 @@ def test_phase_filtering_dispatches_only_matching( "url": "https://hook.example.com/after", "events": ["message.inbound"], "phase": "after_spam", + "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -703,6 +682,7 @@ def test_eml_format_sends_raw_body(self, mock_session, mailbox, parsed_email): settings={ "url": "https://hook.example.com", "events": ["message.inbound"], + "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -730,6 +710,7 @@ def test_jmap_format_sends_jmap_email_json( "url": "https://hook.example.com", "events": ["message.inbound"], "format": "jmap", + "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -762,6 +743,7 @@ def test_jmap_metadata_skips_body_parts(self, mock_session, mailbox, parsed_emai "url": "https://hook.example.com", "events": ["message.inbound"], "format": "jmap_metadata", + "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -804,6 +786,7 @@ def test_envelope_headers_set_for_both_formats( "url": "https://hook.example.com", "events": ["message.inbound"], "format": fmt, + "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -836,6 +819,7 @@ def test_is_spam_header_unknown_when_none( "url": "https://hook.example.com", "events": ["message.inbound"], "phase": "before_spam", + "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -905,6 +889,7 @@ def test_eml_signature_covers_raw_body(self, mock_session, mailbox, parsed_email "url": "https://hook.example.com", "events": ["message.inbound"], "format": "eml", + "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -944,6 +929,7 @@ def test_jmap_signature_covers_exact_serialised_bytes( "url": "https://hook.example.com", "events": ["message.inbound"], "format": "jmap", + "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -983,6 +969,7 @@ def test_api_key_mode_sends_only_api_key_header( "url": "https://hook.example.com", "events": ["message.inbound"], "auth_method": "api_key", + "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -1018,6 +1005,7 @@ def test_jwt_mode_sends_only_hmac_and_jwt_headers( "url": "https://hook.example.com", "events": ["message.inbound"], "auth_method": "jwt", + "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -1078,6 +1066,7 @@ def test_api_key_value_is_derived_not_raw_secret( "url": "https://hook.example.com", "events": ["message.inbound"], "auth_method": "api_key", + "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -1263,6 +1252,7 @@ def test_after_spam_is_spam_header( "url": "https://hook.example.com", "events": ["message.inbound"], "phase": "after_spam", + "blocking": True, }, ) mock_check_spam.return_value = (True, None, None) @@ -1316,9 +1306,10 @@ def test_creation_failure_retries_then_abandons(self, mock_rspamd, mock_create): @pytest.mark.django_db class TestNonBlockingDispatch: - """Non-blocking webhooks must hand the POST to a Celery task (default - queue) rather than do network I/O on the inbound worker — that's the - worker-isolation guarantee.""" + """Non-blocking webhooks are recorded during the pipeline and fired + from a Celery task after the Message exists — the task renders the + payload from the durable ``Message.blob`` (no snapshot, nothing large + on the broker), keeping the inbound worker free of webhook I/O.""" def _ctx(self, mailbox, parsed_email, is_spam=False): return InboundContext( @@ -1331,11 +1322,12 @@ def _ctx(self, mailbox, parsed_email, is_spam=False): is_spam=is_spam, ) - @patch("core.mda.dispatch_webhooks.dispatch_webhook_task") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_non_blocking_enqueues_and_skips_inline_post( - self, mock_session, mock_task, mailbox, parsed_email + def test_non_blocking_records_pending_webhook( + self, mock_session, mailbox, parsed_email ): + # The step records the channel (with phase-time is_spam) and does + # NO network I/O — the actual send happens later from the task. channel = factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -1351,16 +1343,39 @@ def test_non_blocking_enqueues_and_skips_inline_post( for step in webhook_steps_for_mailbox(mailbox, phase=PHASE_AFTER_SPAM): assert step(ctx) == Decision.CONTINUE - # Enqueued exactly once with the phase-correct snapshot... - mock_task.delay.assert_called_once() - args = mock_task.delay.call_args[0] - assert args[0] == str(channel.id) - assert args[1] == str(mailbox.id) - assert args[2] == PHASE_AFTER_SPAM - assert args[3] is False # is_spam - # ...and NO network I/O happened on the inbound worker. + assert ctx.pending_webhooks == [(channel.id, PHASE_AFTER_SPAM, False)] mock_session.return_value.post.assert_not_called() + @patch("core.mda.dispatch_webhooks.dispatch_webhook_task") + def test_dispatch_recorded_webhooks_enqueues_one_task_each( + self, mock_task, mailbox + ): + from core.mda.dispatch_webhooks import dispatch_recorded_webhooks + + message = factories.MessageFactory(raw_mime=b"raw mime") + c1, c2 = uuid.uuid4(), uuid.uuid4() + dispatch_recorded_webhooks( + message, + mailbox, + [(c1, PHASE_AFTER_SPAM, False), (c2, PHASE_BEFORE_SPAM, None)], + ) + + assert mock_task.delay.call_count == 2 + assert mock_task.delay.call_args_list[0][0] == ( + str(message.id), + str(c1), + str(mailbox.id), + PHASE_AFTER_SPAM, + False, + ) + assert mock_task.delay.call_args_list[1][0] == ( + str(message.id), + str(c2), + str(mailbox.id), + PHASE_BEFORE_SPAM, + None, + ) + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_dispatch_webhook_task_posts_signed(self, mock_session, mailbox): channel = factories.ChannelFactory( @@ -1375,39 +1390,99 @@ def test_dispatch_webhook_task_posts_signed(self, mock_session, mailbox): ) mock_session.return_value.post.return_value = _make_response(200) + message = factories.MessageFactory(raw_mime=b"raw mime") dispatch_webhook_task( + str(message.id), str(channel.id), str(mailbox.id), PHASE_AFTER_SPAM, False, - str(mailbox), - "message/rfc822", - base64.b64encode(b"raw mime").decode("ascii"), ) mock_session.return_value.post.assert_called_once() + # The signed body is the message blob content, rendered at task init. + assert mock_session.return_value.post.call_args.kwargs["data"] == b"raw mime" headers = mock_session.return_value.post.call_args.kwargs["headers"] assert headers["X-StMsg-Phase"] == "after_spam" assert headers["X-StMsg-Event"] == enums.WebhookEvents.MESSAGE_INBOUND.value # Signed at send time (jwt auth_method). assert "X-StMsg-Webhook-Signature" in headers + # Fired post-persist, so the platform's Message/Thread ids ride along + # for receiver-side API callbacks. + assert headers["X-StMsg-Message-Id"] == str(message.id) + assert headers["X-StMsg-Thread-Id"] == str(message.thread_id) @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_dispatch_webhook_task_no_ops_when_channel_gone( self, mock_session, mailbox ): # Channel id that doesn't exist → best-effort no-op, no POST, no raise. + message = factories.MessageFactory(raw_mime=b"raw mime") + dispatch_webhook_task( + str(message.id), + str(uuid.uuid4()), + str(mailbox.id), + PHASE_AFTER_SPAM, + False, + ) + mock_session.return_value.post.assert_not_called() + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_dispatch_webhook_task_skips_when_message_gone(self, mock_session, mailbox): + # The source message is gone before the task ran (e.g. deleted) → + # re-validation at init fails closed: no POST, no guessed payload. + channel = factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.inbound"], + "blocking": False, + "auth_method": "jwt", + }, + ) dispatch_webhook_task( str(uuid.uuid4()), + str(channel.id), str(mailbox.id), PHASE_AFTER_SPAM, False, - str(mailbox), - "message/rfc822", - base64.b64encode(b"raw").decode("ascii"), ) mock_session.return_value.post.assert_not_called() + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_dispatch_webhook_task_swallows_send_errors(self, mock_session, mailbox): + # Non-blocking is fire-and-forget: a 5xx, a transport failure, or a + # receiver body must never surface as an exception or side effect. + channel = factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "events": ["message.inbound"], + "blocking": False, + "auth_method": "jwt", + }, + ) + message = factories.MessageFactory(raw_mime=b"raw mime") + args = ( + str(message.id), + str(channel.id), + str(mailbox.id), + PHASE_AFTER_SPAM, + False, + ) + + # 5xx with a drop-shaped body — ignored, no raise. + mock_session.return_value.post.return_value = _make_response( + 500, body=b'{"action": "drop"}' + ) + dispatch_webhook_task(*args) + + # Transport failure — swallowed, no raise. + mock_session.return_value.post.side_effect = SSRFValidationError("blocked") + dispatch_webhook_task(*args) + # --- internal (mailbox-to-mailbox) delivery --- # @@ -1919,9 +1994,10 @@ def test_blocking_is_spam_override_continues( def test_non_blocking_ignores_action_body( self, mock_session, mailbox, parsed_email ): - """Non-blocking webhooks are fire-and-forget. A receiver's body - should never affect delivery — protects against a non-blocking - webhook accidentally returning {"action":"drop"}.""" + """Non-blocking webhooks are fire-and-forget. They're not in the + delivery decision path at all — the step only records them and does + no inline I/O — so a receiver's body (even ``{"action":"drop"}``) + can never affect delivery.""" factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -1931,9 +2007,6 @@ def test_non_blocking_ignores_action_body( "blocking": False, }, ) - mock_session.return_value.post.return_value = _make_response( - 200, body=b'{"action": "drop", "is_spam": true}' - ) outcome = dispatch_webhooks( phase=PHASE_AFTER_SPAM, mailbox=mailbox, @@ -1944,6 +2017,7 @@ def test_non_blocking_ignores_action_body( ) assert outcome.decision == Decision.CONTINUE assert outcome.is_spam_override is None + mock_session.return_value.post.assert_not_called() @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_multi_webhook_drop_wins_and_short_circuits( @@ -2136,9 +2210,7 @@ def test_blocking_webhook_quarantine_delivers_flagged_after_window( ) # Backdate past the quarantine window (auto_now_add → update()). models.InboundMessage.objects.filter(id=inbound_message.id).update( - created_at=dj_timezone.now() - - QUARANTINE_AFTER - - dj_timezone.timedelta(minutes=1) + created_at=dj_timezone.now() - QUARANTINE_AFTER - timedelta(minutes=1) ) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, diff --git a/src/backend/core/tests/mda/test_inbound_spoofed_sender.py b/src/backend/core/tests/mda/test_inbound_spoofed_sender.py index 31e926553..aeab9239f 100644 --- a/src/backend/core/tests/mda/test_inbound_spoofed_sender.py +++ b/src/backend/core/tests/mda/test_inbound_spoofed_sender.py @@ -177,7 +177,7 @@ def test_inbound_spoofed_sender_legit_self_send_dedupes(self, victim_mailbox): } assert ( - deliver_inbound_message(recipient, parsed_email, b"raw", is_import=True) + deliver_inbound_message(recipient, parsed_email, b"raw", is_internal=True) is True ) diff --git a/src/frontend/public/locales/common/en-US.json b/src/frontend/public/locales/common/en-US.json index 9a15ae8bc..60a2f6e84 100755 --- a/src/frontend/public/locales/common/en-US.json +++ b/src/frontend/public/locales/common/en-US.json @@ -425,7 +425,7 @@ "Forced": "Forced", "Forced signature": "Forced signature", "Forward": "Forward", - "Forward every incoming message to a URL of your choice as a JSON POST.": "Forward every incoming message to a URL of your choice as a JSON POST.", + "Forward every incoming message to a URL of your choice.": "Forward every incoming message to a URL of your choice.", "Forwarded message": "Forwarded message", "Friday": "Friday", "From": "From", @@ -439,7 +439,7 @@ "Grant editor access to the thread?": "Grant editor access to the thread?", "Help center & Support": "Help center & Support", "High failure rate": "High failure rate", - "How the receiver authenticates our requests. The secret is shown once at creation.": "How the receiver authenticates our requests. The secret is shown once at creation.", + "How the receiver authenticates our requests. The credential is shown once at creation.": "How the receiver authenticates our requests. The credential is shown once at creation.", "How to allow IMAP connections from your account {{name}}?": "How to allow IMAP connections from your account {{name}}?", "I confirm that this address corresponds to the real identity of a colleague, and I commit to deactivating it when their position ends.": "I confirm that this address corresponds to the real identity of a colleague, and I commit to deactivating it when their position ends.", "I have an issue or a feature request": "I have an issue or a feature request", @@ -900,6 +900,6 @@ "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).": "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).", "Your session has expired. Please log in again.": "Your session has expired. Please log in again.", "URL must start with https://": "URL must start with https://", - "Regenerate secret": "Regenerate secret", - "Regenerating the secret invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Regenerating the secret invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again." + "Regenerate credential": "Regenerate credential", + "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again." } diff --git a/src/frontend/public/locales/common/fr-FR.json b/src/frontend/public/locales/common/fr-FR.json index 60231b2e7..bf0a6ec84 100755 --- a/src/frontend/public/locales/common/fr-FR.json +++ b/src/frontend/public/locales/common/fr-FR.json @@ -502,7 +502,7 @@ "Forced": "Forcée", "Forced signature": "Signature forcée", "Forward": "Transférer", - "Forward every incoming message to a URL of your choice as a JSON POST.": "Transférez chaque message entrant vers une URL de votre choix via un POST JSON.", + "Forward every incoming message to a URL of your choice.": "Transférez chaque message entrant vers une URL de votre choix.", "Forwarded message": "Message transféré", "Friday": "Vendredi", "From": "De", @@ -516,7 +516,7 @@ "Grant editor access to the thread?": "Accorder l'accès en édition à la conversation ?", "Help center & Support": "Centre d'aide et Support", "High failure rate": "Taux d'échec élevé", - "How the receiver authenticates our requests. The secret is shown once at creation.": "Comment le destinataire authentifie nos requêtes. Le secret est affiché une seule fois lors de la création.", + "How the receiver authenticates our requests. The credential is shown once at creation.": "Comment le destinataire authentifie nos requêtes. L'identifiant est affiché une seule fois lors de la création.", "How to allow IMAP connections from your account {{name}}?": "Comment autoriser les connexions IMAP depuis votre compte {{name}} ?", "I confirm that this address corresponds to the real identity of a colleague, and I commit to deactivating it when their position ends.": "Je confirme que cette adresse correspond à l'identité d'une personne physique travaillant avec moi, et m'engage à la désactiver quand son poste prendra fin.", "I have an issue or a feature request": "J'ai un problème ou une demande d'amélioration", @@ -993,6 +993,6 @@ "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).": "Votre point de terminaison recevra une charge utile JSON contenant le message analysé (expéditeur, destinataire, objet, corps, en-têtes, …).", "Your session has expired. Please log in again.": "Votre session a expiré. Veuillez vous reconnecter.", "URL must start with https://": "L'URL doit commencer par https://", - "Regenerate secret": "Régénérer le secret", - "Regenerating the secret invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "La régénération du secret invalide immédiatement l'ancien. Le récepteur doit être mis à jour avec la nouvelle valeur avant de pouvoir vérifier à nouveau les webhooks." + "Regenerate credential": "Régénérer l'identifiant", + "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "La régénération de l'identifiant invalide immédiatement l'ancien. Le récepteur doit être mis à jour avec la nouvelle valeur avant de pouvoir vérifier à nouveau les webhooks." } diff --git a/src/frontend/public/locales/common/nl-NL.json b/src/frontend/public/locales/common/nl-NL.json index 44c586383..8e93f8e34 100644 --- a/src/frontend/public/locales/common/nl-NL.json +++ b/src/frontend/public/locales/common/nl-NL.json @@ -13,8 +13,8 @@ "Each incoming message will be POSTed to this URL in the format selected below.": "Elk binnenkomend bericht wordt via POST naar deze URL verzonden in het hieronder geselecteerde formaat.", "Edit Webhook": "Webhook bewerken", "Endpoint": "Endpoint", - "Forward every incoming message to a URL of your choice as a JSON POST.": "Stuur elk binnenkomend bericht door naar een URL naar keuze als een JSON POST.", - "How the receiver authenticates our requests. The secret is shown once at creation.": "Hoe de ontvanger onze verzoeken authenticeert. Het geheim wordt eenmalig getoond bij het aanmaken.", + "Forward every incoming message to a URL of your choice.": "Stuur elk binnenkomend bericht door naar een URL naar keuze.", + "How the receiver authenticates our requests. The credential is shown once at creation.": "Hoe de ontvanger onze verzoeken authenticeert. Het geheim wordt eenmalig getoond bij het aanmaken.", "JMAP Email JSON (RFC 8621)": "JMAP Email JSON (RFC 8621)", "JMAP Email metadata (notification only)": "JMAP Email-metadata (alleen melding)", "Non-blocking (default) is the safe choice:": "Niet-blokkerend (standaard) is de veilige keuze:", @@ -566,6 +566,6 @@ "Your email...": "Jouw email...", "Your session has expired. Please log in again.": "Je sessie is verlopen. Log opnieuw in.", "URL must start with https://": "URL moet beginnen met https://", - "Regenerate secret": "Geheim opnieuw genereren", - "Regenerating the secret invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Het opnieuw genereren van het geheim maakt het oude direct ongeldig. De ontvanger moet met de nieuwe waarde worden bijgewerkt voordat webhooks weer geverifieerd kunnen worden." + "Regenerate credential": "Geheim opnieuw genereren", + "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Het opnieuw genereren van het geheim maakt het oude direct ongeldig. De ontvanger moet met de nieuwe waarde worden bijgewerkt voordat webhooks weer geverifieerd kunnen worden." } diff --git a/src/frontend/public/locales/common/ru-RU.json b/src/frontend/public/locales/common/ru-RU.json index 13b6d8bd5..9f48cada3 100644 --- a/src/frontend/public/locales/common/ru-RU.json +++ b/src/frontend/public/locales/common/ru-RU.json @@ -843,8 +843,8 @@ "Each incoming message will be POSTed to this URL in the format selected below.": "Каждое входящее сообщение будет отправлено POST-запросом на этот URL в выбранном ниже формате.", "Edit Webhook": "Редактировать вебхук", "Endpoint": "Эндпоинт", - "Forward every incoming message to a URL of your choice as a JSON POST.": "Пересылайте каждое входящее сообщение на выбранный вами URL в виде JSON POST-запроса.", - "How the receiver authenticates our requests. The secret is shown once at creation.": "Как получатель аутентифицирует наши запросы. Секрет показывается один раз при создании.", + "Forward every incoming message to a URL of your choice.": "Пересылайте каждое входящее сообщение на выбранный вами URL.", + "How the receiver authenticates our requests. The credential is shown once at creation.": "Как получатель аутентифицирует наши запросы. Секрет показывается один раз при создании.", "JMAP Email JSON (RFC 8621)": "JMAP Email JSON (RFC 8621)", "JMAP Email metadata (notification only)": "Метаданные JMAP Email (только уведомление)", "Non-blocking (default) is the safe choice:": "Неблокирующий (по умолчанию) — безопасный выбор:", @@ -865,6 +865,6 @@ "Whether the webhook fires before or after the message is checked for spam.": "Срабатывает ли вебхук до или после проверки сообщения на спам.", "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).": "Ваш эндпоинт получит полезную нагрузку JSON с разобранным сообщением (отправитель, получатель, тема, тело, заголовки, …).", "URL must start with https://": "URL должен начинаться с https://", - "Regenerate secret": "Сгенерировать новый секрет", - "Regenerating the secret invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Генерация нового секрета немедленно делает старый недействительным. Получателя необходимо обновить новым значением, прежде чем он снова сможет проверять вебхуки." + "Regenerate credential": "Сгенерировать новый секрет", + "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Генерация нового секрета немедленно делает старый недействительным. Получателя необходимо обновить новым значением, прежде чем он снова сможет проверять вебхуки." } diff --git a/src/frontend/public/locales/common/uk-UA.json b/src/frontend/public/locales/common/uk-UA.json index 2eadc09cb..661bcee04 100644 --- a/src/frontend/public/locales/common/uk-UA.json +++ b/src/frontend/public/locales/common/uk-UA.json @@ -843,8 +843,8 @@ "Each incoming message will be POSTed to this URL in the format selected below.": "Кожне вхідне повідомлення буде надіслано POST-запитом на цей URL у вибраному нижче форматі.", "Edit Webhook": "Редагувати вебхук", "Endpoint": "Ендпоінт", - "Forward every incoming message to a URL of your choice as a JSON POST.": "Пересилайте кожне вхідне повідомлення на вибраний вами URL у вигляді JSON POST-запиту.", - "How the receiver authenticates our requests. The secret is shown once at creation.": "Як отримувач автентифікує наші запити. Секрет показується один раз під час створення.", + "Forward every incoming message to a URL of your choice.": "Пересилайте кожне вхідне повідомлення на вибраний вами URL.", + "How the receiver authenticates our requests. The credential is shown once at creation.": "Як отримувач автентифікує наші запити. Секрет показується один раз під час створення.", "JMAP Email JSON (RFC 8621)": "JMAP Email JSON (RFC 8621)", "JMAP Email metadata (notification only)": "Метадані JMAP Email (лише сповіщення)", "Non-blocking (default) is the safe choice:": "Неблокувальний (за замовчуванням) — безпечний вибір:", @@ -865,6 +865,6 @@ "Whether the webhook fires before or after the message is checked for spam.": "Чи спрацьовує вебхук до або після перевірки повідомлення на спам.", "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).": "Ваш ендпоінт отримає корисне навантаження JSON з розібраним повідомленням (відправник, отримувач, тема, тіло, заголовки, …).", "URL must start with https://": "URL має починатися з https://", - "Regenerate secret": "Згенерувати новий секрет", - "Regenerating the secret invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Генерація нового секрета негайно робить старий недійсним. Отримувача потрібно оновити новим значенням, перш ніж він знову зможе перевіряти вебхуки." + "Regenerate credential": "Згенерувати новий секрет", + "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Генерація нового секрета негайно робить старий недійсним. Отримувача потрібно оновити новим значенням, перш ніж він знову зможе перевіряти вебхуки." } diff --git a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/index.tsx b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/index.tsx index 6decf8225..fad1fd345 100644 --- a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/index.tsx +++ b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/index.tsx @@ -53,7 +53,7 @@ const CHANNEL_TYPE_METADATA: Record = { webhook: { type: "webhook", title: i18n.t("Outbound Webhook"), - description: i18n.t("Forward every incoming message to a URL of your choice as a JSON POST."), + description: i18n.t("Forward every incoming message to a URL of your choice."), icon: "webhook", }, }; diff --git a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx index 0916883fa..e049c49b0 100644 --- a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx +++ b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx @@ -312,7 +312,7 @@ export const WebhookIntegrationForm = ({ }, ]} text={t( - "How the receiver authenticates our requests. The secret is shown once at creation.", + "How the receiver authenticates our requests. The credential is shown once at creation.", )} fullWidth /> @@ -395,7 +395,7 @@ export const WebhookIntegrationForm = ({

    {t("Authentication")}

    {t( - "Regenerating the secret invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.", + "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.", )}
    )} diff --git a/src/frontend/src/features/layouts/components/thread-view/components/thread-event/assignment-message.test.ts b/src/frontend/src/features/layouts/components/thread-view/components/thread-event/assignment-message.test.ts index 45ad78752..ed53e70a0 100644 --- a/src/frontend/src/features/layouts/components/thread-view/components/thread-event/assignment-message.test.ts +++ b/src/frontend/src/features/layouts/components/thread-view/components/thread-event/assignment-message.test.ts @@ -19,6 +19,10 @@ const makeEvent = ( type: 'assign' | 'unassign', authorId: string | null, assignees: { id: string; name: string }[], + // Override author_display independently of authorId so tests can build the + // webhook/channel actor shape (author === null but author_display set), which + // buildAssignmentMessage special-cases as a named third party. + authorDisplay?: string | null, ): AssignmentEvent => ({ id: 'e', thread: 't', @@ -27,7 +31,10 @@ const makeEvent = ( author: authorId === null ? (null as unknown as ThreadEvent['author']) : ({ id: authorId, full_name: `User ${authorId}`, email: `${authorId}@ex.com` } as ThreadEvent['author']), - author_display: authorId === null ? null : `User ${authorId}`, + author_display: + authorDisplay !== undefined + ? authorDisplay + : authorId === null ? null : `User ${authorId}`, data: { assignees }, has_unread_mention: false, is_editable: false, @@ -107,6 +114,26 @@ describe('buildAssignmentMessage', () => { expect(buildAssignmentMessage(event, SELF_ID, fakeT)).toBe('User alice unassigned themself'); }); + it('webhook actor (author null, author_display set) assigns others', () => { + const event = makeEvent( + ThreadEventTypeEnum.assign, + null, + [{ id: 'bob', name: 'Bob' }], + 'Webhook: CRM', + ); + expect(buildAssignmentMessage(event, SELF_ID, fakeT)).toBe('Webhook: CRM assigned Bob'); + }); + + it('webhook actor (author null, author_display set) assigns the current user', () => { + const event = makeEvent( + ThreadEventTypeEnum.assign, + null, + [{ id: SELF_ID, name: 'Me' }], + 'Webhook: CRM', + ); + expect(buildAssignmentMessage(event, SELF_ID, fakeT)).toBe('Webhook: CRM assigned you'); + }); + it('K — system unassigned the current user', () => { const event = makeEvent(ThreadEventTypeEnum.unassign, null, [{ id: SELF_ID, name: 'Me' }]); expect(buildAssignmentMessage(event, SELF_ID, fakeT)).toBe('You were unassigned'); diff --git a/src/frontend/src/features/layouts/components/thread-view/components/thread-event/index.tsx b/src/frontend/src/features/layouts/components/thread-view/components/thread-event/index.tsx index 6abf788c4..2f76b4369 100755 --- a/src/frontend/src/features/layouts/components/thread-view/components/thread-event/index.tsx +++ b/src/frontend/src/features/layouts/components/thread-view/components/thread-event/index.tsx @@ -218,7 +218,10 @@ export const ThreadEvent = ({ event, isCondensed = false, onEdit, onDelete, ment }; if (isIMEvent(event)) { - const authorName = event.author?.full_name || event.author?.email || ""; + // Prefer ``author_display`` so webhook/channel-authored events (which + // have no ``author``) get a stable color from their displayed name + // instead of falling back to an empty string. + const authorName = event.author_display || event.author?.full_name || event.author?.email || ""; const avatarColor = getAvatarColor(authorName); const isMentioned = user ? event.data?.mentions?.map((m) => m.id)?.includes(user.id) From 869f88f05c57f7849416dde3c376db1478056e95 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Sun, 28 Jun 2026 13:45:06 +0200 Subject: [PATCH 10/21] fix lint --- src/backend/core/mda/dispatch_webhooks.py | 77 +------------------- src/backend/core/mda/webhook_payload.py | 86 +++++++++++++++++++++++ 2 files changed, 87 insertions(+), 76 deletions(-) create mode 100644 src/backend/core/mda/webhook_payload.py diff --git a/src/backend/core/mda/dispatch_webhooks.py b/src/backend/core/mda/dispatch_webhooks.py index f8f6f8269..fa03740f6 100644 --- a/src/backend/core/mda/dispatch_webhooks.py +++ b/src/backend/core/mda/dispatch_webhooks.py @@ -39,7 +39,6 @@ import time import uuid as uuid_module from dataclasses import dataclass, field -from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Set, Tuple from urllib.parse import urlparse @@ -50,6 +49,7 @@ from core import enums, models from core.mda.inbound_pipeline import Decision, InboundContext, Step +from core.mda.webhook_payload import build_jmap_email from core.services.ssrf import SSRFSafeSession, SSRFValidationError from messages.celery_app import app as celery_app @@ -446,81 +446,6 @@ def find_webhook_channels_for_mailbox( ) -# --- payload builders --- # - - -def _utcdate(value: Any) -> Optional[str]: - """Format a datetime as a JMAP ``UTCDate`` (RFC 3339 with ``Z`` suffix). - - Falls back to the raw value if it isn't a datetime — the parser may - have given us a pre-formatted string. ``None`` stays ``None``. - """ - if isinstance(value, datetime): - return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - return value - - -def _strip_body_part(part: Dict[str, Any]) -> Dict[str, Any]: - """A JMAP ``EmailBodyPart`` without our parser's project extensions. - - ``content`` (raw bytes on attachment parts) and ``sha256`` are - parser extensions, NOT RFC 8621. Attachment bytes in particular are - never embedded in the JSON body — JMAP keeps them behind a - ``blobId`` (which we don't have at webhook-fire time), and raw bytes - aren't JSON-serialisable anyway. Receivers that need the bytes use - ``format=eml``. - """ - return {k: v for k, v in part.items() if k not in ("content", "sha256")} - - -def build_jmap_email( - parsed_email: JmapEmail, *, include_body: bool = True -) -> Dict[str, Any]: - """Project the parsed JMAP ``Email`` object into the webhook payload. - - ``parse_email`` already returns a strict JMAP Email object - (RFC 8621 §4.1), so this is mostly a copy. We stamp ``receivedAt`` - (the moment the webhook fires) and strip the parser's project - extensions (``_ext`` and per-part ``content`` / ``sha256``) so the - body is strict JMAP. Storage-time fields (``id``, ``blobId``, - ``threadId``, ``mailboxIds``, ``keywords``) don't exist at - webhook-fire time and are simply absent. - - With ``include_body=False`` the body parts, ``bodyValues`` and - ``attachments`` are dropped — receivers get a notification-only - payload (subject + envelope addresses + headers) without the - message body content ever leaving the instance over the wire. - ``hasAttachment`` is preserved so receivers can still tell whether - the message had any. - """ - email: Dict[str, Any] = dict(parsed_email) - email.pop("_ext", None) # project extension, not strict JMAP - email["receivedAt"] = _utcdate(datetime.now(timezone.utc)) - - if include_body: - email["textBody"] = [ - _strip_body_part(p) for p in parsed_email.get("textBody") or [] - ] - email["htmlBody"] = [ - _strip_body_part(p) for p in parsed_email.get("htmlBody") or [] - ] - email["attachments"] = [ - _strip_body_part(p) for p in parsed_email.get("attachments") or [] - ] - else: - for key in ( - "textBody", - "htmlBody", - "bodyValues", - "bodyStructure", - "attachments", - "preview", - ): - email.pop(key, None) - - return email - - # --- envelope headers --- # diff --git a/src/backend/core/mda/webhook_payload.py b/src/backend/core/mda/webhook_payload.py new file mode 100644 index 000000000..7faaee156 --- /dev/null +++ b/src/backend/core/mda/webhook_payload.py @@ -0,0 +1,86 @@ +"""JMAP payload builders for outbound webhooks. + +Projects the parsed JMAP ``Email`` object (RFC 8621 §4.1) into the +webhook wire payload. Kept separate from ``dispatch_webhooks`` (the HTTP +plumbing / signing / dispatch glue) because it's a pure data +transformation with no network or model dependencies. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Dict, Optional + +from jmap_email import JmapEmail + + +def _utcdate(value: Any) -> Optional[str]: + """Format a datetime as a JMAP ``UTCDate`` (RFC 3339 with ``Z`` suffix). + + Falls back to the raw value if it isn't a datetime — the parser may + have given us a pre-formatted string. ``None`` stays ``None``. + """ + if isinstance(value, datetime): + return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + return value + + +def _strip_body_part(part: Dict[str, Any]) -> Dict[str, Any]: + """A JMAP ``EmailBodyPart`` without our parser's project extensions. + + ``content`` (raw bytes on attachment parts) and ``sha256`` are + parser extensions, NOT RFC 8621. Attachment bytes in particular are + never embedded in the JSON body — JMAP keeps them behind a + ``blobId`` (which we don't have at webhook-fire time), and raw bytes + aren't JSON-serialisable anyway. Receivers that need the bytes use + ``format=eml``. + """ + return {k: v for k, v in part.items() if k not in ("content", "sha256")} + + +def build_jmap_email( + parsed_email: JmapEmail, *, include_body: bool = True +) -> Dict[str, Any]: + """Project the parsed JMAP ``Email`` object into the webhook payload. + + ``parse_email`` already returns a strict JMAP Email object + (RFC 8621 §4.1), so this is mostly a copy. We stamp ``receivedAt`` + (the moment the webhook fires) and strip the parser's project + extensions (``_ext`` and per-part ``content`` / ``sha256``) so the + body is strict JMAP. Storage-time fields (``id``, ``blobId``, + ``threadId``, ``mailboxIds``, ``keywords``) don't exist at + webhook-fire time and are simply absent. + + With ``include_body=False`` the body parts, ``bodyValues`` and + ``attachments`` are dropped — receivers get a notification-only + payload (subject + envelope addresses + headers) without the + message body content ever leaving the instance over the wire. + ``hasAttachment`` is preserved so receivers can still tell whether + the message had any. + """ + email: Dict[str, Any] = dict(parsed_email) + email.pop("_ext", None) # project extension, not strict JMAP + email["receivedAt"] = _utcdate(datetime.now(timezone.utc)) + + if include_body: + email["textBody"] = [ + _strip_body_part(p) for p in parsed_email.get("textBody") or [] + ] + email["htmlBody"] = [ + _strip_body_part(p) for p in parsed_email.get("htmlBody") or [] + ] + email["attachments"] = [ + _strip_body_part(p) for p in parsed_email.get("attachments") or [] + ] + else: + for key in ( + "textBody", + "htmlBody", + "bodyValues", + "bodyStructure", + "attachments", + "preview", + ): + email.pop(key, None) + + return email From 43e38473354305e738f0034c2ac2cca5b6a31542 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Mon, 29 Jun 2026 01:04:03 +0200 Subject: [PATCH 11/21] review fixes --- docs/webhooks.md | 108 +++--- src/backend/core/admin.py | 19 +- src/backend/core/api/serializers.py | 60 ++- src/backend/core/api/viewsets/channel.py | 38 +- src/backend/core/enums.py | 67 +++- src/backend/core/factories.py | 14 - src/backend/core/mda/dispatch_webhooks.py | 129 ++++--- src/backend/core/mda/inbound_create.py | 2 +- src/backend/core/mda/inbound_pipeline.py | 14 +- src/backend/core/mda/inbound_tasks.py | 7 + src/backend/core/mda/outbound.py | 22 +- src/backend/core/mda/selfcheck.py | 8 +- src/backend/core/models.py | 27 ++ src/backend/core/services/ssrf.py | 50 ++- .../admin/core/channel/change_form.html | 6 +- .../tests/api/test_channel_scope_level.py | 2 +- src/backend/core/tests/api/test_channels.py | 104 +++-- .../core/tests/api/test_messages_create.py | 6 +- .../api/test_messages_delivery_statuses.py | 23 +- .../core/tests/api/test_messages_import.py | 7 +- .../core/tests/api/test_prometheus_metrics.py | 16 +- .../core/tests/api/test_threads_list.py | 2 +- src/backend/core/tests/mda/test_autoreply.py | 34 +- .../core/tests/mda/test_dispatch_webhooks.py | 356 ++++++++++-------- .../tests/mda/test_inbound_spoofed_sender.py | 2 +- src/backend/core/tests/mda/test_outbound.py | 65 +++- .../core/tests/mda/test_outbound_e2e.py | 7 +- src/backend/core/tests/mda/test_retry.py | 6 +- src/backend/core/tests/services/test_ssrf.py | 68 ++++ src/backend/core/tests/test_signals.py | 36 +- .../e2e/management/commands/e2e_demo.py | 4 +- src/backend/messages/settings.py | 11 + .../webhook-integration-form.tsx | 64 ++-- .../thread-message/thread-message-header.tsx | 4 + 34 files changed, 922 insertions(+), 466 deletions(-) diff --git a/docs/webhooks.md b/docs/webhooks.md index 9d3e27124..671a30880 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -11,38 +11,40 @@ of this document. ## When does it fire? -For each inbound message accepted by the MTA the delivery pipeline goes -through two webhook phases: - -1. **`before_spam`** — fires right after the message has been parsed, - before any spam check. `is_spam` is not yet known. -2. **`after_spam`** — fires after the spam verdict but before the - `Message` row is created. `is_spam` is known. - -Each webhook channel picks **one** phase via `settings.phase` -(default `after_spam`). - -A channel may also be **blocking** (`settings.blocking: true`). A -blocking webhook gets to shape delivery: it can drop the message, ask -to be retried later, or return a small JSON body that overrides the -spam verdict and/or attaches labels to the resulting thread (see -[Response contract](#response-contract) below). Blocking webhooks run -**inline** at their phase, on the pipeline worker. - -Non-blocking webhooks are fire-and-forget — failures are logged and the -pipeline continues unchanged. Because they can't influence delivery, -they don't run on the pipeline worker at all: at their phase the channel -is just **recorded** (capturing the phase and the phase-time `is_spam`), -and the actual POST is handed to a background task **after the `Message` -is persisted**. The task renders the body from the stored message, so +A webhook channel has a single **`trigger`**: the point in a message's +lifecycle that fires it. The event name says both *when* it fires and +*whether* it can influence delivery — there's no separate blocking flag, +so invalid combinations can't be expressed: + +| `trigger` | Fires | Blocking? | `is_spam` | +| --------------------- | ------------------------------------------------- | --------- | --------- | +| `message.inbound` | The message just arrived, before the spam check | yes (sync) | pending | +| `message.delivering` | After the spam verdict, while delivery is in flight | yes (sync) | known | +| `message.delivered` | After the message has landed in the mailbox | no (async) | final | + +This flat list replaces the older `(events, phase, blocking)` triple. +Future lifecycle events (e.g. `message.sent`) extend it with new +`trigger` values. + +**`message.inbound`** and **`message.delivering`** are *synchronous*: they +run **inline** on the pipeline worker and get to shape delivery — drop the +message, ask to be retried later, or return a small JSON body that +overrides the spam verdict and/or attaches labels to the resulting thread +(see [Response contract](#response-contract)). + +**`message.delivered`** is *asynchronous* — fire-and-forget; failures are +logged and the pipeline continues unchanged. Because it can't influence +delivery, it doesn't run on the pipeline worker at all: the channel is +**recorded** during the pipeline (capturing the final `is_spam`) and the +actual POST is handed to a background task **after the `Message` is +persisted**. The task renders the body from the stored message, so nothing is copied or sent through the broker. Two consequences: -* A non-blocking webhook fires only for messages that become a - `Message` — not for one a blocking webhook later **drops**. (Spam is - *not* a drop: it still lands in the spam folder, so it still fires, - with `X-StMsg-Is-Spam: true`.) -* The POST body is the canonical stored message regardless of phase; the - `phase`/`is_spam` context lives in the `X-StMsg-*` headers as before. +* It fires only for messages that become a `Message` — not for one a + blocking webhook later **drops**. (Spam is *not* a drop: it still lands + in the spam folder, so it still fires, with `X-StMsg-Is-Spam: true`.) +* It always runs after the spam step, so its `X-StMsg-Is-Spam` is the + final verdict. ## Channel scopes @@ -66,10 +68,8 @@ A webhook channel stores its configuration in `Channel.settings` ```json { "url": "https://example.com/inbox-hook", - "events": ["message.inbound"], - "phase": "after_spam", + "trigger": "message.delivered", "format": "eml", - "blocking": false, "auth_method": "jwt" } ``` @@ -77,10 +77,8 @@ A webhook channel stores its configuration in `Channel.settings` | Key | Type | Default | Description | | ------------- | -------- | -------------- | --------------------------------------------------------------------------- | | `url` | string | **required** | `https://` endpoint, validated by the SSRF guard at each call. `http://` is accepted only when Django `DEBUG` is on (the local-dev escape hatch). | -| `events` | string[] | **required** | Currently only `message.inbound` is implemented. | -| `phase` | string | `after_spam` | `before_spam` or `after_spam`. | +| `trigger` | string | **required** | `message.inbound`, `message.delivering`, or `message.delivered` (see [When does it fire?](#when-does-it-fire)). | | `format` | string | `eml` | `eml`, `jmap`, or `jmap_metadata` (see [Payload formats](#payload-formats)). | -| `blocking` | bool | `false` | If true, the webhook response determines delivery (see [Response contract](#response-contract) below). | | `auth_method` | string | **required** | `jwt` or `api_key` (see [Authentication](#authentication)). | The serializer validates every change to `settings`, on create **and** @@ -94,8 +92,11 @@ Every call is: * `POST` to `settings.url`. * `User-Agent: Messages-Webhook/1.0`. * 30-second timeout. -* HTTP `3xx` is **not** followed — receivers must respond on the URL - configured, not after a redirect. +* HTTP `3xx` **is** followed (small hop limit), but **every hop is + re-validated and re-pinned** by the SSRF guard and the `POST` is + re-issued (method + body preserved), so a receiver behind a load + balancer or URL canonicaliser still gets the signed payload — and a + redirect can't point the delivery at an internal target. * The destination hostname/IP must pass the shared SSRF check (no loopback, link-local, private, multicast, reserved, or cloud metadata addresses; no IP literals). @@ -144,12 +145,11 @@ message lands. | Header | Value | | --------------------- | ---------------------------------------------------------------- | | `Content-Type` | `message/rfc822` for `eml`, `application/json` for both JMAP variants | -| `X-StMsg-Event` | `message.inbound` | -| `X-StMsg-Phase` | `before_spam` or `after_spam` | +| `X-StMsg-Trigger` | The lifecycle event that fired (`message.inbound` / `message.delivering` / `message.delivered`). Route on this — it says what happened and, implicitly, whether the webhook blocked. | | `X-StMsg-Channel-Id` | UUID of the firing webhook Channel | | `X-StMsg-Mailbox` | Destination mailbox address | | `X-StMsg-Recipient` | Envelope `RCPT TO` (usually the same as `X-StMsg-Mailbox`) | -| `X-StMsg-Is-Spam` | `true`, `false`, or `unknown` (`unknown` in the `before_spam` phase) | +| `X-StMsg-Is-Spam` | `true`, `false`, or `pending` (`pending` for `message.inbound`, which fires before the spam check) | | `X-StMsg-Message-Id` | UUID of the stored `Message` — **non-blocking only** (see note) | | `X-StMsg-Thread-Id` | UUID of the `Message`'s `Thread` — **non-blocking only** | @@ -253,7 +253,7 @@ optional; unknown keys are ignored. | Key | Type | Meaning | | ---------------- | -------------- | ---------------------------------------------------------------------------------------------------------------- | | `action` | `"drop"` / `"retry"` | `"drop"` drops the message at this phase; `"retry"` re-queues the inbound task (bounded by the 48h quarantine window). Any other value (or omission) is treated as accept. Case-insensitive. | -| `is_spam` | bool | Override the spam verdict. Acts as a full antispam: in the `before_spam` phase this **skips rspamd**. | +| `is_spam` | bool | Override the spam verdict. Acts as a full antispam: for a `message.inbound` webhook this **skips rspamd**. | | `add_labels` | string[] | UUIDs of `Label` rows in the destination mailbox to attach to the thread once it is created. | | `assign_to` | string[] | OIDC emails of users to assign to the resulting thread (one `ThreadEvent ASSIGN` per webhook, channel-attributed). | | `mark_starred` | bool (true only) | Star the resulting thread for the destination mailbox. | @@ -380,8 +380,7 @@ MTA received them. ```http POST /inbox-hook HTTP/1.1 Content-Type: message/rfc822 -X-StMsg-Event: message.inbound -X-StMsg-Phase: after_spam +X-StMsg-Trigger: message.delivered X-StMsg-Channel-Id: 05f1f991-c2e9-4fa7-8a78-98c3aa904c7c X-StMsg-Mailbox: alice@example.com X-StMsg-Recipient: alice@example.com @@ -503,7 +502,7 @@ def inbox_hook(): return "unsupported", 415 # Echo envelope metadata for logging. - print("phase:", request.headers["X-StMsg-Phase"]) + print("trigger:", request.headers["X-StMsg-Trigger"]) print("is_spam:", request.headers["X-StMsg-Is-Spam"]) print("mailbox:", request.headers["X-StMsg-Mailbox"]) return "", 200 @@ -520,9 +519,28 @@ def inbox_hook(): * The validated IP is **pinned** for the actual connection, defeating DNS-rebinding (TOCTOU). For HTTPS the TLS certificate is verified against the original hostname. + * Redirects are followed (up to a small hop limit) but **each hop is + re-validated and re-pinned**, so an endpoint can't 3xx-redirect the + delivery to an internal target. The `POST` is re-issued on each hop + (method + body preserved), so a receiver behind a load balancer or + URL canonicaliser still gets the signed payload. * Blocking webhooks are silent for the original sender — the inbound SMTP transaction has already been accepted. A blocking-drop is visible only through logs and the pipeline's `dropped_by_webhook` return value. +## Performance notes & future work + +* **Per-webhook blob re-fetch (non-blocking).** Each non-blocking + webhook for a message runs in its own task that independently + re-fetches `Message.blob` (object-storage download + decrypt + + decompress) and re-parses the MIME. With *K* non-blocking webhooks on + the same message that's *K* downloads and *K* parses of identical + bytes — wasteful for large messages. This is an accepted trade-off for + now (each task stays self-contained and the payload never rides the + broker). A future optimization is a **short-lived blob/parse cache** + (keyed by `blob_id`, scoped to a single message's fan-out) so the + bytes are fetched and parsed once and reused across that message's + webhook tasks. + [rfc8621]: https://www.rfc-editor.org/rfc/rfc8621#section-4.1 diff --git a/src/backend/core/admin.py b/src/backend/core/admin.py index 8ab8a317e..575b728ec 100644 --- a/src/backend/core/admin.py +++ b/src/backend/core/admin.py @@ -611,19 +611,12 @@ def regenerate_secret_view(self, request, object_id): root = channel.rotate_secret() - # Show the credential the *receiver* actually presents, matching - # the DRF ``_attach_credential`` flow. ``rotate_secret`` always - # returns the root signing secret, but an ``api_key``-auth webhook - # authenticates with the derived ``whk_…`` value (the root never - # touches the wire) — showing the root here would hand the operator - # a credential the receiver never sees. - if ( - channel.type == ChannelTypes.WEBHOOK - and (channel.settings or {}).get("auth_method") == "api_key" - ): - display_secret = channel.get_webhook_api_key() - else: - display_secret = root + # Show the credential the *receiver* actually presents. The + # webhook (jwt→secret / api_key→derived) rule lives on the model + # so this and the DRF ``_attach_credential`` flow can't drift; + # api_key *channels* (not webhooks) fall back to the rotated root. + credential = channel.get_webhook_surfaced_credential() + display_secret = credential[1] if credential else root context = { **self.admin_site.each_context(request), diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index e8f3aa849..4c422aebd 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -17,9 +17,7 @@ from core import enums, models from core.mda.dispatch_webhooks import ( - PHASE_AFTER_SPAM, VALID_FORMATS, - VALID_PHASES, ) from core.mda.inline_images import extract_inline_images_html from core.services.blob_gc import schedule_for_gc @@ -2302,32 +2300,21 @@ def _validate_webhook_settings(self, attrs): {"settings": "webhook settings.url must use https://."} ) - events = settings_data.get("events") - if not events or not isinstance(events, list): - raise serializers.ValidationError( - { - "settings": "webhook channels require settings.events (a non-empty list)." - } - ) - unknown = [e for e in events if e not in enums.WebhookEvents] - if unknown: - raise serializers.ValidationError( - {"settings": f"Unknown webhook events: {unknown}"} - ) - - # Optional dispatcher knobs. Validated here so a malformed PATCH - # to ``settings`` can't strand a webhook channel in a state the - # dispatcher silently treats as defaults. - phase = settings_data.get("phase", PHASE_AFTER_SPAM) - if phase not in VALID_PHASES: + # A single ``trigger`` lifecycle event describes the webhook — it + # encodes the event, the pipeline phase, and whether it blocks + # delivery, so invalid combinations (e.g. non-blocking before-spam) + # can't be expressed (see ``enums.WebhookTrigger``). + trigger = settings_data.get("trigger") + if trigger not in enums.WebhookTrigger: raise serializers.ValidationError( { "settings": ( - f"webhook settings.phase must be one of: " - f"{', '.join(sorted(VALID_PHASES))}." + "webhook channels require settings.trigger, one of: " + f"{', '.join(t.value for t in enums.WebhookTrigger)}." ) } ) + body_format = settings_data.get("format", "eml") if body_format not in VALID_FORMATS: raise serializers.ValidationError( @@ -2338,18 +2325,29 @@ def _validate_webhook_settings(self, attrs): ) } ) - if "blocking" in settings_data and not isinstance( - settings_data["blocking"], bool - ): - raise serializers.ValidationError( - {"settings": "webhook settings.blocking must be a boolean."} - ) - if settings_data.get("auth_method") not in ("jwt", "api_key"): + # auth_method selects how the stored secret is presented on each + # POST; it pairs with the credential in encrypted_settings, which a + # settings PATCH never touches. So treat it like that credential: + # carry the existing value forward when a partial PATCH omits it, + # instead of forcing the caller to re-send it on every edit (and + # silently dropping it on the replace-on-PATCH save). Only a value + # that is actually supplied — or a create with none — is rejected. + auth_method = settings_data.get("auth_method") + if auth_method is None and self.instance is not None: + existing = (self.instance.settings or {}).get("auth_method") + if existing in enums.WebhookAuthMethod and isinstance( + attrs.get("settings"), dict + ): + # Persist it through the replace-save so the dispatcher + # still knows how to present the secret. + attrs["settings"]["auth_method"] = existing + auth_method = existing + if auth_method not in enums.WebhookAuthMethod: raise serializers.ValidationError( { "settings": ( - "webhook settings.auth_method is required and must " - "be 'jwt' or 'api_key'." + "webhook settings.auth_method is required, one of: " + f"{', '.join(m.value for m in enums.WebhookAuthMethod)}." ) } ) diff --git a/src/backend/core/api/viewsets/channel.py b/src/backend/core/api/viewsets/channel.py index b3e130f74..0590a3ac7 100644 --- a/src/backend/core/api/viewsets/channel.py +++ b/src/backend/core/api/viewsets/channel.py @@ -15,7 +15,7 @@ from rest_framework.response import Response from core import models -from core.enums import ChannelScopeLevel, ChannelTypes +from core.enums import ChannelScopeLevel, ChannelTypes, WebhookAuthMethod from .. import permissions, serializers @@ -45,16 +45,12 @@ def _attach_credential(data: dict, channel: models.Channel) -> None: if plaintext: data["api_key"] = plaintext return - if channel.type == ChannelTypes.WEBHOOK: - auth_method = (channel.settings or {}).get("auth_method") - if auth_method == "jwt": - root = (channel.encrypted_settings or {}).get("secret") - if root: - data["secret"] = root - elif auth_method == "api_key": - derived = channel.get_webhook_api_key() - if derived: - data["api_key"] = derived + # Webhook channels: the (jwt→secret / api_key→derived) rule lives on the + # model so this and the Django-admin regenerate view can't drift. + credential = channel.get_webhook_surfaced_credential() + if credential: + key, value = credential + data[key] = value @extend_schema( @@ -248,6 +244,26 @@ def regenerate_secret(self, request, *args, **kwargs): feature available via Django admin. """ instance = self.get_object() + + # Guard before rotating: a webhook channel whose auth_method isn't + # one ``_attach_credential`` knows how to surface would have its old + # secret invalidated by ``rotate_secret`` while the freshly minted + # one is dropped from the response — permanently bricking the + # webhook with no way to learn the new secret. Reject up front so + # rotation only runs when we can hand the result back. + if ( + instance.type == ChannelTypes.WEBHOOK + and (instance.settings or {}).get("auth_method") not in WebhookAuthMethod + ): + raise ValidationError( + { + "settings": ( + "webhook settings.auth_method must be 'jwt' or " + "'api_key' before the secret can be rotated." + ) + } + ) + try: plaintext = instance.rotate_secret() except ValueError as exc: diff --git a/src/backend/core/enums.py b/src/backend/core/enums.py index 922351e5f..add59c126 100644 --- a/src/backend/core/enums.py +++ b/src/backend/core/enums.py @@ -62,10 +62,24 @@ class MessageRecipientTypeChoices(models.IntegerChoices): class MessageDeliveryStatusChoices(models.IntegerChoices): - """Defines the possible statuses of a message delivery.""" + """Defines the possible statuses of a message delivery. + + FOOTGUN: ``SENT_INTERNAL`` and ``SENT_EXTERNAL`` are *both* terminal + "delivered"-class statuses — ``SENT_INTERNAL`` for same-instance + (mailbox-to-mailbox) delivery, ``SENT_EXTERNAL`` for external (handed to + the MTA). They render identically to the user. So any "did it send / is + it delivered?" check must treat them together: a bare ``== SENT_EXTERNAL`` + / ``!= SENT_EXTERNAL`` silently mishandles internal mail. + (``mda/selfcheck.py`` gets away with ``!= SENT_EXTERNAL`` only because it + forces ``force_mta_out=True``, taking the external path.) + + The integer values and string labels are unchanged from the old + ``INTERNAL``/``SENT`` members (1/"internal", 2/"sent") — only the Python + member names changed — so this is not a DB or wire change. + """ - INTERNAL = 1, "internal" - SENT = 2, "sent" + SENT_INTERNAL = 1, "internal" + SENT_EXTERNAL = 2, "sent" FAILED = 3, "failed" RETRY = 4, "retry" CANCELLED = 5, "cancelled" @@ -221,19 +235,48 @@ class ChannelTypes(StrEnum): CALDAV = "caldav" -class WebhookEvents(StrEnum): - """Known webhook event identifiers. +class WebhookTrigger(StrEnum): + """The lifecycle event that fires a webhook, stored as + ``Channel.settings["trigger"]`` and surfaced verbatim in the + ``X-StMsg-Trigger`` header. + + A single flat value — the event name *is* the behaviour, so there's no + separate blocking/phase flag and invalid combinations can't be + expressed. Which pipeline phase each runs at and whether it blocks + delivery is the spec in ``docs/webhooks.md``: + + - ``MESSAGE_INBOUND`` — the message just arrived, **before** the spam + check. Synchronous (blocking): the receiver can DROP / RETRY / + mutate it and sees no spam verdict yet. + - ``MESSAGE_DELIVERING`` — **after** the spam check, while delivery + is still in flight. Synchronous (blocking): can DROP / RETRY / + mutate and sees the verdict. + - ``MESSAGE_DELIVERED`` — the message has landed in the mailbox. + Asynchronous (fire-and-forget): a notification that can't influence + delivery, always reflecting the final spam verdict. - Stored as strings in ``Channel.settings["events"]``; validated by the - serializer at write time. Adding a new event is a Python-only change. + Future lifecycle events (e.g. ``message.sent``) are added here. """ MESSAGE_INBOUND = "message.inbound" - # MESSAGE_OUTBOUND ("message.outbound") is not wired yet — the dispatcher - # only ever fires on inbound. Kept commented so the serializer rejects it - # at write time instead of accepting a webhook that silently never fires. - # Uncomment once outbound dispatch is implemented. - # MESSAGE_OUTBOUND = "message.outbound" + MESSAGE_DELIVERING = "message.delivering" + MESSAGE_DELIVERED = "message.delivered" + + +class WebhookAuthMethod(StrEnum): + """How a webhook channel presents its single stored secret on each + POST, stored as ``Channel.settings["auth_method"]``. + + - ``JWT`` — an HMAC signature over the body plus a short-TTL HS256 + JWT, both keyed by the root secret (which never travels the wire). + - ``API_KEY`` — a static ``whk_…`` header value HMAC-derived from + the root, for receivers that can only check a header. + + See ``docs/webhooks.md`` for the wire details. + """ + + JWT = "jwt" + API_KEY = "api_key" class ChannelApiKeyScope(models.TextChoices): diff --git a/src/backend/core/factories.py b/src/backend/core/factories.py index 9bd670101..e01c2ddd6 100644 --- a/src/backend/core/factories.py +++ b/src/backend/core/factories.py @@ -380,20 +380,6 @@ class Meta: ) ) - @factory.post_generation - def _default_webhook_auth_method( # pylint: disable=unused-argument - self, create, extracted, **kwargs - ): - """Inject ``auth_method='jwt'`` into webhook channel settings if a - test didn't set one. The serializer requires it on real creates; - this just spares every webhook test from spelling it out.""" - if not create or self.type != "webhook": - return - if (self.settings or {}).get("auth_method"): - return - self.settings = {**(self.settings or {}), "auth_method": "jwt"} - self.save(update_fields=["settings"]) - def make_api_key_channel( *, diff --git a/src/backend/core/mda/dispatch_webhooks.py b/src/backend/core/mda/dispatch_webhooks.py index fa03740f6..f91716370 100644 --- a/src/backend/core/mda/dispatch_webhooks.py +++ b/src/backend/core/mda/dispatch_webhooks.py @@ -4,10 +4,12 @@ ``inbound_pipeline.py``) iterates every webhook-type Channel that matches the destination mailbox (``scope_level=MAILBOX``), its domain (``scope_level=MAILDOMAIN``), or any global channel -(``scope_level=GLOBAL``). The webhook fires either ``before_spam`` or -``after_spam`` according to ``settings.phase``; a ``blocking`` webhook -can abort delivery (DROP), request retry (RETRY), override the spam -verdict, or attach labels via its JSON response body. +(``scope_level=GLOBAL``). A single ``settings.trigger`` lifecycle event +(see ``enums.WebhookTrigger`` and ``docs/webhooks.md``) describes a webhook: +``message.inbound`` (before spam) and ``message.delivering`` (after spam) +run inline and can abort delivery (DROP), request retry (RETRY), override +the spam verdict, or attach labels via their JSON response body; +``message.delivered`` is fire-and-forget after the message is created. This file is webhook-specific: HTTP plumbing, signing (HMAC + JWT or API key), JMAP body building, SSRF-safe POST, response classification. @@ -452,7 +454,6 @@ def find_webhook_channels_for_mailbox( def _envelope_headers( *, channel: models.Channel, - phase: str, mailbox: models.Mailbox, recipient_email: str, is_spam: Optional[bool], @@ -461,6 +462,13 @@ def _envelope_headers( """Build the ``X-StMsg-*`` envelope headers attached to every webhook POST regardless of body format. Same shape for ``eml`` and ``jmap``. + The firing lifecycle event is sent as ``X-StMsg-Trigger`` (e.g. + ``message.delivered``) — a single header that already says what + happened and, implicitly, how (``message.inbound``/``message.delivering`` + are blocking, ``message.delivered`` is fire-and-forget). No separate + event/phase/mode headers: they'd only duplicate it, and + ``X-StMsg-Is-Spam: pending`` already marks a before-spam firing. + The *MIME* message-id is intentionally *not* a header: every body format already carries it (``messageId`` in the jmap variants, the raw ``Message-ID:`` header in ``eml``), so a header would only @@ -472,13 +480,14 @@ def _envelope_headers( webhooks fire before the row exists, so they don't carry them. """ if is_spam is None: - spam_value = "unknown" + # No verdict yet — the spam check hasn't run (a ``message.inbound`` + # webhook fires before it). "pending" says that plainly. + spam_value = "pending" else: spam_value = "true" if is_spam else "false" headers = { "User-Agent": USER_AGENT, - "X-StMsg-Event": enums.WebhookEvents.MESSAGE_INBOUND.value, - "X-StMsg-Phase": phase, + "X-StMsg-Trigger": (channel.settings or {}).get("trigger", ""), "X-StMsg-Channel-Id": str(channel.id), "X-StMsg-Mailbox": str(mailbox), "X-StMsg-Recipient": recipient_email, @@ -519,30 +528,35 @@ class UserWebhookStep: def __init__(self, channel: models.Channel, phase: str): self.channel = channel - self.phase = phase # Phase suffix in the name lets the task return value carry # "which phase did this drop happen at" without a separate field. self.name = f"webhook[{channel.id}]:{phase}" def __call__(self, ctx: InboundContext) -> Decision: cfg = self.channel.settings or {} - blocking = bool(cfg.get("blocking", False)) + # Only message.delivered is fire-and-forget; the others block and run + # inline (see docs/webhooks.md). Unknown/missing → non-blocking + # (fail-closed; the step is only built for a valid trigger anyway). + blocking = cfg.get("trigger") in ( + enums.WebhookTrigger.MESSAGE_INBOUND, + enums.WebhookTrigger.MESSAGE_DELIVERING, + ) if not blocking: - # Non-blocking webhooks can't influence delivery, so they don't - # run network I/O on the inbound worker. We also don't render or - # snapshot the body here: we record the channel (with the - # phase-time ``is_spam``) and fire it AFTER the Message is - # created — see ``dispatch_recorded_webhooks``. The task then - # renders the payload from the durable ``Message.blob``, so the - # email bytes never get copied or pushed through the broker. + # Non-blocking (``message.delivered``) webhooks can't influence + # delivery, so they don't run network I/O on the inbound worker. + # We also don't render or snapshot the body here: we record the + # channel with the spam verdict at this point and fire it AFTER + # the Message is created — see ``dispatch_recorded_webhooks``. + # The task then renders the payload from the durable + # ``Message.blob``, so the email bytes never get copied or pushed + # through the broker. ``message.delivered`` always runs after the + # spam step, so the recorded verdict is the final one. # # Consequence (intended): a non-blocking webhook fires only for # messages that actually become a Message — not for ones a - # blocking webhook later DROPs. The X-StMsg-* headers still carry - # the phase/spam context, and the body is the canonical stored - # MIME regardless of phase. - ctx.pending_webhooks.append((self.channel.id, self.phase, ctx.is_spam)) + # blocking webhook later DROPs. + ctx.pending_webhooks.append((self.channel.id, ctx.is_spam)) return Decision.CONTINUE body_format = cfg.get("format", DEFAULT_FORMAT) @@ -553,7 +567,6 @@ def __call__(self, ctx: InboundContext) -> Decision: result = _dispatch_webhook( channel=self.channel, mailbox=ctx.mailbox, - phase=self.phase, is_spam=ctx.is_spam, recipient_email=ctx.recipient_email, content_type=content_type, @@ -597,14 +610,14 @@ def _build_auth_headers( ``None`` when the channel is misconfigured (caller fails closed).""" auth_method = (channel.settings or {}).get("auth_method") - if auth_method == "api_key": + if auth_method == enums.WebhookAuthMethod.API_KEY: # Derived from the root secret via HMAC. The raw root never # touches the wire — a receiver-side log leak of this value # reveals nothing about the root, so HMAC/JWT verification # remains unforgeable. return {"X-StMsg-Api-Key": channel.get_webhook_api_key()} - if auth_method == "jwt": + if auth_method == enums.WebhookAuthMethod.JWT: # HMAC signature over the body + short-TTL HS256 JWT, both keyed # by the root secret. Signed at send time (here / in the task), # so the JWT TTL is measured from the actual POST, not enqueue. @@ -766,7 +779,6 @@ def _dispatch_webhook( *, channel: models.Channel, mailbox: models.Mailbox, - phase: str, is_spam: Optional[bool], recipient_email: str, content_type: str, @@ -791,7 +803,6 @@ def _dispatch_webhook( return _failure(blocking, Decision.RETRY) envelope_headers = _envelope_headers( channel=channel, - phase=phase, mailbox=mailbox, recipient_email=recipient_email, is_spam=is_spam, @@ -817,16 +828,32 @@ def _resolve_body_from_message( ``Message.blob`` (re-parsed the same way the pipeline parsed them) instead of a transient snapshot, so there's no second copy of the email. Mirrors ``_resolve_body``'s output contract. + + Future optimization: with K non-blocking webhooks on one message this + re-fetches and re-parses the same blob K times (one per task). A + short-lived blob/parse cache keyed by ``blob_id`` would let a single + message's fan-out fetch + parse once. See "Performance notes" in + docs/webhooks.md. """ raw_data = message.blob.get_content() - parsed_email = parse_email(raw_data) or {} - return _resolve_body(body_format, raw_data, parsed_email) + parsed_email = parse_email(raw_data) + if parsed_email is None and body_format != FORMAT_EML: + # The stored MIME can't be re-parsed, so we can't render a faithful + # JMAP body. Fail loudly (the caller logs and the dispatch is + # treated as a failure) rather than POST a near-empty Email that + # silently drops the message's identity and content. The EML format + # is exempt: it ships the raw bytes verbatim and ignores the parse. + raise ValueError( + f"cannot parse stored blob for message {message.id} into " + f"webhook format {body_format!r}" + ) + return _resolve_body(body_format, raw_data, parsed_email or {}) def dispatch_recorded_webhooks( message: models.Message, mailbox: models.Mailbox, - pending: List[Tuple[Any, str, Optional[bool]]], + pending: List[Tuple[Any, Optional[bool]]], ) -> None: """Fire the non-blocking webhooks recorded during the pipeline. @@ -841,10 +868,8 @@ def dispatch_recorded_webhooks( return message_id = str(message.id) mailbox_id = str(mailbox.id) - for channel_id, phase, is_spam in pending: - dispatch_webhook_task.delay( - message_id, str(channel_id), mailbox_id, phase, is_spam - ) + for channel_id, is_spam in pending: + dispatch_webhook_task.delay(message_id, str(channel_id), mailbox_id, is_spam) @celery_app.task @@ -852,7 +877,6 @@ def dispatch_webhook_task( message_id: str, channel_id: str, mailbox_id: str, - phase: str, is_spam: Optional[bool], ) -> None: """Deliver one non-blocking webhook off the inbound worker. @@ -889,7 +913,6 @@ def dispatch_webhook_task( _dispatch_webhook( channel=channel, mailbox=mailbox, - phase=phase, is_spam=is_spam, recipient_email=str(mailbox), content_type=content_type, @@ -911,9 +934,11 @@ def webhook_steps_for_mailbox( ) -> List[Step]: """Build one ``UserWebhookStep`` per matching channel for the phase. - Channels are filtered here (phase, events, url present, valid - format) rather than at run time so the pipeline iterator sees a - flat list of ready-to-call steps. + Channels are filtered here (the trigger's phase, url present, valid + format) rather than at run time so the pipeline iterator sees a flat + list of ready-to-call steps. A channel runs at exactly the phase its + ``trigger`` maps to (``message.delivered`` and ``message.delivering`` + → after-spam; ``message.inbound`` → before-spam). ``channels`` may be passed in to reuse a single channel-set query across both phases (the set is identical before- and after-spam); @@ -928,25 +953,23 @@ def webhook_steps_for_mailbox( steps: List[Step] = [] for channel in channels: cfg = channel.settings or {} - if cfg.get("phase", PHASE_AFTER_SPAM) != phase: - continue - events = cfg.get("events") - if events is None: - # Only a missing/null key falls back to the default. An explicit - # empty list means "subscribed to nothing" and must stay empty so - # the membership check below correctly skips the channel. - events = [enums.WebhookEvents.MESSAGE_INBOUND.value] - if not isinstance(events, list): - # Validator guarantees a list on write; a non-list here is a - # misconfigured row. Fail closed — a bare string would make the - # ``in`` check below match substrings instead of members. + trigger = cfg.get("trigger") + if trigger not in enums.WebhookTrigger: + # Misconfigured/legacy row (or a future, non-inbound event). + # Fail closed. logger.warning( - "Webhook channel %s has non-list events=%r — skipping", + "Webhook channel %s has unknown trigger=%r — skipping", channel.id, - events, + trigger, ) continue - if enums.WebhookEvents.MESSAGE_INBOUND.value not in events: + # Only message.inbound fires before the spam check (docs/webhooks.md). + runs_at = ( + PHASE_BEFORE_SPAM + if trigger == enums.WebhookTrigger.MESSAGE_INBOUND + else PHASE_AFTER_SPAM + ) + if runs_at != phase: continue if not cfg.get("url"): continue diff --git a/src/backend/core/mda/inbound_create.py b/src/backend/core/mda/inbound_create.py index 09bf83364..5ef5e0460 100644 --- a/src/backend/core/mda/inbound_create.py +++ b/src/backend/core/mda/inbound_create.py @@ -549,7 +549,7 @@ def _create_message_from_inbound( # pylint: disable=too-many-arguments defaults = {} if is_import and not message.is_draft: defaults["delivery_status"] = ( - enums.MessageDeliveryStatusChoices.SENT + enums.MessageDeliveryStatusChoices.SENT_EXTERNAL ) models.MessageRecipient.objects.get_or_create( message=message, diff --git a/src/backend/core/mda/inbound_pipeline.py b/src/backend/core/mda/inbound_pipeline.py index 1f32813da..ada16575b 100644 --- a/src/backend/core/mda/inbound_pipeline.py +++ b/src/backend/core/mda/inbound_pipeline.py @@ -114,14 +114,12 @@ class InboundContext: # pylint: disable=too-many-instance-attributes pending_drafts: List[Tuple[Any, str]] = field(default_factory=list) # Deferred non-blocking webhook dispatches. Each entry is - # ``(channel_id, phase, is_spam_at_phase)`` — recorded when the webhook - # step runs and fired AFTER the Message exists, so the task renders the - # payload from the durable ``Message`` (no transient snapshot, nothing - # large on the broker). Same record-now / fire-after-Message pattern - # push notifications use, so the two slot together cleanly. - pending_webhooks: List[Tuple[Any, str, Optional[bool]]] = field( - default_factory=list - ) + # ``(channel_id, is_spam)`` — recorded when the webhook step runs and + # fired AFTER the Message exists, so the task renders the payload from + # the durable ``Message`` (no transient snapshot, nothing large on the + # broker). ``message.delivered`` always runs after the spam step, so the + # recorded verdict is final. + pending_webhooks: List[Tuple[Any, Optional[bool]]] = field(default_factory=list) # Blocking-webhook flag actions (OR-merged across webhooks). All # default to False and are only ever flipped to True by a diff --git a/src/backend/core/mda/inbound_tasks.py b/src/backend/core/mda/inbound_tasks.py index b7cd4f375..d4ce93bd5 100644 --- a/src/backend/core/mda/inbound_tasks.py +++ b/src/backend/core/mda/inbound_tasks.py @@ -271,6 +271,13 @@ def process_inbound_message_task(self, inbound_message_id: str): # treated as spam. Normalize ctx.is_spam so downstream consumers # (autoreply gate, task result) agree with where it actually lands. ctx.is_spam = False + # ...but a quarantine delivery means a processing step never + # completed: the forced is_spam=False is a placement decision, + # not a real spam verdict, and a blocking step that wanted to + # suppress the reply (or classify the sender as spam) never got + # to run. Don't fire an autoreply to a sender we couldn't fully + # vet — suppress it for quarantined messages. + ctx.skip_autoreply = True inbound_msg = _create_message_from_inbound( recipient_email=ctx.recipient_email, diff --git a/src/backend/core/mda/outbound.py b/src/backend/core/mda/outbound.py index adac5a848..6fdec5f15 100644 --- a/src/backend/core/mda/outbound.py +++ b/src/backend/core/mda/outbound.py @@ -597,10 +597,22 @@ def _mark_delivered( # TODO also update message.updated_at? envelope_to[recipient_email].delivered_at = timezone.now() envelope_to[recipient_email].delivery_message = None + # Same-instance delivery gets its own SENT_INTERNAL + # status, distinct from external SENT_EXTERNAL, so the + # internal/external split stays visible in the data and + # metrics. Both render as "delivered" to the user. + # + # FOOTGUN: SENT_INTERNAL is a second "delivered"-class + # status alongside SENT_EXTERNAL. Any "did it send?" check + # written as ``== SENT_EXTERNAL`` / ``!= SENT_EXTERNAL`` + # must also accept SENT_INTERNAL or it silently mishandles + # internal mail (see the note on + # ``MessageDeliveryStatusChoices`` and the + # ``force_mta_out`` guard in ``mda/selfcheck.py``). envelope_to[recipient_email].delivery_status = ( - MessageDeliveryStatusChoices.INTERNAL + MessageDeliveryStatusChoices.SENT_INTERNAL if internal - else MessageDeliveryStatusChoices.SENT + else MessageDeliveryStatusChoices.SENT_EXTERNAL ) envelope_to[recipient_email].save( update_fields=[ @@ -640,9 +652,15 @@ def _mark_delivered( external_recipients = set() for recipient_email in envelope_to: + # ``MESSAGES_ALLOW_INTERNAL_DELIVERY=False`` forces local + # recipients out through the MTA instead of the internal fast + # path. ``check_local_recipient`` stays first so its + # create-if-missing side effect is unchanged either way (the + # mailbox still needs to exist to receive the MTA round-trip). if ( check_local_recipient(recipient_email, create_if_missing=True) and not force_mta_out + and settings.MESSAGES_ALLOW_INTERNAL_DELIVERY ): try: # Reference the sender's already-committed blob so diff --git a/src/backend/core/mda/selfcheck.py b/src/backend/core/mda/selfcheck.py index b58fd4030..3055d9aba 100644 --- a/src/backend/core/mda/selfcheck.py +++ b/src/backend/core/mda/selfcheck.py @@ -91,7 +91,13 @@ def create_and_send_draft( # Check message delivery status recipient_status = message.recipients.first().delivery_status # pylint: disable=no-member - if recipient_status != models.MessageDeliveryStatusChoices.SENT: + # ``!= SENT_EXTERNAL`` is safe here ONLY because + # ``send_message(force_mta_out=True)`` above forces the external MTA path, + # which yields SENT_EXTERNAL. A same-instance recipient delivered + # internally would be SENT_INTERNAL (also "delivered") and would wrongly + # fail this check — see the footgun note on MessageDeliveryStatusChoices. + # Don't drop force_mta_out without also accepting SENT_INTERNAL here. + if recipient_status != models.MessageDeliveryStatusChoices.SENT_EXTERNAL: raise SelfCheckError("Message not delivered") return message diff --git a/src/backend/core/models.py b/src/backend/core/models.py index 5b8ddf02c..69b962ef0 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -660,6 +660,33 @@ def get_webhook_api_key(self) -> Optional[str]: ).hexdigest() return "whk_" + derived + def get_webhook_surfaced_credential(self) -> "Optional[tuple[str, str]]": + """The credential a webhook receiver actually presents, as + ``(response_key, value)``: + + - ``auth_method='jwt'`` → ``("secret", )`` + - ``auth_method='api_key'`` → ``("api_key", )`` + + Returns ``None`` for a non-webhook channel, an unknown/missing + ``auth_method``, or an absent secret. Single source of truth for + "which credential is surfaceable", shared by the DRF + create/regenerate response (``_attach_credential``) and the + Django-admin regenerate view, so the two can't drift. + """ + # pylint: disable-next=import-outside-toplevel + from core.enums import ChannelTypes, WebhookAuthMethod + + if self.type != ChannelTypes.WEBHOOK: + return None + auth_method = (self.settings or {}).get("auth_method") + if auth_method == WebhookAuthMethod.JWT: + root = (self.encrypted_settings or {}).get("secret") + return ("secret", root) if root else None + if auth_method == WebhookAuthMethod.API_KEY: + derived = self.get_webhook_api_key() + return ("api_key", derived) if derived else None + return None + def api_key_covers( self, *, mailbox=None, maildomain=None, mailbox_roles=None ) -> bool: diff --git a/src/backend/core/services/ssrf.py b/src/backend/core/services/ssrf.py index 25fa9f74d..a7ab3de91 100644 --- a/src/backend/core/services/ssrf.py +++ b/src/backend/core/services/ssrf.py @@ -237,13 +237,23 @@ def _pinned_session(self, url: str) -> tuple[requests.Session, str]: session.mount("https://", adapter) return session, hostname - def get(self, url: str, timeout: int, **kwargs) -> requests.Response: - """Perform a safe HTTP GET with per-hop SSRF validation on redirects. - - Redirects are followed manually up to MAX_REDIRECTS hops. Each Location - URL is re-validated from scratch, so an attacker-controlled server - cannot redirect to an internal address or a different private target - on a later hop. + def _request_with_redirects( + self, method: str, url: str, timeout: int, **kwargs + ) -> requests.Response: + """Issue ``method`` to ``url``, following redirects manually with + per-hop SSRF validation. + + Redirects are followed up to MAX_REDIRECTS hops. Each Location URL is + re-validated and re-pinned from scratch, so an attacker-controlled + server cannot redirect to an internal address or a different private + target on a later hop. The HTTP method is preserved across hops (we + re-issue the same verb rather than downgrading a 30x to GET), so a + POST body reaches the final, validated destination intact. + + ``method`` is the lowercase session method name (``"get"`` / + ``"post"``) — we call that bound method directly rather than + ``Session.request`` so each verb keeps a distinct, individually + mockable entry point. """ # We always handle redirects ourselves — strip any caller override so # the underlying requests session never follows a redirect unchecked. @@ -253,7 +263,7 @@ def get(self, url: str, timeout: int, **kwargs) -> requests.Response: for _ in range(MAX_REDIRECTS + 1): session, _ = self._pinned_session(current_url) - response = session.get( + response = getattr(session, method)( current_url, timeout=timeout, allow_redirects=False, **kwargs ) @@ -271,14 +281,20 @@ def get(self, url: str, timeout: int, **kwargs) -> requests.Response: raise SSRFValidationError(f"Too many redirects (max {MAX_REDIRECTS})") - def post(self, url: str, timeout: int, **kwargs) -> requests.Response: - """Perform a safe HTTP POST. + def get(self, url: str, timeout: int, **kwargs) -> requests.Response: + """Perform a safe HTTP GET with per-hop SSRF validation on redirects.""" + return self._request_with_redirects("get", url, timeout, **kwargs) - Redirects are NOT followed: webhook providers conventionally don't - chase 3xx on POST, and silently turning a 30x into a follow-up GET - (which is what most clients do for 301/302/303) would surprise the - caller. A 3xx is returned to the caller as-is. + def post(self, url: str, timeout: int, **kwargs) -> requests.Response: + """Perform a safe HTTP POST with per-hop SSRF validation on redirects. + + Redirects are followed — re-validating and re-pinning each hop — and + the POST is re-issued (method + body preserved) to the validated + Location, so a webhook endpoint that 3xx-redirects (e.g. behind a + load balancer or URL canonicaliser) still receives the signed + payload. The signature is computed over the body, which is unchanged + across hops, so it stays valid at the final destination. We never + downgrade to GET — silently dropping the body would surprise the + caller and defeat the delivery. """ - kwargs.pop("allow_redirects", None) - session, _ = self._pinned_session(url) - return session.post(url, timeout=timeout, allow_redirects=False, **kwargs) + return self._request_with_redirects("post", url, timeout, **kwargs) diff --git a/src/backend/core/templates/admin/core/channel/change_form.html b/src/backend/core/templates/admin/core/channel/change_form.html index b4709beff..3dbfd460a 100644 --- a/src/backend/core/templates/admin/core/channel/change_form.html +++ b/src/backend/core/templates/admin/core/channel/change_form.html @@ -7,15 +7,15 @@ {% endblock %} {% block object-tools-items %} -{% if original and original.type == "api_key" %} +{% if original and original.type == "api_key" or original and original.type == "webhook" %}
  • + onsubmit="return confirm('Regenerate the secret for this channel? The previous secret will stop working immediately.');"> {% csrf_token %}
    • diff --git a/src/backend/core/tests/api/test_channel_scope_level.py b/src/backend/core/tests/api/test_channel_scope_level.py index 8788705ee..fe53edf0a 100644 --- a/src/backend/core/tests/api/test_channel_scope_level.py +++ b/src/backend/core/tests/api/test_channel_scope_level.py @@ -1025,7 +1025,7 @@ def test_personal_webhook_channel(self, api_client): "type": "webhook", "settings": { "url": "https://hook.example.com/me", - "events": ["message.inbound"], + "trigger": "message.delivered", "auth_method": "jwt", }, }, diff --git a/src/backend/core/tests/api/test_channels.py b/src/backend/core/tests/api/test_channels.py index 03bcb0800..fccf2a17c 100644 --- a/src/backend/core/tests/api/test_channels.py +++ b/src/backend/core/tests/api/test_channels.py @@ -594,7 +594,7 @@ def test_create_minimal_webhook(self, api_client, mailbox): mailbox, { "url": "https://hook.example.com/in", - "events": ["message.inbound"], + "trigger": "message.delivered", "auth_method": "jwt", }, ) @@ -611,7 +611,7 @@ def test_create_minimal_webhook_api_key(self, api_client, mailbox): mailbox, { "url": "https://hook.example.com/in", - "events": ["message.inbound"], + "trigger": "message.delivered", "auth_method": "api_key", }, ) @@ -628,7 +628,7 @@ def test_regenerate_secret_jwt_webhook(self, api_client, mailbox): mailbox, { "url": "https://hook.example.com/in", - "events": ["message.inbound"], + "trigger": "message.delivered", "auth_method": "jwt", }, ) @@ -656,7 +656,7 @@ def test_regenerate_secret_api_key_webhook(self, api_client, mailbox): mailbox, { "url": "https://hook.example.com/in", - "events": ["message.inbound"], + "trigger": "message.delivered", "auth_method": "api_key", }, ) @@ -684,17 +684,14 @@ def test_create_with_all_dispatcher_options(self, api_client, mailbox): mailbox, { "url": "https://hook.example.com/in", - "events": ["message.inbound"], + "trigger": "message.inbound", "auth_method": "jwt", - "phase": "before_spam", - "blocking": True, "format": "jmap", }, ) assert response.status_code == status.HTTP_201_CREATED, response.content channel = models.Channel.objects.get(id=response.data["id"]) - assert channel.settings["phase"] == "before_spam" - assert channel.settings["blocking"] is True + assert channel.settings["trigger"] == "message.inbound" assert channel.settings["format"] == "jmap" @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) @@ -705,7 +702,7 @@ def test_rejects_invalid_format(self, api_client, mailbox): mailbox, { "url": "https://hook.example.com/in", - "events": ["message.inbound"], + "trigger": "message.delivered", "auth_method": "jwt", "format": "yaml", }, @@ -720,7 +717,7 @@ def test_accepts_jmap_metadata_format(self, api_client, mailbox): mailbox, { "url": "https://hook.example.com/in", - "events": ["message.inbound"], + "trigger": "message.delivered", "auth_method": "jwt", "format": "jmap_metadata", }, @@ -728,37 +725,34 @@ def test_accepts_jmap_metadata_format(self, api_client, mailbox): assert response.status_code == status.HTTP_201_CREATED, response.content @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) - def test_rejects_invalid_phase(self, api_client, mailbox): - """An unknown webhook ``phase`` is rejected with HTTP 400.""" + def test_rejects_invalid_trigger(self, api_client, mailbox): + """An unknown webhook ``trigger`` is rejected with HTTP 400.""" response = self._post( api_client, mailbox, { "url": "https://hook.example.com/in", - "events": ["message.inbound"], + "trigger": "whenever", "auth_method": "jwt", - "phase": "whenever", }, ) assert response.status_code == status.HTTP_400_BAD_REQUEST @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) - def test_rejects_non_bool_blocking(self, api_client, mailbox): - """A non-boolean ``blocking`` value is rejected with HTTP 400.""" + def test_rejects_missing_trigger(self, api_client, mailbox): + """A webhook channel without a ``trigger`` is rejected with HTTP 400.""" response = self._post( api_client, mailbox, { "url": "https://hook.example.com/in", - "events": ["message.inbound"], "auth_method": "jwt", - "blocking": "yes", }, ) assert response.status_code == status.HTTP_400_BAD_REQUEST @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) - def test_rejects_patch_with_invalid_phase(self, api_client, mailbox): + def test_rejects_patch_with_invalid_trigger(self, api_client, mailbox): """A PATCH that touches only ``settings`` must still re-run the webhook validator (same airtight rule as api_key scopes).""" create = self._post( @@ -766,7 +760,7 @@ def test_rejects_patch_with_invalid_phase(self, api_client, mailbox): mailbox, { "url": "https://hook.example.com/in", - "events": ["message.inbound"], + "trigger": "message.delivered", "auth_method": "jwt", }, ) @@ -780,11 +774,75 @@ def test_rejects_patch_with_invalid_phase(self, api_client, mailbox): data={ "settings": { "url": "https://hook.example.com/in", - "events": ["message.inbound"], + "trigger": "bogus", "auth_method": "jwt", - "phase": "bogus", } }, format="json", ) assert response.status_code == status.HTTP_400_BAD_REQUEST + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + def test_partial_patch_preserves_auth_method(self, api_client, mailbox): + """A settings PATCH that omits ``auth_method`` keeps the existing + one instead of being rejected — auth_method pairs with the stored + secret and shouldn't have to be re-sent on every unrelated edit.""" + create = self._post( + api_client, + mailbox, + { + "url": "https://hook.example.com/in", + "trigger": "message.delivered", + "auth_method": "api_key", + }, + ) + assert create.status_code == status.HTTP_201_CREATED, create.content + channel_id = create.data["id"] + + url = reverse( + "mailbox-channels-detail", + kwargs={"mailbox_id": mailbox.id, "pk": channel_id}, + ) + response = api_client.patch( + url, + data={ + "settings": { + "url": "https://hook.example.com/changed", + "trigger": "message.delivered", + } + }, + format="json", + ) + assert response.status_code == status.HTTP_200_OK, response.content + channel = models.Channel.objects.get(id=channel_id) + # The new url stuck, and the omitted auth_method was carried forward + # and persisted (not dropped on the replace-save). + assert channel.settings["url"] == "https://hook.example.com/changed" + assert channel.settings["auth_method"] == "api_key" + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + def test_regenerate_secret_rejects_unknown_auth_method(self, api_client, mailbox): + """Rotating a webhook whose auth_method we can't surface is rejected + up front, so the old secret is never invalidated for a new one the + caller could never learn.""" + # An unknown auth_method (legacy/admin-edited row). A truthy value + # also stops the factory from auto-filling a valid "jwt". + channel = ChannelFactory( + type="webhook", + mailbox=mailbox, + settings={ + "url": "https://hook.example.com/in", + "trigger": "message.delivered", + "auth_method": "bogus", + }, + encrypted_settings={"secret": "whsec_original"}, + ) + url = reverse( + "mailbox-channels-regenerate-secret", + kwargs={"mailbox_id": mailbox.id, "pk": channel.id}, + ) + response = api_client.post(url) + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.content + channel.refresh_from_db() + # Old secret untouched — rotation never ran. + assert channel.encrypted_settings["secret"] == "whsec_original" diff --git a/src/backend/core/tests/api/test_messages_create.py b/src/backend/core/tests/api/test_messages_create.py index 366ffaf79..7f14b094c 100644 --- a/src/backend/core/tests/api/test_messages_create.py +++ b/src/backend/core/tests/api/test_messages_create.py @@ -380,7 +380,8 @@ def test_draft_and_send_message_success_delegated_access( assert sent_message.thread.messaged_at is not None assert all( - recipient.delivery_status == enums.MessageDeliveryStatusChoices.SENT + recipient.delivery_status + == enums.MessageDeliveryStatusChoices.SENT_EXTERNAL for recipient in sent_message.recipients.all() ) assert all( @@ -480,7 +481,8 @@ def test_send_message_failure( contact__email="success@external.com" ) assert ( - success_recipient.delivery_status == enums.MessageDeliveryStatusChoices.SENT + success_recipient.delivery_status + == enums.MessageDeliveryStatusChoices.SENT_EXTERNAL ) assert success_recipient.delivered_at is not None assert success_recipient.retry_count == 0 diff --git a/src/backend/core/tests/api/test_messages_delivery_statuses.py b/src/backend/core/tests/api/test_messages_delivery_statuses.py index 6c1714c7c..4a414a3f4 100644 --- a/src/backend/core/tests/api/test_messages_delivery_statuses.py +++ b/src/backend/core/tests/api/test_messages_delivery_statuses.py @@ -596,7 +596,7 @@ def test_api_messages_delivery_statuses_invalid_transition_sent_to_cancelled(sel ) recipient = factories.MessageRecipientFactory( message=message, - delivery_status=enums.MessageDeliveryStatusChoices.SENT, + delivery_status=enums.MessageDeliveryStatusChoices.SENT_EXTERNAL, ) client = APIClient() @@ -646,7 +646,7 @@ def test_api_messages_delivery_statuses_multiple_recipients(self): ) recipient_sent = factories.MessageRecipientFactory( message=message, - delivery_status=enums.MessageDeliveryStatusChoices.SENT, + delivery_status=enums.MessageDeliveryStatusChoices.SENT_EXTERNAL, ) thread.update_stats() @@ -686,7 +686,10 @@ def test_api_messages_delivery_statuses_multiple_recipients(self): recipient_retry.delivery_status == enums.MessageDeliveryStatusChoices.CANCELLED ) - assert recipient_sent.delivery_status == enums.MessageDeliveryStatusChoices.SENT + assert ( + recipient_sent.delivery_status + == enums.MessageDeliveryStatusChoices.SENT_EXTERNAL + ) thread.refresh_from_db() assert thread.has_delivery_failed is False @@ -720,7 +723,7 @@ def test_api_messages_delivery_statuses_partial_failure(self): ) recipient_sent = factories.MessageRecipientFactory( message=message, - delivery_status=enums.MessageDeliveryStatusChoices.SENT, + delivery_status=enums.MessageDeliveryStatusChoices.SENT_EXTERNAL, ) client = APIClient() @@ -743,7 +746,10 @@ def test_api_messages_delivery_statuses_partial_failure(self): recipient_failed.delivery_status == enums.MessageDeliveryStatusChoices.FAILED ) - assert recipient_sent.delivery_status == enums.MessageDeliveryStatusChoices.SENT + assert ( + recipient_sent.delivery_status + == enums.MessageDeliveryStatusChoices.SENT_EXTERNAL + ) def test_api_messages_delivery_statuses_retry_multiple_failed_recipients(self): """Test retry of multiple failed recipients at once.""" @@ -783,7 +789,7 @@ def test_api_messages_delivery_statuses_retry_multiple_failed_recipients(self): ) recipient_sent = factories.MessageRecipientFactory( message=message, - delivery_status=enums.MessageDeliveryStatusChoices.SENT, + delivery_status=enums.MessageDeliveryStatusChoices.SENT_EXTERNAL, ) thread.update_stats() @@ -824,7 +830,10 @@ def test_api_messages_delivery_statuses_retry_multiple_failed_recipients(self): assert recipient_failed_2.retry_at is None assert recipient_failed_2.delivery_message is None - assert recipient_sent.delivery_status == enums.MessageDeliveryStatusChoices.SENT + assert ( + recipient_sent.delivery_status + == enums.MessageDeliveryStatusChoices.SENT_EXTERNAL + ) thread.refresh_from_db() assert thread.has_delivery_failed is False diff --git a/src/backend/core/tests/api/test_messages_import.py b/src/backend/core/tests/api/test_messages_import.py index 032b4683f..01826bab0 100644 --- a/src/backend/core/tests/api/test_messages_import.py +++ b/src/backend/core/tests/api/test_messages_import.py @@ -155,7 +155,10 @@ def test_api_import_eml_file(api_client, user, mailbox, eml_file): assert message.sent_at == datetime.datetime( 2025, 5, 26, 20, 13, 44, tzinfo=datetime.timezone.utc ) - assert message.recipients.get().delivery_status == MessageDeliveryStatusChoices.SENT + assert ( + message.recipients.get().delivery_status + == MessageDeliveryStatusChoices.SENT_EXTERNAL + ) def test_api_import_mbox_file(api_client, user, mailbox, mbox_file): @@ -982,7 +985,7 @@ def test_api_import_imap_delivery_status_for_non_draft_messages( recipients = message.recipients.all() assert recipients.count() == 1 for recipient in recipients: - assert recipient.delivery_status == MessageDeliveryStatusChoices.SENT + assert recipient.delivery_status == MessageDeliveryStatusChoices.SENT_EXTERNAL # def test_api_import_mbox_multiple_times_threading(api_client, user, mailbox, mbox_file_path): diff --git a/src/backend/core/tests/api/test_prometheus_metrics.py b/src/backend/core/tests/api/test_prometheus_metrics.py index 6ae3e0c69..f700ff299 100644 --- a/src/backend/core/tests/api/test_prometheus_metrics.py +++ b/src/backend/core/tests/api/test_prometheus_metrics.py @@ -116,12 +116,20 @@ def test_get_messages_with_status_count_zero(self, api_client, settings, url): metrics = response_to_metrics_dict(response, with_label="status") assert ( - metrics[("message_status_count", MessageDeliveryStatusChoices.SENT.label)] + metrics[ + ( + "message_status_count", + MessageDeliveryStatusChoices.SENT_EXTERNAL.label, + ) + ] == 0 ) assert ( metrics[ - ("message_status_count", MessageDeliveryStatusChoices.INTERNAL.label) + ( + "message_status_count", + MessageDeliveryStatusChoices.SENT_INTERNAL.label, + ) ] == 0 ) @@ -144,8 +152,8 @@ def test_get_messages_with_status_count(self, api_client, settings, url): """ statuses_to_count = { - MessageDeliveryStatusChoices.SENT: 1, - MessageDeliveryStatusChoices.INTERNAL: 2, + MessageDeliveryStatusChoices.SENT_EXTERNAL: 1, + MessageDeliveryStatusChoices.SENT_INTERNAL: 2, MessageDeliveryStatusChoices.FAILED: 3, MessageDeliveryStatusChoices.RETRY: 4, } diff --git a/src/backend/core/tests/api/test_threads_list.py b/src/backend/core/tests/api/test_threads_list.py index b4dc3c056..65d64a046 100644 --- a/src/backend/core/tests/api/test_threads_list.py +++ b/src/backend/core/tests/api/test_threads_list.py @@ -1521,7 +1521,7 @@ def test_stats_mixed_delivery_statuses(self, api_client, url): # SENT - should not affect flags MessageRecipientFactory( message=message, - delivery_status=enums.MessageDeliveryStatusChoices.SENT, + delivery_status=enums.MessageDeliveryStatusChoices.SENT_EXTERNAL, ) # FAILED - should set has_delivery_failed MessageRecipientFactory( diff --git a/src/backend/core/tests/mda/test_autoreply.py b/src/backend/core/tests/mda/test_autoreply.py index 0342105dc..db6c0d15a 100644 --- a/src/backend/core/tests/mda/test_autoreply.py +++ b/src/backend/core/tests/mda/test_autoreply.py @@ -1,5 +1,5 @@ """Tests for autoreply MDA logic.""" -# pylint: disable=redefined-outer-name,unused-argument +# pylint: disable=redefined-outer-name,unused-argument,too-many-lines import base64 from datetime import datetime, timedelta @@ -795,6 +795,38 @@ def test_auto_reply_headers_in_mime( assert "Auto-Submitted: auto-replied" in mime_str assert "Precedence: bulk" in mime_str + @patch("core.mda.outbound_tasks.send_message_task", new_callable=MagicMock) + @patch("core.mda.outbound.sign_message_dkim", return_value=None) + def test_our_autoreply_does_not_trigger_another_autoreply( + self, mock_dkim, mock_send_task, mailbox, autoreply_template, inbound_message + ): + """Loop prevention, end to end: the MIME we compose for our own + autoreply must be classified as an automatic message when parsed + back, so feeding it through delivery never fires a second autoreply. + + This closes the loop between the *compose* side (which stamps + ``Auto-Submitted`` / ``X-Auto-Response-Suppress`` / ``Precedence``) + and the *detect* side (``_is_auto_reply_message``): a regression in + either — a renamed header, a parser change — would reopen the loop, + and the isolated unit tests on each side wouldn't catch it. + """ + # pylint: disable-next=import-outside-toplevel + from jmap_email import parse_email + + send_autoreply_for_message(autoreply_template, mailbox, inbound_message) + autoreply_msg = models.Message.objects.filter( + parent=inbound_message, is_sender=True + ).last() + + parsed = parse_email(autoreply_msg.blob.get_content()) + assert parsed is not None + + # The detector flags our own autoreply... + assert _is_auto_reply_message(parsed) is True + # ...and the full gate refuses to reply to it, so no loop can form + # even if this message were delivered back to an autoresponding box. + assert should_send_autoreply(mailbox, parsed) is None + @override_settings(MAX_OUTGOING_ATTACHMENT_SIZE=10) @patch("core.mda.outbound_tasks.send_message_task", new_callable=MagicMock) @patch("core.mda.outbound.sign_message_dkim", return_value=None) diff --git a/src/backend/core/tests/mda/test_dispatch_webhooks.py b/src/backend/core/tests/mda/test_dispatch_webhooks.py index 8a206f458..2b764c227 100644 --- a/src/backend/core/tests/mda/test_dispatch_webhooks.py +++ b/src/backend/core/tests/mda/test_dispatch_webhooks.py @@ -163,7 +163,8 @@ def test_finds_mailbox_scoped(self, mailbox): mailbox=mailbox, settings={ "url": "https://hook.example.com/a", - "events": ["message.inbound"], + "trigger": "message.delivered", + "auth_method": "jwt", }, ) assert list(find_webhook_channels_for_mailbox(mailbox)) == [ch] @@ -175,7 +176,8 @@ def test_finds_maildomain_scoped(self, mailbox): maildomain=mailbox.domain, settings={ "url": "https://hook.example.com/d", - "events": ["message.inbound"], + "trigger": "message.delivered", + "auth_method": "jwt", }, ) result = list(find_webhook_channels_for_mailbox(mailbox)) @@ -189,7 +191,8 @@ def test_finds_global_scoped(self, mailbox): scope_level=enums.ChannelScopeLevel.GLOBAL, settings={ "url": "https://hook.example.com/g", - "events": ["message.inbound"], + "trigger": "message.delivered", + "auth_method": "jwt", }, ) result = list(find_webhook_channels_for_mailbox(mailbox)) @@ -205,7 +208,8 @@ def test_global_fires_for_other_mailbox_too(self): scope_level=enums.ChannelScopeLevel.GLOBAL, settings={ "url": "https://hook.example.com/g", - "events": ["message.inbound"], + "trigger": "message.delivered", + "auth_method": "jwt", }, ) assert ch in find_webhook_channels_for_mailbox(mb_a) @@ -218,7 +222,8 @@ def test_excludes_other_mailbox(self, mailbox): mailbox=other, settings={ "url": "https://hook.example.com/x", - "events": ["message.inbound"], + "trigger": "message.delivered", + "auth_method": "jwt", }, ) assert not list(find_webhook_channels_for_mailbox(mailbox)) @@ -381,8 +386,8 @@ def test_skips_channel_with_wrong_phase(self, mock_session, mailbox, parsed_emai mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "before_spam", + "trigger": "message.inbound", + "auth_method": "jwt", }, ) outcome = dispatch_webhooks( @@ -396,15 +401,19 @@ def test_skips_channel_with_wrong_phase(self, mock_session, mailbox, parsed_emai mock_session.assert_not_called() @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_skips_channel_without_matching_event( + def test_skips_channel_with_unknown_trigger( self, mock_session, mailbox, parsed_email ): + """A channel whose trigger isn't a known WebhookTrigger fails closed + — it builds no step and never fires (e.g. a not-yet-supported event + like ``message.sent``).""" factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.sent"], + "trigger": "message.sent", + "auth_method": "jwt", }, ) outcome = dispatch_webhooks( @@ -427,8 +436,8 @@ def test_non_blocking_does_no_inline_io(self, mock_session, mailbox, parsed_emai mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": False, + "trigger": "message.delivered", + "auth_method": "jwt", }, ) outcome = dispatch_webhooks( @@ -450,8 +459,8 @@ def test_blocking_retries_on_5xx(self, mock_session, mailbox, parsed_email): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_session.return_value.post.return_value = _make_response(503) @@ -474,8 +483,8 @@ def test_blocking_retries_on_4xx(self, mock_session, mailbox, parsed_email): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_session.return_value.post.return_value = _make_response(403) @@ -497,8 +506,8 @@ def test_blocking_retries_on_408(self, mock_session, mailbox, parsed_email): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_session.return_value.post.return_value = _make_response(408) @@ -520,8 +529,8 @@ def test_blocking_retries_on_429(self, mock_session, mailbox, parsed_email): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_session.return_value.post.return_value = _make_response(429) @@ -546,8 +555,8 @@ def test_blocking_retries_on_ssrf_rejection( mailbox=mailbox, settings={ "url": "https://internal.example.com", - "events": ["message.inbound"], - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_session.return_value.post.side_effect = SSRFValidationError("blocked") @@ -569,8 +578,8 @@ def test_blocking_retries_on_timeout(self, mock_session, mailbox, parsed_email): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_session.return_value.post.side_effect = requests_lib.Timeout("timed out") @@ -594,8 +603,8 @@ def test_blocking_retries_on_connection_error( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_session.return_value.post.side_effect = requests_lib.ConnectionError( @@ -623,8 +632,8 @@ def test_blocking_retries_on_unknown_exception( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_session.return_value.post.side_effect = RuntimeError("boom") @@ -647,9 +656,8 @@ def test_phase_filtering_dispatches_only_matching( mailbox=mailbox, settings={ "url": "https://hook.example.com/before", - "events": ["message.inbound"], - "phase": "before_spam", - "blocking": True, + "trigger": "message.inbound", + "auth_method": "jwt", }, ) factories.ChannelFactory( @@ -657,9 +665,8 @@ def test_phase_filtering_dispatches_only_matching( mailbox=mailbox, settings={ "url": "https://hook.example.com/after", - "events": ["message.inbound"], - "phase": "after_spam", - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -681,8 +688,8 @@ def test_eml_format_sends_raw_body(self, mock_session, mailbox, parsed_email): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -708,9 +715,9 @@ def test_jmap_format_sends_jmap_email_json( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], + "trigger": "message.delivering", + "auth_method": "jwt", "format": "jmap", - "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -730,7 +737,7 @@ def test_jmap_format_sends_jmap_email_json( # Body IS the JMAP Email object — no wrapping envelope. assert body["messageId"] == ["mid@example.com"] assert body["from"] == [{"email": "sender@example.com", "name": "Sender"}] - assert "X-StMsg-Event" not in body + assert "X-StMsg-Trigger" not in body assert kwargs["headers"]["Content-Type"] == "application/json" @patch("core.mda.dispatch_webhooks.SSRFSafeSession") @@ -741,9 +748,9 @@ def test_jmap_metadata_skips_body_parts(self, mock_session, mailbox, parsed_emai mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], + "trigger": "message.delivering", + "auth_method": "jwt", "format": "jmap_metadata", - "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -784,9 +791,9 @@ def test_envelope_headers_set_for_both_formats( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], + "trigger": "message.delivering", + "auth_method": "jwt", "format": fmt, - "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -799,8 +806,7 @@ def test_envelope_headers_set_for_both_formats( is_spam=True, ) headers = mock_session.return_value.post.call_args.kwargs["headers"] - assert headers["X-StMsg-Event"] == "message.inbound" - assert headers["X-StMsg-Phase"] == "after_spam" + assert headers["X-StMsg-Trigger"] == "message.delivering" assert headers["X-StMsg-Mailbox"] == str(mailbox) assert headers["X-StMsg-Recipient"] == str(mailbox) assert headers["X-StMsg-Is-Spam"] == "true" @@ -817,9 +823,8 @@ def test_is_spam_header_unknown_when_none( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "before_spam", - "blocking": True, + "trigger": "message.inbound", + "auth_method": "jwt", }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -832,7 +837,7 @@ def test_is_spam_header_unknown_when_none( is_spam=None, ) headers = mock_session.return_value.post.call_args.kwargs["headers"] - assert headers["X-StMsg-Is-Spam"] == "unknown" + assert headers["X-StMsg-Is-Spam"] == "pending" @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_invalid_format_skips_dispatch(self, mock_session, mailbox, parsed_email): @@ -843,7 +848,8 @@ def test_invalid_format_skips_dispatch(self, mock_session, mailbox, parsed_email mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], + "trigger": "message.delivered", + "auth_method": "jwt", "format": "yaml", }, ) @@ -887,9 +893,9 @@ def test_eml_signature_covers_raw_body(self, mock_session, mailbox, parsed_email mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], + "trigger": "message.delivering", + "auth_method": "jwt", "format": "eml", - "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -927,9 +933,9 @@ def test_jmap_signature_covers_exact_serialised_bytes( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], + "trigger": "message.delivering", + "auth_method": "jwt", "format": "jmap", - "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -967,9 +973,8 @@ def test_api_key_mode_sends_only_api_key_header( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], + "trigger": "message.delivering", "auth_method": "api_key", - "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -1003,9 +1008,8 @@ def test_jwt_mode_sends_only_hmac_and_jwt_headers( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], + "trigger": "message.delivering", "auth_method": "jwt", - "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -1038,7 +1042,8 @@ def test_missing_auth_method_fails_closed( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], + "trigger": "message.delivered", + "auth_method": "jwt", }, encrypted_settings={"secret": "whsec_test"}, ) @@ -1064,9 +1069,8 @@ def test_api_key_value_is_derived_not_raw_secret( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], + "trigger": "message.delivering", "auth_method": "api_key", - "blocking": True, }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -1103,7 +1107,8 @@ def test_missing_secret_fails_closed(self, mock_session, mailbox, parsed_email): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], + "trigger": "message.delivered", + "auth_method": "jwt", }, encrypted_settings={}, ) @@ -1132,8 +1137,8 @@ def test_missing_secret_retries_when_blocking( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, encrypted_settings={}, ) @@ -1174,9 +1179,8 @@ def test_before_spam_blocking_retries_message( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "before_spam", - "blocking": True, + "trigger": "message.inbound", + "auth_method": "jwt", }, ) # 4xx is a webhook error, not an explicit drop → hold for RETRY. @@ -1212,9 +1216,8 @@ def test_after_spam_blocking_retries_message( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "after_spam", - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_check_spam.return_value = (False, None, None) @@ -1250,9 +1253,8 @@ def test_after_spam_is_spam_header( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "after_spam", - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_check_spam.return_value = (True, None, None) @@ -1265,7 +1267,7 @@ def test_after_spam_is_spam_header( # is_spam=True surfaces as the X-StMsg-Is-Spam header. headers = mock_session.return_value.post.call_args.kwargs["headers"] assert headers["X-StMsg-Is-Spam"] == "true" - assert headers["X-StMsg-Phase"] == "after_spam" + assert headers["X-StMsg-Trigger"] == "message.delivering" @patch("core.mda.inbound_tasks._create_message_from_inbound") @patch("core.mda.inbound_pipeline._call_rspamd") @@ -1333,8 +1335,7 @@ def test_non_blocking_records_pending_webhook( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": False, + "trigger": "message.delivered", "auth_method": "jwt", }, ) @@ -1343,7 +1344,7 @@ def test_non_blocking_records_pending_webhook( for step in webhook_steps_for_mailbox(mailbox, phase=PHASE_AFTER_SPAM): assert step(ctx) == Decision.CONTINUE - assert ctx.pending_webhooks == [(channel.id, PHASE_AFTER_SPAM, False)] + assert ctx.pending_webhooks == [(channel.id, False)] mock_session.return_value.post.assert_not_called() @patch("core.mda.dispatch_webhooks.dispatch_webhook_task") @@ -1357,7 +1358,7 @@ def test_dispatch_recorded_webhooks_enqueues_one_task_each( dispatch_recorded_webhooks( message, mailbox, - [(c1, PHASE_AFTER_SPAM, False), (c2, PHASE_BEFORE_SPAM, None)], + [(c1, False), (c2, None)], ) assert mock_task.delay.call_count == 2 @@ -1365,14 +1366,12 @@ def test_dispatch_recorded_webhooks_enqueues_one_task_each( str(message.id), str(c1), str(mailbox.id), - PHASE_AFTER_SPAM, False, ) assert mock_task.delay.call_args_list[1][0] == ( str(message.id), str(c2), str(mailbox.id), - PHASE_BEFORE_SPAM, None, ) @@ -1383,8 +1382,7 @@ def test_dispatch_webhook_task_posts_signed(self, mock_session, mailbox): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": False, + "trigger": "message.delivered", "auth_method": "jwt", }, ) @@ -1395,7 +1393,6 @@ def test_dispatch_webhook_task_posts_signed(self, mock_session, mailbox): str(message.id), str(channel.id), str(mailbox.id), - PHASE_AFTER_SPAM, False, ) @@ -1403,8 +1400,7 @@ def test_dispatch_webhook_task_posts_signed(self, mock_session, mailbox): # The signed body is the message blob content, rendered at task init. assert mock_session.return_value.post.call_args.kwargs["data"] == b"raw mime" headers = mock_session.return_value.post.call_args.kwargs["headers"] - assert headers["X-StMsg-Phase"] == "after_spam" - assert headers["X-StMsg-Event"] == enums.WebhookEvents.MESSAGE_INBOUND.value + assert headers["X-StMsg-Trigger"] == "message.delivered" # Signed at send time (jwt auth_method). assert "X-StMsg-Webhook-Signature" in headers # Fired post-persist, so the platform's Message/Thread ids ride along @@ -1422,7 +1418,6 @@ def test_dispatch_webhook_task_no_ops_when_channel_gone( str(message.id), str(uuid.uuid4()), str(mailbox.id), - PHASE_AFTER_SPAM, False, ) mock_session.return_value.post.assert_not_called() @@ -1436,8 +1431,7 @@ def test_dispatch_webhook_task_skips_when_message_gone(self, mock_session, mailb mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": False, + "trigger": "message.delivered", "auth_method": "jwt", }, ) @@ -1445,7 +1439,6 @@ def test_dispatch_webhook_task_skips_when_message_gone(self, mock_session, mailb str(uuid.uuid4()), str(channel.id), str(mailbox.id), - PHASE_AFTER_SPAM, False, ) mock_session.return_value.post.assert_not_called() @@ -1459,8 +1452,7 @@ def test_dispatch_webhook_task_swallows_send_errors(self, mock_session, mailbox) mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": False, + "trigger": "message.delivered", "auth_method": "jwt", }, ) @@ -1469,7 +1461,6 @@ def test_dispatch_webhook_task_swallows_send_errors(self, mock_session, mailbox) str(message.id), str(channel.id), str(mailbox.id), - PHASE_AFTER_SPAM, False, ) @@ -1497,8 +1488,9 @@ class TestInternalDeliveryWebhooks: sender and recipient may be fully unrelated tenants (different domains on the same instance), so the recipient's webhook outcome (drop / retry / failure) must NOT leak back into the sender's - delivery status: the sender sees ``INTERNAL`` the moment the message - is handed off, and the webhook plays out on the recipient's side. + delivery status: the sender sees ``SENT_INTERNAL`` the moment the + message is handed off to the recipient's async pipeline, and the webhook + plays out on the recipient's side. """ def _build_internal_message(self, sender_mailbox, recipient_email): @@ -1554,7 +1546,8 @@ def test_internal_delivery_fires_recipient_webhook( mailbox=recipient_mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], + "trigger": "message.delivered", + "auth_method": "jwt", }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -1562,18 +1555,22 @@ def test_internal_delivery_fires_recipient_webhook( message = self._build_internal_message(sender_mailbox, recipient_email) outbound.send_message(message) - # The recipient's webhook fired with the inbound event, addressed + # The recipient's webhook fired for the inbound message, addressed # to the recipient mailbox. assert mock_session.return_value.post.called headers = mock_session.return_value.post.call_args.kwargs["headers"] - assert headers["X-StMsg-Event"] == enums.WebhookEvents.MESSAGE_INBOUND.value + assert headers["X-StMsg-Trigger"] == "message.delivered" assert headers["X-StMsg-Recipient"] == recipient_email # Sender's view is decoupled from the recipient's webhook: it sees - # a clean internal handoff regardless of what the webhook returns. + # a clean internal handoff (SENT_INTERNAL) regardless of what the webhook + # returns. recipient = message.recipients.first() recipient.refresh_from_db() - assert recipient.delivery_status == enums.MessageDeliveryStatusChoices.INTERNAL + assert ( + recipient.delivery_status + == enums.MessageDeliveryStatusChoices.SENT_INTERNAL + ) @patch("core.mda.inbound_pipeline._call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") @@ -1592,9 +1589,8 @@ def test_recipient_webhook_failure_does_not_affect_sender( mailbox=recipient_mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "before_spam", - "blocking": True, + "trigger": "message.inbound", + "auth_method": "jwt", }, ) # 4xx is a webhook error → the recipient pipeline holds for RETRY. @@ -1603,10 +1599,13 @@ def test_recipient_webhook_failure_does_not_affect_sender( message = self._build_internal_message(sender_mailbox, recipient_email) outbound.send_message(message) - # Sender still sees a clean handoff. + # Sender still sees a clean internal handoff (SENT_INTERNAL). recipient = message.recipients.first() recipient.refresh_from_db() - assert recipient.delivery_status == enums.MessageDeliveryStatusChoices.INTERNAL + assert ( + recipient.delivery_status + == enums.MessageDeliveryStatusChoices.SENT_INTERNAL + ) # The recipient's queue row is held (not lost, not delivered), # and no message has landed in their mailbox yet. @@ -1631,7 +1630,8 @@ def test_internal_delivery_skips_spam_scan(self, mock_session, mock_rspamd): mailbox=recipient_mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], + "trigger": "message.delivered", + "auth_method": "jwt", }, ) mock_session.return_value.post.return_value = _make_response(200) @@ -1946,8 +1946,8 @@ def test_blocking_drops_on_action_drop_body( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_session.return_value.post.return_value = _make_response( @@ -1972,8 +1972,8 @@ def test_blocking_is_spam_override_continues( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_session.return_value.post.return_value = _make_response( @@ -2003,8 +2003,8 @@ def test_non_blocking_ignores_action_body( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": False, + "trigger": "message.delivered", + "auth_method": "jwt", }, ) outcome = dispatch_webhooks( @@ -2029,8 +2029,8 @@ def test_multi_webhook_drop_wins_and_short_circuits( mailbox=mailbox, settings={ "url": "https://hook.example.com/first", - "events": ["message.inbound"], - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) factories.ChannelFactory( @@ -2038,8 +2038,8 @@ def test_multi_webhook_drop_wins_and_short_circuits( mailbox=mailbox, settings={ "url": "https://hook.example.com/second", - "events": ["message.inbound"], - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) # First call drops, second should never fire. @@ -2071,8 +2071,8 @@ def test_response_body_is_capped(self, mock_session, mailbox, parsed_email): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) # Expose a stream far larger than the cap and count how much of @@ -2136,9 +2136,8 @@ def test_5xx_retries_and_keeps_inbound_message( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "before_spam", - "blocking": True, + "trigger": "message.inbound", + "auth_method": "jwt", }, ) mock_session.return_value.post.return_value = _make_response(503) @@ -2174,9 +2173,8 @@ def test_timeout_retries_and_keeps_inbound_message( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "before_spam", - "blocking": True, + "trigger": "message.inbound", + "auth_method": "jwt", }, ) mock_session.return_value.post.side_effect = requests_lib.Timeout("timed out") @@ -2217,9 +2215,8 @@ def test_blocking_webhook_quarantine_delivers_flagged_after_window( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "before_spam", - "blocking": True, + "trigger": "message.inbound", + "auth_method": "jwt", }, ) mock_session.return_value.post.return_value = _make_response(503) @@ -2242,6 +2239,50 @@ def test_blocking_webhook_quarantine_delivers_flagged_after_window( # Queue row consumed — not pinned, not dropped silently. assert not models.InboundMessage.objects.filter(id=inbound_message.id).exists() + @patch("core.mda.autoreply.try_send_autoreply") + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_blocking_webhook_quarantine_suppresses_autoreply( + self, mock_session, _mock_rspamd, mock_autoreply + ): + """A quarantine delivery forces ``is_spam=False`` to surface the + warning banner, but must NOT fire an autoreply: the spam verdict is + unverified and the blocking step that might have suppressed the + reply never completed.""" + mailbox = factories.MailboxFactory() + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: test\r\n" + b"Message-ID: \r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + # Backdate past the quarantine window so the failing webhook stops + # holding and the message is force-delivered. + models.InboundMessage.objects.filter(id=inbound_message.id).update( + created_at=dj_timezone.now() - QUARANTINE_AFTER - timedelta(minutes=1) + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "trigger": "message.inbound", + "auth_method": "jwt", + }, + ) + # Webhook keeps failing → RETRY → quarantine-delivered. + mock_session.return_value.post.return_value = _make_response(503) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + process_inbound_message_task.run(str(inbound_message.id)) + + # Force-delivered (row consumed) but no autoreply fired. + assert not models.InboundMessage.objects.filter(id=inbound_message.id).exists() + mock_autoreply.assert_not_called() + @pytest.mark.django_db class TestPipelineWebhookAntispam: @@ -2267,9 +2308,8 @@ def test_before_spam_is_spam_override_short_circuits_rspamd( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "before_spam", - "blocking": True, + "trigger": "message.inbound", + "auth_method": "jwt", }, ) mock_session.return_value.post.return_value = _make_response( @@ -2307,9 +2347,8 @@ def test_after_spam_is_spam_override_replaces_verdict( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "after_spam", - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) # rspamd says ham; webhook says spam. @@ -2354,9 +2393,8 @@ def test_webhook_label_attached_to_thread(self, mock_session, mock_check_spam): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "after_spam", - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_check_spam.return_value = (False, None, None) @@ -2417,9 +2455,8 @@ def test_assign_to_resolves_email_and_attributes_channel( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "after_spam", - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_check_spam.return_value = (False, None, None) @@ -2493,9 +2530,8 @@ def test_assign_to_skips_unknown_ambiguous_and_viewer( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "after_spam", - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_check_spam.return_value = (False, None, None) @@ -2558,9 +2594,8 @@ def test_two_webhooks_each_produce_own_threadevent( mailbox=mailbox, settings={ "url": "https://hook.example.com/a", - "events": ["message.inbound"], - "phase": "after_spam", - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) ch_b = factories.ChannelFactory( @@ -2568,9 +2603,8 @@ def test_two_webhooks_each_produce_own_threadevent( mailbox=mailbox, settings={ "url": "https://hook.example.com/b", - "events": ["message.inbound"], - "phase": "after_spam", - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_check_spam.return_value = (False, None, None) @@ -2626,9 +2660,8 @@ def _send(self, mailbox, mime_id, action_body: bytes): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "after_spam", - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) with ( @@ -2688,9 +2721,8 @@ def test_skip_autoreply_suppresses_autoreply_call( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "after_spam", - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_rspamd.return_value = (False, None, None) @@ -2729,9 +2761,8 @@ def test_add_event_im_creates_threadevent(self, mock_session, mock_rspamd): mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "after_spam", - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_rspamd.return_value = (False, None, None) @@ -2803,9 +2834,8 @@ def test_reply_draft_creates_draft_with_template_body( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "after_spam", - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_rspamd.return_value = (False, None, None) @@ -2865,9 +2895,8 @@ def test_reply_draft_out_of_scope_template_skipped(self, mock_session, mock_rspa mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "after_spam", - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_rspamd.return_value = (False, None, None) @@ -2928,9 +2957,8 @@ def test_assign_failure_does_not_skip_labels( mailbox=mailbox, settings={ "url": "https://hook.example.com", - "events": ["message.inbound"], - "phase": "after_spam", - "blocking": True, + "trigger": "message.delivering", + "auth_method": "jwt", }, ) mock_rspamd.return_value = (False, None, None) diff --git a/src/backend/core/tests/mda/test_inbound_spoofed_sender.py b/src/backend/core/tests/mda/test_inbound_spoofed_sender.py index aeab9239f..e377710f3 100644 --- a/src/backend/core/tests/mda/test_inbound_spoofed_sender.py +++ b/src/backend/core/tests/mda/test_inbound_spoofed_sender.py @@ -228,7 +228,7 @@ def test_inbound_spoofed_sender_import_path_never_picked_up_by_retry( # guarantee that imports never enter the retry pipeline. assert ( message.recipients.filter( - delivery_status=enums.MessageDeliveryStatusChoices.SENT + delivery_status=enums.MessageDeliveryStatusChoices.SENT_EXTERNAL ).count() == message.recipients.count() ) diff --git a/src/backend/core/tests/mda/test_outbound.py b/src/backend/core/tests/mda/test_outbound.py index 4afa71e1b..6305db4c1 100644 --- a/src/backend/core/tests/mda/test_outbound.py +++ b/src/backend/core/tests/mda/test_outbound.py @@ -223,7 +223,7 @@ def test_outbound_send_relay(self, mock_smtp_send, draft_message): assert draft_message.recipients.count() == 4 assert ( draft_message.recipients.filter( - delivery_status=enums.MessageDeliveryStatusChoices.SENT + delivery_status=enums.MessageDeliveryStatusChoices.SENT_EXTERNAL ).count() == 2 ) @@ -328,7 +328,7 @@ def resolve_return_value(domain, record_type, **kwargs): assert draft_message.recipients.count() == 4 assert ( draft_message.recipients.filter( - delivery_status=enums.MessageDeliveryStatusChoices.SENT + delivery_status=enums.MessageDeliveryStatusChoices.SENT_EXTERNAL ).count() == 2 ) @@ -456,7 +456,7 @@ def smtp_return_value(*args, **kwargs): assert ( draft_message.recipients.filter( contact__email="bcc@example2.com", - delivery_status=enums.MessageDeliveryStatusChoices.SENT, + delivery_status=enums.MessageDeliveryStatusChoices.SENT_EXTERNAL, ).count() == 1 ) @@ -1287,7 +1287,10 @@ def mock_dns_resolve_func(query_name, record_type, **kwargs): # Verify message was sent (not marked for retry) message.refresh_from_db() recipient = message.recipients.first() - assert recipient.delivery_status == enums.MessageDeliveryStatusChoices.SENT + assert ( + recipient.delivery_status + == enums.MessageDeliveryStatusChoices.SENT_EXTERNAL + ) assert mock_send_outbound.called @override_settings(MESSAGES_DKIM_VERIFY_OUTGOING=True) @@ -1424,6 +1427,60 @@ def test_dkim_verification_skipped_for_internal_recipients( # Verify internal delivery was attempted assert mock_deliver_inbound.called + @override_settings(MESSAGES_ALLOW_INTERNAL_DELIVERY=False) + @patch("core.mda.outbound.send_outbound_message") + @patch("core.mda.outbound.deliver_inbound_message") + def test_internal_delivery_disabled_routes_local_recipient_external( + self, mock_deliver_inbound, mock_send_outbound, mailbox_sender + ): + """With MESSAGES_ALLOW_INTERNAL_DELIVERY=False, a same-instance + recipient is routed out through the MTA instead of the internal + fast path.""" + thread = factories.ThreadFactory() + factories.ThreadAccessFactory( + mailbox=mailbox_sender, + thread=thread, + role=enums.ThreadAccessRoleChoices.EDITOR, + ) + sender_contact = factories.ContactFactory(mailbox=mailbox_sender) + message = factories.MessageFactory( + thread=thread, + sender=sender_contact, + is_draft=False, + is_sender=True, + subject="No internal fast path", + ) + recipient_email = f"internal@{mailbox_sender.domain.name}" + raw_mime = ( + f"From: {sender_contact.email}\r\n" + + f"To: {recipient_email}\r\n" + + "Subject: Test\r\n\r\nBody\r\n" + ).encode() + message.blob = factories.BlobFactory( + mailbox=mailbox_sender, content=raw_mime, content_type="message/rfc822" + ) + message.save() + # A genuinely local recipient (same instance). + internal_mailbox = factories.MailboxFactory( + domain=mailbox_sender.domain, local_part="internal" + ) + internal_contact = factories.ContactFactory( + mailbox=internal_mailbox, email=recipient_email + ) + factories.MessageRecipientFactory( + message=message, + contact=internal_contact, + type=models.MessageRecipientTypeChoices.TO, + ) + mock_send_outbound.return_value = {recipient_email: {"delivered": True}} + + outbound.send_message(message) + + # The local recipient went out through the MTA, not the internal + # fast path. + assert not mock_deliver_inbound.called + assert mock_send_outbound.called + def _create_spf_test_message(mailbox_sender): """Create the common objects needed by SPF check tests. diff --git a/src/backend/core/tests/mda/test_outbound_e2e.py b/src/backend/core/tests/mda/test_outbound_e2e.py index 6dc3ec786..eed85ae0a 100644 --- a/src/backend/core/tests/mda/test_outbound_e2e.py +++ b/src/backend/core/tests/mda/test_outbound_e2e.py @@ -215,17 +215,20 @@ def _test( ) assert send_response.status_code == status.HTTP_200_OK, send_response.content + # Same-instance recipients are delivered internally and marked + # SENT_INTERNAL; external recipients are marked SENT_EXTERNAL on MTA + # hand-off. assert ( models.MessageRecipient.objects.filter( message__id=draft_message_id, - delivery_status=enums.MessageDeliveryStatusChoices.INTERNAL, + delivery_status=enums.MessageDeliveryStatusChoices.SENT_INTERNAL, ).count() == 1 ) assert ( models.MessageRecipient.objects.filter( message__id=draft_message_id, - delivery_status=enums.MessageDeliveryStatusChoices.SENT, + delivery_status=enums.MessageDeliveryStatusChoices.SENT_EXTERNAL, ).count() == 3 ) diff --git a/src/backend/core/tests/mda/test_retry.py b/src/backend/core/tests/mda/test_retry.py index 98b5a897d..385cf339f 100644 --- a/src/backend/core/tests/mda/test_retry.py +++ b/src/backend/core/tests/mda/test_retry.py @@ -79,7 +79,7 @@ def message_with_recipients(self, mailbox_sender, thread): message=message, contact=bcc_contact, type=models.MessageRecipientTypeChoices.BCC, - delivery_status=enums.MessageDeliveryStatusChoices.SENT, + delivery_status=enums.MessageDeliveryStatusChoices.SENT_EXTERNAL, delivered_at=timezone.now(), ) @@ -410,7 +410,7 @@ def test_retry_mixed_recipient_statuses( message=message, contact=sent_contact, type=models.MessageRecipientTypeChoices.CC, - delivery_status=enums.MessageDeliveryStatusChoices.SENT, + delivery_status=enums.MessageDeliveryStatusChoices.SENT_EXTERNAL, delivered_at=timezone.now(), ) @@ -465,7 +465,7 @@ def test_retry_message_with_no_retryable_recipients( message=message, contact=sent_contact, type=models.MessageRecipientTypeChoices.TO, - delivery_status=enums.MessageDeliveryStatusChoices.SENT, + delivery_status=enums.MessageDeliveryStatusChoices.SENT_EXTERNAL, delivered_at=timezone.now(), ) diff --git a/src/backend/core/tests/services/test_ssrf.py b/src/backend/core/tests/services/test_ssrf.py index 689f42f10..362314660 100644 --- a/src/backend/core/tests/services/test_ssrf.py +++ b/src/backend/core/tests/services/test_ssrf.py @@ -307,3 +307,71 @@ def test_intermediate_response_closed(self, mock_dns, mock_get): intermediate.close.assert_called_once() final.close.assert_not_called() + + +class TestSSRFSafeSessionPostRedirects: + """Redirect-handling contract for SSRFSafeSession.post. + + POST follows redirects too (a webhook endpoint behind a load balancer / + canonicaliser commonly 3xx-redirects), re-validating each hop and + re-issuing the POST so the signed body reaches the final destination. + """ + + @patch("core.services.ssrf.requests.Session.post") + @patch("core.services.ssrf.socket.getaddrinfo") + def test_no_redirect_returns_response_directly(self, mock_dns, mock_post): + """A 2xx POST is returned without any extra request.""" + mock_dns.return_value = _addrinfo(PUBLIC_IP) + mock_post.return_value = _mock_response(200) + + response = SSRFSafeSession().post( + "https://hook.legit.com/in", timeout=10, data=b"payload" + ) + + assert response.status_code == 200 + assert mock_post.call_count == 1 + + @pytest.mark.parametrize("redirect_status", [301, 302, 303, 307, 308]) + @patch("core.services.ssrf.requests.Session.post") + @patch("core.services.ssrf.socket.getaddrinfo") + def test_post_follows_redirect_preserving_method_and_body( + self, mock_dns, mock_post, redirect_status + ): + """A 3xx on POST is followed by re-POSTing the same body to the + validated Location (method preserved, never downgraded to GET).""" + mock_dns.return_value = _addrinfo(PUBLIC_IP) + mock_post.side_effect = [ + _mock_response(redirect_status, location="https://cdn.legit.com/in"), + _mock_response(200), + ] + + response = SSRFSafeSession().post( + "https://hook.legit.com/in", timeout=10, data=b"payload" + ) + + assert response.status_code == 200 + assert mock_post.call_count == 2 + # The body rode along on the followed hop, and we never fell back to GET. + assert mock_post.call_args.kwargs.get("data") == b"payload" + + @patch("core.services.ssrf.requests.Session.post") + @patch("core.services.ssrf.socket.getaddrinfo") + def test_post_blocks_redirect_to_private_ip(self, mock_dns, mock_post): + """A POST Location resolving to a private IP is rejected mid-chain.""" + + def dns_side_effect(host, *_args, **_kwargs): + if host == "hook.legit.com": + return _addrinfo(PUBLIC_IP) + if host == "internal.evil.com": + return _addrinfo(PRIVATE_IP) + raise AssertionError(f"unexpected DNS lookup: {host}") + + mock_dns.side_effect = dns_side_effect + mock_post.return_value = _mock_response( + 302, location="https://internal.evil.com/pwn" + ) + + with pytest.raises(SSRFValidationError, match="private IP"): + SSRFSafeSession().post("https://hook.legit.com/in", timeout=10, data=b"x") + + assert mock_post.call_count == 1 diff --git a/src/backend/core/tests/test_signals.py b/src/backend/core/tests/test_signals.py index 4bc7b1b07..b09761390 100644 --- a/src/backend/core/tests/test_signals.py +++ b/src/backend/core/tests/test_signals.py @@ -52,7 +52,7 @@ def test_signal_triggers_on_delivery_status_change(self): ) with patch.object(thread, "update_stats") as mock_update_stats: - recipient.delivery_status = enums.MessageDeliveryStatusChoices.SENT + recipient.delivery_status = enums.MessageDeliveryStatusChoices.SENT_EXTERNAL recipient.save(update_fields=["delivery_status"]) mock_update_stats.assert_called_once() @@ -72,7 +72,7 @@ def test_signal_does_not_trigger_for_non_sender_message(self): ) with patch.object(thread, "update_stats") as mock_update_stats: - recipient.delivery_status = enums.MessageDeliveryStatusChoices.SENT + recipient.delivery_status = enums.MessageDeliveryStatusChoices.SENT_EXTERNAL recipient.save(update_fields=["delivery_status"]) mock_update_stats.assert_not_called() @@ -92,7 +92,7 @@ def test_signal_does_not_trigger_for_draft_message(self): ) with patch.object(thread, "update_stats") as mock_update_stats: - recipient.delivery_status = enums.MessageDeliveryStatusChoices.SENT + recipient.delivery_status = enums.MessageDeliveryStatusChoices.SENT_EXTERNAL recipient.save(update_fields=["delivery_status"]) mock_update_stats.assert_not_called() @@ -112,7 +112,7 @@ def test_signal_does_not_trigger_for_trashed_message(self): ) with patch.object(thread, "update_stats") as mock_update_stats: - recipient.delivery_status = enums.MessageDeliveryStatusChoices.SENT + recipient.delivery_status = enums.MessageDeliveryStatusChoices.SENT_EXTERNAL recipient.save(update_fields=["delivery_status"]) mock_update_stats.assert_not_called() @@ -128,7 +128,7 @@ def test_signal_does_not_trigger_for_other_field_changes(self): ) recipient = factories.MessageRecipientFactory( message=message, - delivery_status=enums.MessageDeliveryStatusChoices.SENT, + delivery_status=enums.MessageDeliveryStatusChoices.SENT_EXTERNAL, ) with patch.object(thread, "update_stats") as mock_update_stats: @@ -161,7 +161,9 @@ def test_defers_update_until_context_exit(self): with patch("core.models.Thread.update_stats") as mock_update_stats: with ThreadStatsUpdateDeferrer.defer(): - recipient1.delivery_status = enums.MessageDeliveryStatusChoices.SENT + recipient1.delivery_status = ( + enums.MessageDeliveryStatusChoices.SENT_EXTERNAL + ) recipient1.save(update_fields=["delivery_status"]) recipient2.delivery_status = enums.MessageDeliveryStatusChoices.FAILED @@ -190,7 +192,9 @@ def test_nested_contexts_only_update_once(self): with patch("core.models.Thread.update_stats") as mock_update_stats: with ThreadStatsUpdateDeferrer.defer(): with ThreadStatsUpdateDeferrer.defer(): - recipient.delivery_status = enums.MessageDeliveryStatusChoices.SENT + recipient.delivery_status = ( + enums.MessageDeliveryStatusChoices.SENT_EXTERNAL + ) recipient.save(update_fields=["delivery_status"]) # Inner context exited, should not have been called yet @@ -226,10 +230,14 @@ def test_multiple_threads_updated(self): with patch("core.models.Thread.update_stats") as mock_update_stats: with ThreadStatsUpdateDeferrer.defer(): - recipient1.delivery_status = enums.MessageDeliveryStatusChoices.SENT + recipient1.delivery_status = ( + enums.MessageDeliveryStatusChoices.SENT_EXTERNAL + ) recipient1.save(update_fields=["delivery_status"]) - recipient2.delivery_status = enums.MessageDeliveryStatusChoices.SENT + recipient2.delivery_status = ( + enums.MessageDeliveryStatusChoices.SENT_EXTERNAL + ) recipient2.save(update_fields=["delivery_status"]) # Should be called twice, once per thread @@ -267,10 +275,14 @@ def test_update_stats_error_does_not_propagate(self): ) as mock_update_stats: # Should not raise, error is caught and logged with ThreadStatsUpdateDeferrer.defer(): - recipient1.delivery_status = enums.MessageDeliveryStatusChoices.SENT + recipient1.delivery_status = ( + enums.MessageDeliveryStatusChoices.SENT_EXTERNAL + ) recipient1.save(update_fields=["delivery_status"]) - recipient2.delivery_status = enums.MessageDeliveryStatusChoices.SENT + recipient2.delivery_status = ( + enums.MessageDeliveryStatusChoices.SENT_EXTERNAL + ) recipient2.save(update_fields=["delivery_status"]) # Both should have been attempted @@ -517,7 +529,7 @@ def test_recipient_update_after_import_still_coalesces( patch("core.signals.enqueue_thread_reindex") as mock_enqueue, django_capture_on_commit_callbacks(execute=True), ): - recipient.delivery_status = enums.MessageDeliveryStatusChoices.SENT + recipient.delivery_status = enums.MessageDeliveryStatusChoices.SENT_EXTERNAL recipient.save(update_fields=["delivery_status"]) mock_enqueue.assert_called_once_with(message.thread_id) diff --git a/src/backend/e2e/management/commands/e2e_demo.py b/src/backend/e2e/management/commands/e2e_demo.py index f9cd17581..a277487ce 100644 --- a/src/backend/e2e/management/commands/e2e_demo.py +++ b/src/backend/e2e/management/commands/e2e_demo.py @@ -321,7 +321,7 @@ def _create_outbox_test_data(self, domain, browser): ( "sent@external.invalid", "Sent Recipient", - MessageDeliveryStatusChoices.SENT, + MessageDeliveryStatusChoices.SENT_EXTERNAL, None, {"delivered_at": timezone.now()}, ), @@ -352,7 +352,7 @@ def _create_outbox_test_data(self, domain, browser): ( "delivered@external.invalid", "Delivered Recipient", - MessageDeliveryStatusChoices.SENT, + MessageDeliveryStatusChoices.SENT_EXTERNAL, None, {"delivered_at": timezone.now()}, ), diff --git a/src/backend/messages/settings.py b/src/backend/messages/settings.py index 30a637158..228278e9a 100644 --- a/src/backend/messages/settings.py +++ b/src/backend/messages/settings.py @@ -620,6 +620,17 @@ class Base(Configuration): environ_prefix=None, ) + # Deliver mailbox-to-mailbox mail through the internal inbound pipeline + # (the fast path) instead of routing it out through the MTA. Set False to + # force every message — including same-instance recipients — through the + # external MTA path, e.g. so all mail passes the same scanning/archiving + # as outbound. Default True keeps the local fast path. + MESSAGES_ALLOW_INTERNAL_DELIVERY = values.BooleanValue( + default=True, + environ_name="MESSAGES_ALLOW_INTERNAL_DELIVERY", + environ_prefix=None, + ) + # Technical domain for DNS records (MX, SPF, DKIM hosting) MESSAGES_TECHNICAL_DOMAIN = values.Value( "localhost", environ_name="MESSAGES_TECHNICAL_DOMAIN", environ_prefix=None diff --git a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx index e049c49b0..3d8f9ec3e 100644 --- a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx +++ b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx @@ -19,7 +19,6 @@ import { import { RhfInput, RhfSelect, - RhfCheckbox, } from "@/features/forms/components/react-hook-form"; import { addToast, ToasterItem } from "@/features/ui/components/toaster"; import { Banner } from "@/features/ui/components/banner"; @@ -31,12 +30,19 @@ import { handle } from "@/features/utils/errors"; // URLs; everywhere else https is required, so mirror that client-side. const DEV_ENVIRONMENTS = ["development", "developmentminimal", "e2e"]; +// A single flat trigger lifecycle event replaces the old (events, phase, +// blocking) triple — the event name itself says when it fires and whether +// it blocks, so only valid combinations are representable. Mirrors +// enums.WebhookTrigger. +type WebhookTrigger = + | "message.inbound" + | "message.delivering" + | "message.delivered"; + type WebhookChannelSettings = { url?: string; - events?: string[]; - phase?: "before_spam" | "after_spam"; + trigger?: WebhookTrigger; format?: "eml" | "jmap" | "jmap_metadata"; - blocking?: boolean; auth_method?: "jwt" | "api_key"; }; @@ -66,9 +72,12 @@ const createFormSchema = ( ? t("URL must start with http:// or https://") : t("URL must start with https://"), }), - phase: z.enum(["before_spam", "after_spam"]), + trigger: z.enum([ + "message.inbound", + "message.delivering", + "message.delivered", + ]), format: z.enum(["eml", "jmap", "jmap_metadata"]), - blocking: z.boolean(), auth_method: z.enum(["jwt", "api_key"]), }); @@ -102,9 +111,8 @@ export const WebhookIntegrationForm = ({ defaultValues: { name: channel?.name || "", url: settings?.url || "", - phase: settings?.phase || "after_spam", + trigger: settings?.trigger || "message.delivered", format: settings?.format || "eml", - blocking: settings?.blocking ?? false, auth_method: settings?.auth_method || "jwt", }, }); @@ -169,10 +177,8 @@ export const WebhookIntegrationForm = ({ const newSettings: WebhookChannelSettings = { url: data.url, - events: ["message.inbound"], - phase: data.phase, + trigger: data.trigger, format: data.format, - blocking: data.blocking, auth_method: data.auth_method, }; @@ -320,21 +326,33 @@ export const WebhookIntegrationForm = ({

    {t("Behavior")}

    + {/* One flat trigger: a lifecycle event whose name says + both when it fires and whether it blocks delivery. */} @@ -360,17 +378,11 @@ export const WebhookIntegrationForm = ({ )} fullWidth /> -

    {t( - "Non-blocking (default) is the safe choice:", + "Message delivered (default) is the safe choice:", )} {" "} {t( @@ -380,7 +392,7 @@ export const WebhookIntegrationForm = ({

    {t( - "Blocking lets the receiver act on this single message", + "Blocking triggers let the receiver act on this single message", )} {" "} {t( diff --git a/src/frontend/src/features/layouts/components/thread-view/components/thread-message/thread-message-header.tsx b/src/frontend/src/features/layouts/components/thread-view/components/thread-message/thread-message-header.tsx index 9e8432da1..5c86bb848 100644 --- a/src/frontend/src/features/layouts/components/thread-view/components/thread-message/thread-message-header.tsx +++ b/src/frontend/src/features/layouts/components/thread-view/components/thread-message/thread-message-header.tsx @@ -78,6 +78,10 @@ const ThreadMessageHeader = ({ return { status: 'delivering', timestamp: recipient.retry_at, message: recipient.delivery_message }; case MessageDeliveryStatusChoices.cancelled: return { status: 'cancelled', timestamp: recipient.retry_at, message: recipient.delivery_message }; + // `sent` (external) and `internal` (same-instance) are both + // terminal "delivered" states — keep them together. Dropping the + // `internal` case would silently render internal mail as "no + // status". See MessageDeliveryStatusChoices on the backend. case MessageDeliveryStatusChoices.sent: case MessageDeliveryStatusChoices.internal: return { status: 'delivered', timestamp: recipient.delivered_at, message: recipient.delivery_message }; From 3a04607b764c4cf49bc4cf7b46fdbbec3d7ea5b2 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Mon, 29 Jun 2026 01:34:40 +0200 Subject: [PATCH 12/21] improve comments --- src/backend/core/models.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/backend/core/models.py b/src/backend/core/models.py index 69b962ef0..bc97d7bbb 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -618,6 +618,10 @@ def rotate_secret(self, *, save: bool = True) -> str: if self.type == ChannelTypes.API_KEY: plaintext = "msgk_" + secrets.token_urlsafe(32) + # SHA-256 (not Argon2/bcrypt) is correct: this hashes a 256-bit + # random token for O(1) lookup, not a low-entropy password — a + # slow KDF would add nothing. (CodeQL py/weak-sensitive-data- + # hashing false positive.) digest = hashlib.sha256(plaintext.encode("utf-8")).hexdigest() self.encrypted_settings = { **(self.encrypted_settings or {}), @@ -653,6 +657,10 @@ def get_webhook_api_key(self) -> Optional[str]: root = (self.encrypted_settings or {}).get("secret") if not root: return None + # HMAC-SHA256 key derivation from a 256-bit random root (not password + # hashing): a deterministic, unsalted PRF is required so the same root + # always yields the same key. Argon2/bcrypt are slow/salted and would + # break that. (CodeQL py/weak-sensitive-data-hashing false positive.) derived = hmac.new( root.encode("utf-8"), self.WEBHOOK_API_KEY_KDF_LABEL, From 27222a77b3e578cf3af66ee370c9c64ac282c123 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Mon, 29 Jun 2026 09:47:47 +0200 Subject: [PATCH 13/21] continue reviewing --- src/backend/core/admin.py | 17 +- src/backend/core/mda/dispatch_webhooks.py | 22 +- src/backend/core/mda/inbound_pipeline.py | 16 +- src/backend/core/mda/inbound_tasks.py | 9 +- .../admin/core/channel/change_form.html | 2 +- .../tests/api/test_channel_scope_level.py | 4 +- .../api/test_messages_delivery_statuses.py | 47 +- src/backend/core/tests/mda/test_autoreply.py | 24 +- .../core/tests/mda/test_dispatch_webhooks.py | 107 ++++ src/frontend/package-lock.json | 522 ------------------ src/frontend/public/locales/common/nl-NL.json | 6 +- src/frontend/public/locales/common/ru-RU.json | 6 +- src/frontend/public/locales/common/uk-UA.json | 6 +- .../modal-compose-integration/_index.scss | 31 +- .../webhook-integration-form.tsx | 22 +- .../components/thread-event/index.tsx | 15 +- 16 files changed, 278 insertions(+), 578 deletions(-) diff --git a/src/backend/core/admin.py b/src/backend/core/admin.py index 575b728ec..becd8303b 100644 --- a/src/backend/core/admin.py +++ b/src/backend/core/admin.py @@ -593,7 +593,7 @@ def regenerate_secret_view(self, request, object_id): through signed-cookie message storage. """ # pylint: disable=import-outside-toplevel - from core.enums import ChannelTypes + from core.enums import ChannelTypes, WebhookAuthMethod if request.method != "POST": return HttpResponseNotAllowed(["POST"]) @@ -608,6 +608,21 @@ def regenerate_secret_view(self, request, object_id): "Only api_key and webhook channels can have their secret regenerated.", ) return redirect("..") + # Guard before rotating: a webhook channel whose auth_method isn't + # one ``get_webhook_surfaced_credential`` knows how to surface would + # have its old secret invalidated by ``rotate_secret`` while the + # freshly minted one can't be handed back — permanently bricking the + # webhook. Reject up front, mirroring the DRF regenerate flow. + if ( + channel.type == ChannelTypes.WEBHOOK + and (channel.settings or {}).get("auth_method") not in WebhookAuthMethod + ): + messages.error( + request, + "Webhook auth_method must be 'jwt' or 'api_key' before " + "the secret can be rotated.", + ) + return redirect("..") root = channel.rotate_secret() diff --git a/src/backend/core/mda/dispatch_webhooks.py b/src/backend/core/mda/dispatch_webhooks.py index f91716370..fade9f5e5 100644 --- a/src/backend/core/mda/dispatch_webhooks.py +++ b/src/backend/core/mda/dispatch_webhooks.py @@ -153,9 +153,13 @@ def _read_capped_body(response, deadline: Optional[float] = None) -> bytes: break chunks.append(chunk) received += len(chunk) - except TimeoutError: - raise except Exception as exc: + # TimeoutError MUST propagate so the caller treats it as RETRY. + # Everything else (network blip mid-stream) is logged silently — + # the body we have so far (possibly empty) is returned and the + # caller classifies it as a benign empty-body CONTINUE. + if isinstance(exc, TimeoutError): + raise # Don't interpolate ``exc`` — its text can echo the request URL or # body and leak receiver secrets into logs. The type name is enough. logger.warning("Truncated response body read failed (%s)", type(exc).__name__) @@ -985,4 +989,18 @@ def webhook_steps_for_mailbox( ) continue steps.append(UserWebhookStep(channel, phase=phase)) + + # Within the after-spam phase, blocking triggers (message.delivering) + # must run before non-blocking ones (message.delivered) so the latter + # records the final ``ctx.is_spam`` — a delivering webhook that + # overrides the verdict after a delivered webhook has already captured + # it would leave the non-blocking dispatch with a stale value. + if phase == PHASE_AFTER_SPAM: + steps.sort( + key=lambda s: ( + s.channel.settings.get("trigger") + == enums.WebhookTrigger.MESSAGE_DELIVERING + ), + reverse=True, + ) return steps diff --git a/src/backend/core/mda/inbound_pipeline.py b/src/backend/core/mda/inbound_pipeline.py index ada16575b..e83b668b4 100644 --- a/src/backend/core/mda/inbound_pipeline.py +++ b/src/backend/core/mda/inbound_pipeline.py @@ -39,7 +39,6 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple from django.db.models import Q -from django.db.models.functions import Lower from django.utils import timezone import requests @@ -476,11 +475,10 @@ def _resolve_assignable_users( ) -> List[Dict[str, Any]]: """Resolve OIDC emails → user dicts ready for ``assign_users``. - A single SQL query case-folds both sides (``Lower("email")``) and - fetches all matching users at once — no N+1. Ambiguity (≥2 users - sharing one email) and unknown emails are logged and skipped. - NEVER auto-creates users: a webhook receiver must not be able to - pollute the ``User`` table. + A single SQL query fetches all matching users at once via + ``email__in`` — no N+1. Ambiguity (≥2 users sharing one email) and + unknown emails are logged and skipped. NEVER auto-creates users: a + webhook receiver must not be able to pollute the ``User`` table. The survivors are then filtered to users that currently hold one of the assignable mailbox roles on this thread (editor / sender / @@ -499,9 +497,9 @@ def _resolve_assignable_users( return [] matches = list( - models.User.objects.annotate(_lemail=Lower("email")) - .filter(_lemail__in=target_emails) - .only("id", "email", "full_name") + models.User.objects.filter(email__in=target_emails).only( + "id", "email", "full_name" + ) ) # Group by lowercased email to detect ambiguity per address. diff --git a/src/backend/core/mda/inbound_tasks.py b/src/backend/core/mda/inbound_tasks.py index d4ce93bd5..589cccabe 100644 --- a/src/backend/core/mda/inbound_tasks.py +++ b/src/backend/core/mda/inbound_tasks.py @@ -203,11 +203,10 @@ def process_inbound_message_task(self, inbound_message_id: str): raw_data_bytes = inbound_message.get_raw_bytes() parsed_email = parse_email(raw_data_bytes) if parsed_email is None: - error_msg = "Failed to parse email message" - logger.error(error_msg) - inbound_message.error_message = error_msg - inbound_message.save(update_fields=["error_message"]) - return {"success": False, "error": error_msg} + # A deterministic parse failure never succeeds on retry — + # route through ``_retry_or_abandon`` so it's bounded by the + # quarantine window instead of looping on every 5-min sweep. + return _retry_or_abandon(inbound_message, "Failed to parse email message") mailbox = inbound_message.mailbox recipient_email = str(mailbox) diff --git a/src/backend/core/templates/admin/core/channel/change_form.html b/src/backend/core/templates/admin/core/channel/change_form.html index 3dbfd460a..0321eb4ef 100644 --- a/src/backend/core/templates/admin/core/channel/change_form.html +++ b/src/backend/core/templates/admin/core/channel/change_form.html @@ -12,7 +12,7 @@

    + onsubmit="return confirm('{% blocktranslate %}Regenerate the secret for this channel? The previous secret will stop working immediately.{% endblocktranslate %}');"> {% csrf_token %} @@ -268,9 +268,9 @@ export const WebhookIntegrationForm = ({ -
    +

    {t("General")}

    -
    +

    {t("Endpoint")}

    -
    +

    {t("Behavior")}

    {/* One flat trigger: a lifecycle event whose name says both when it fires and whether it blocks delivery. */} @@ -403,7 +403,7 @@ export const WebhookIntegrationForm = ({
    {isEditing && ( -
    +

    {t("Authentication")}

    {t( @@ -423,7 +423,7 @@ export const WebhookIntegrationForm = ({ {error && {error}} -
    +
    @@ -440,7 +440,7 @@ export const WebhookIntegrationForm = ({
    {!isEditing && ( -
    +

    {t( diff --git a/src/frontend/src/features/layouts/components/thread-view/components/thread-event/index.tsx b/src/frontend/src/features/layouts/components/thread-view/components/thread-event/index.tsx index 2f76b4369..fdb09cab0 100755 --- a/src/frontend/src/features/layouts/components/thread-view/components/thread-event/index.tsx +++ b/src/frontend/src/features/layouts/components/thread-view/components/thread-event/index.tsx @@ -218,10 +218,13 @@ export const ThreadEvent = ({ event, isCondensed = false, onEdit, onDelete, ment }; if (isIMEvent(event)) { - // Prefer ``author_display`` so webhook/channel-authored events (which - // have no ``author``) get a stable color from their displayed name - // instead of falling back to an empty string. - const authorName = event.author_display || event.author?.full_name || event.author?.email || ""; + // Single shared label for both the color hash and the rendered + // header — ``author_display`` (set server-side for webhook/channel + // events) takes priority, then the human author's name/email, then + // "Unknown". Without this, stale payloads that lack + // ``author_display`` but have a real ``author.full_name`` would + // render "Unknown" while the color hash used the real name. + const authorName = event.author_display || event.author?.full_name || event.author?.email || t("Unknown"); const avatarColor = getAvatarColor(authorName); const isMentioned = user ? event.data?.mentions?.map((m) => m.id)?.includes(user.id) @@ -258,8 +261,8 @@ export const ThreadEvent = ({ event, isCondensed = false, onEdit, onDelete, ment {!isCondensed && (

    - - {event.author_display || t("Unknown")} + + {authorName} {t('{{date}} at {{time}}', { From 50fa75a955635fb9d3a829d52838b5bfdfd2e43d Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Mon, 29 Jun 2026 22:08:09 +0200 Subject: [PATCH 14/21] review fix --- src/frontend/public/locales/common/en-US.json | 19 ++- src/frontend/public/locales/common/fr-FR.json | 19 ++- src/frontend/public/locales/common/nl-NL.json | 19 ++- src/frontend/public/locales/common/ru-RU.json | 19 ++- src/frontend/public/locales/common/uk-UA.json | 19 ++- .../modal-compose-integration/_index.scss | 6 + .../webhook-integration-form.tsx | 149 ++++++++---------- 7 files changed, 131 insertions(+), 119 deletions(-) diff --git a/src/frontend/public/locales/common/en-US.json b/src/frontend/public/locales/common/en-US.json index 60a2f6e84..edcb2e1a7 100755 --- a/src/frontend/public/locales/common/en-US.json +++ b/src/frontend/public/locales/common/en-US.json @@ -240,11 +240,10 @@ "bcc": "bcc", "BCC: ": "BCC: ", "Before spam check": "Before spam check", - "Behavior": "Behavior", "Blind copy: ": "Blind copy: ", "Blocking — let this endpoint shape what happens to the message": "Blocking — let this endpoint shape what happens to the message", "Blocking lets the receiver act on this single message": "Blocking lets the receiver act on this single message", - "Body posted to the endpoint. Envelope metadata is always sent as X-StMsg-* headers.": "Body posted to the endpoint. Envelope metadata is always sent as X-StMsg-* headers.", + "What we post in the request body.": "What we post in the request body.", "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.": "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.", "Calendar invite": "Calendar invite", "Calendar service unavailable": "Calendar service unavailable", @@ -359,7 +358,7 @@ "Drag and drop an archive here": "Drag and drop an archive here", "Drop your attachments here": "Drop your attachments here", "Duplicate": "Duplicate", - "Each incoming message will be POSTed to this URL in the format selected below.": "Each incoming message will be POSTed to this URL in the format selected below.", + "Data will be POSTed to this URL in the format selected below.": "Data will be POSTed to this URL in the format selected below.", "Edit": "Edit", "Edit {{mailbox}} address": "Edit {{mailbox}} address", "Edit auto-reply \"{{autoreply}}\"": "Edit auto-reply \"{{autoreply}}\"", @@ -476,8 +475,8 @@ "Integration updated!": "Integration updated!", "Integrations": "Integrations", "Its actual content doesn't match its declared type. Open it only if you trust the sender.": "Its actual content doesn't match its declared type. Open it only if you trust the sender.", - "JMAP Email JSON (RFC 8621)": "JMAP Email JSON (RFC 8621)", - "JMAP Email metadata (notification only)": "JMAP Email metadata (notification only)", + "JMAP Email (full message, RFC 8621)": "JMAP Email (full message, RFC 8621)", + "JMAP Email (metadata only, no body)": "JMAP Email (metadata only, no body)", "just now": "just now", "Label \"{{label}}\" assigned and {{count}} threads archived._one": "Label \"{{label}}\" assigned and thread archived.", "Label \"{{label}}\" assigned and {{count}} threads archived._other": "Label \"{{label}}\" assigned and {{count}} threads archived.", @@ -897,9 +896,15 @@ "You were unassigned": "You were unassigned", "You will no longer have access to the mailbox \"{{mailboxName}}\".": "You will no longer have access to the mailbox \"{{mailboxName}}\".", "Your email...": "Your email...", - "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).": "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).", "Your session has expired. Please log in again.": "Your session has expired. Please log in again.", "URL must start with https://": "URL must start with https://", "Regenerate credential": "Regenerate credential", - "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again." + "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.", + "Trigger": "Trigger", + "Which point in the message's lifecycle fires this webhook, and whether it can influence delivery.": "Which point in the message's lifecycle fires this webhook, and whether it can influence delivery.", + "Message inbound — blocking, before the spam check; can shape the message before it is scanned": "Message inbound — blocking, before the spam check; can shape the message before it is scanned", + "Message delivering — blocking, after the spam check; can shape the message and sees the verdict": "Message delivering — blocking, after the spam check; can shape the message and sees the verdict", + "Message delivered (recommended) — fire after delivery, response ignored": "Message delivered (recommended) — fire after delivery, response ignored", + "Read the webhook documentation for all technical details": "Read the webhook documentation for all technical details", + "Method": "Method" } diff --git a/src/frontend/public/locales/common/fr-FR.json b/src/frontend/public/locales/common/fr-FR.json index bf0a6ec84..99a5b8a2e 100755 --- a/src/frontend/public/locales/common/fr-FR.json +++ b/src/frontend/public/locales/common/fr-FR.json @@ -312,11 +312,10 @@ "bcc": "cci", "BCC: ": "CCI : ", "Before spam check": "Avant le contrôle anti-spam", - "Behavior": "Comportement", "Blind copy: ": "Copie cachée : ", "Blocking — let this endpoint shape what happens to the message": "Bloquant — laissez ce point de terminaison déterminer le sort du message", "Blocking lets the receiver act on this single message": "Le mode bloquant permet au destinataire d'agir sur ce message précis", - "Body posted to the endpoint. Envelope metadata is always sent as X-StMsg-* headers.": "Corps envoyé au point de terminaison. Les métadonnées d'enveloppe sont toujours transmises sous forme d'en-têtes X-StMsg-*.", + "What we post in the request body.": "Ce qui est envoyé dans le corps de la requête.", "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.": "en renvoyant un corps JSON. Actions disponibles (toutes limitées au message en cours de réception) : abandonner / réessayer la distribution, remplacer le verdict anti-spam, ajouter des libellés, assigner des utilisateurs par e-mail, marquer comme suivi / lu / supprimé / archivé, désactiver la réponse automatique, ajouter un commentaire interne au fil de discussion et créer un brouillon de réponse à partir d'un modèle. N'utilisez le mode bloquant qu'avec des destinataires de confiance.", "Calendar invite": "Invitation calendrier", "Calendar service unavailable": "Service d'agenda indisponible", @@ -431,7 +430,7 @@ "Drag and drop an archive here": "Glissez-déposez une archive ici", "Drop your attachments here": "Déposez vos pièces jointes ici", "Duplicate": "Dupliqué", - "Each incoming message will be POSTed to this URL in the format selected below.": "Chaque message entrant sera envoyé via POST à cette URL au format sélectionné ci-dessous.", + "Data will be POSTed to this URL in the format selected below.": "Les données seront envoyées via POST à cette URL au format sélectionné ci-dessous.", "Edit": "Modifier", "Edit {{mailbox}} address": "Modifier l'adresse {{mailbox}}", "Edit auto-reply \"{{autoreply}}\"": "Modifier la réponse automatique \"{{autoreply}}\"", @@ -554,8 +553,8 @@ "Integration updated!": "Intégration mise à jour !", "Integrations": "Intégrations", "Its actual content doesn't match its declared type. Open it only if you trust the sender.": "Son contenu réel ne correspond pas au type déclaré. Ne l'ouvrez que si vous avez confiance en l'expéditeur.", - "JMAP Email JSON (RFC 8621)": "JMAP Email JSON (RFC 8621)", - "JMAP Email metadata (notification only)": "Métadonnées JMAP Email (notification uniquement)", + "JMAP Email (full message, RFC 8621)": "JMAP Email (message complet, RFC 8621)", + "JMAP Email (metadata only, no body)": "JMAP Email (métadonnées uniquement, sans corps)", "just now": "à l'instant", "Label \"{{label}}\" assigned and {{count}} threads archived._one": "Libellé \"{{label}}\" assigné et conversation archivée.", "Label \"{{label}}\" assigned and {{count}} threads archived._many": "Libellé \"{{label}}\" assigné et {{count}} conversations archivées.", @@ -990,9 +989,15 @@ "You were unassigned": "Vous avez été désassigné·e", "You will no longer have access to the mailbox \"{{mailboxName}}\".": "Vous n'aurez plus accès à la boîte aux lettres « {{mailboxName}} ».", "Your email...": "Renseigner votre email...", - "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).": "Votre point de terminaison recevra une charge utile JSON contenant le message analysé (expéditeur, destinataire, objet, corps, en-têtes, …).", "Your session has expired. Please log in again.": "Votre session a expiré. Veuillez vous reconnecter.", "URL must start with https://": "L'URL doit commencer par https://", "Regenerate credential": "Régénérer l'identifiant", - "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "La régénération de l'identifiant invalide immédiatement l'ancien. Le récepteur doit être mis à jour avec la nouvelle valeur avant de pouvoir vérifier à nouveau les webhooks." + "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "La régénération de l'identifiant invalide immédiatement l'ancien. Le récepteur doit être mis à jour avec la nouvelle valeur avant de pouvoir vérifier à nouveau les webhooks.", + "Trigger": "Déclencheur", + "Which point in the message's lifecycle fires this webhook, and whether it can influence delivery.": "À quel moment du cycle de vie du message ce webhook se déclenche, et s'il peut influencer la distribution.", + "Message inbound — blocking, before the spam check; can shape the message before it is scanned": "Message entrant — bloquant, avant le contrôle anti-spam ; peut modifier le message avant son analyse", + "Message delivering — blocking, after the spam check; can shape the message and sees the verdict": "Message en cours de distribution — bloquant, après le contrôle anti-spam ; peut modifier le message et reçoit le verdict anti-spam", + "Message delivered (recommended) — fire after delivery, response ignored": "Message distribué (recommandé) — se déclenche après la distribution, la réponse du point de terminaison est ignorée", + "Read the webhook documentation for all technical details": "Consultez la documentation des webhooks pour tous les détails techniques", + "Method": "Méthode" } diff --git a/src/frontend/public/locales/common/nl-NL.json b/src/frontend/public/locales/common/nl-NL.json index 68630cc58..b66b6b5c6 100644 --- a/src/frontend/public/locales/common/nl-NL.json +++ b/src/frontend/public/locales/common/nl-NL.json @@ -3,20 +3,19 @@ "API key in header — for receivers that can only check a static header value": "API-sleutel in header — voor ontvangers die alleen een statische headerwaarde kunnen controleren", "Authentication": "Authenticatie", "Before spam check": "Voor spamcontrole", - "Behavior": "Gedrag", "Blocking lets the receiver act on this single message": "Blokkerend laat de ontvanger ingrijpen op dit ene bericht", "Blocking — let this endpoint shape what happens to the message": "Blokkerend — laat dit endpoint bepalen wat er met het bericht gebeurt", - "Body posted to the endpoint. Envelope metadata is always sent as X-StMsg-* headers.": "Body die naar het endpoint wordt verzonden. Envelopmetadata wordt altijd verzonden als X-StMsg-* headers.", + "What we post in the request body.": "Wat we in de request-body verzenden.", "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.": "door een JSON-body terug te sturen. Beschikbare acties (allemaal beperkt tot het ontvangen bericht): de bezorging weigeren / opnieuw proberen, het spamoordeel overschrijven, labels toevoegen, gebruikers toewijzen op e-mailadres, markeren als met ster / gelezen / verwijderd / gearchiveerd, het automatisch antwoord onderdrukken, een interne opmerking aan het gesprek toevoegen en een conceptantwoord vanuit een sjabloon maken. Gebruik blokkerend alleen bij ontvangers die je vertrouwt.", "Create a Webhook": "Een Webhook maken", "Done": "Klaar", - "Each incoming message will be POSTed to this URL in the format selected below.": "Elk binnenkomend bericht wordt via POST naar deze URL verzonden in het hieronder geselecteerde formaat.", + "Data will be POSTed to this URL in the format selected below.": "De gegevens worden via POST naar deze URL verzonden in het hieronder geselecteerde formaat.", "Edit Webhook": "Webhook bewerken", "Endpoint": "Endpoint", "Forward every incoming message to a URL of your choice.": "Stuur elk binnenkomend bericht door naar een URL naar keuze.", "How the receiver authenticates our requests. The credential is shown once at creation.": "Hoe de ontvanger onze verzoeken authenticeert. De toegangsgegevens worden eenmalig getoond bij het aanmaken.", - "JMAP Email JSON (RFC 8621)": "JMAP Email JSON (RFC 8621)", - "JMAP Email metadata (notification only)": "JMAP Email-metadata (alleen melding)", + "JMAP Email (full message, RFC 8621)": "JMAP Email (volledig bericht, RFC 8621)", + "JMAP Email (metadata only, no body)": "JMAP Email (alleen metadata, geen body)", "Non-blocking (default) is the safe choice:": "Niet-blokkerend (standaard) is de veilige keuze:", "Outbound Webhook": "Uitgaande webhook", "Payload format": "Payloadformaat", @@ -33,7 +32,6 @@ "Webhook signing secret": "Webhook-ondertekeningsgeheim", "When to fire": "Wanneer activeren", "Whether the webhook fires before or after the message is checked for spam.": "Of de webhook wordt geactiveerd voordat of nadat het bericht op spam is gecontroleerd.", - "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).": "Je endpoint ontvangt een JSON-payload met het geparseerde bericht (from, to, subject, body, headers, …).", "{{count}} attachments_one": "{{count}} bijlage", "{{count}} attachments_other": "{{count}} bijlagen", "{{count}} days ago_one": "{{count}} dag geleden", @@ -567,5 +565,12 @@ "Your session has expired. Please log in again.": "Je sessie is verlopen. Log opnieuw in.", "URL must start with https://": "URL moet beginnen met https://", "Regenerate credential": "Toegangsgegevens opnieuw genereren", - "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Het opnieuw genereren van de toegangsgegevens maakt de oude direct ongeldig. De ontvanger moet met de nieuwe waarde worden bijgewerkt voordat webhooks weer geverifieerd kunnen worden." + "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Het opnieuw genereren van de toegangsgegevens maakt de oude direct ongeldig. De ontvanger moet met de nieuwe waarde worden bijgewerkt voordat webhooks weer geverifieerd kunnen worden.", + "Trigger": "Trigger", + "Which point in the message's lifecycle fires this webhook, and whether it can influence delivery.": "Op welk punt in de levenscyclus van het bericht deze webhook wordt geactiveerd, en of deze de bezorging kan beïnvloeden.", + "Message inbound — blocking, before the spam check; can shape the message before it is scanned": "Bericht inkomend — blokkerend, vóór de spamcontrole; kan het bericht aanpassen voordat het wordt gescand", + "Message delivering — blocking, after the spam check; can shape the message and sees the verdict": "Bericht wordt bezorgd — blokkerend, na de spamcontrole; kan het bericht aanpassen en ziet het oordeel", + "Message delivered (recommended) — fire after delivery, response ignored": "Bericht bezorgd (aanbevolen) — wordt geactiveerd na bezorging, antwoord wordt genegeerd", + "Read the webhook documentation for all technical details": "Lees de webhookdocumentatie voor alle technische details", + "Method": "Methode" } diff --git a/src/frontend/public/locales/common/ru-RU.json b/src/frontend/public/locales/common/ru-RU.json index 2d63c3e76..3061153f9 100644 --- a/src/frontend/public/locales/common/ru-RU.json +++ b/src/frontend/public/locales/common/ru-RU.json @@ -833,20 +833,19 @@ "API key in header — for receivers that can only check a static header value": "API-ключ в заголовке — для получателей, которые могут проверять только статическое значение заголовка", "Authentication": "Аутентификация", "Before spam check": "До проверки на спам", - "Behavior": "Поведение", "Blocking — let this endpoint shape what happens to the message": "Блокирующий — позволить этому эндпоинту влиять на судьбу сообщения", "Blocking lets the receiver act on this single message": "Блокирующий режим позволяет получателю воздействовать на это конкретное сообщение", - "Body posted to the endpoint. Envelope metadata is always sent as X-StMsg-* headers.": "Тело запроса, отправляемое на эндпоинт. Метаданные конверта всегда передаются в заголовках X-StMsg-*.", + "What we post in the request body.": "Что мы отправляем в теле запроса.", "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.": "вернув тело JSON. Доступные действия (все применяются только к получаемому сообщению): отклонить / повторить доставку, переопределить вердикт о спаме, добавить метки, назначить пользователей по email, пометить как избранное / прочитанное / удалённое / архивированное, отключить автоответ, добавить внутренний комментарий к цепочке и создать черновик ответа из шаблона. Используйте блокирующий режим только с доверенными получателями.", "Create a Webhook": "Создать вебхук", "Done": "Готово", - "Each incoming message will be POSTed to this URL in the format selected below.": "Каждое входящее сообщение будет отправлено POST-запросом на этот URL в выбранном ниже формате.", + "Data will be POSTed to this URL in the format selected below.": "Данные будут отправлены POST-запросом на этот URL в выбранном ниже формате.", "Edit Webhook": "Редактировать вебхук", "Endpoint": "Эндпоинт", "Forward every incoming message to a URL of your choice.": "Пересылайте каждое входящее сообщение на выбранный вами URL.", "How the receiver authenticates our requests. The credential is shown once at creation.": "Как получатель аутентифицирует наши запросы. Учётные данные показываются один раз при создании.", - "JMAP Email JSON (RFC 8621)": "JMAP Email JSON (RFC 8621)", - "JMAP Email metadata (notification only)": "Метаданные JMAP Email (только уведомление)", + "JMAP Email (full message, RFC 8621)": "JMAP Email (полное сообщение, RFC 8621)", + "JMAP Email (metadata only, no body)": "JMAP Email (только метаданные, без тела)", "Non-blocking (default) is the safe choice:": "Неблокирующий (по умолчанию) — безопасный выбор:", "Outbound Webhook": "Исходящий вебхук", "Payload format": "Формат полезной нагрузки", @@ -863,8 +862,14 @@ "Webhook signing secret": "Секрет подписи вебхука", "When to fire": "Когда срабатывать", "Whether the webhook fires before or after the message is checked for spam.": "Срабатывает ли вебхук до или после проверки сообщения на спам.", - "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).": "Ваш эндпоинт получит полезную нагрузку JSON с разобранным сообщением (отправитель, получатель, тема, тело, заголовки, …).", "URL must start with https://": "URL должен начинаться с https://", "Regenerate credential": "Сгенерировать новые учётные данные", - "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Генерация новых учётных данных немедленно делает старые недействительными. Получателя необходимо обновить новым значением, прежде чем он снова сможет проверять вебхуки." + "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Генерация новых учётных данных немедленно делает старые недействительными. Получателя необходимо обновить новым значением, прежде чем он снова сможет проверять вебхуки.", + "Trigger": "Триггер", + "Which point in the message's lifecycle fires this webhook, and whether it can influence delivery.": "В какой момент жизненного цикла сообщения срабатывает этот вебхук и может ли он влиять на доставку.", + "Message inbound — blocking, before the spam check; can shape the message before it is scanned": "Входящее сообщение — блокирующий, до проверки на спам; может изменить сообщение до его сканирования", + "Message delivering — blocking, after the spam check; can shape the message and sees the verdict": "Доставка сообщения — блокирующий, после проверки на спам; может изменить сообщение и видит вердикт", + "Message delivered (recommended) — fire after delivery, response ignored": "Сообщение доставлено (рекомендуется) — срабатывает после доставки, ответ игнорируется", + "Read the webhook documentation for all technical details": "Подробные технические сведения см. в документации по вебхукам", + "Method": "Метод" } diff --git a/src/frontend/public/locales/common/uk-UA.json b/src/frontend/public/locales/common/uk-UA.json index 39674c1a2..51ff7a23b 100644 --- a/src/frontend/public/locales/common/uk-UA.json +++ b/src/frontend/public/locales/common/uk-UA.json @@ -833,20 +833,19 @@ "API key in header — for receivers that can only check a static header value": "API-ключ у заголовку — для отримувачів, які можуть перевіряти лише статичне значення заголовка", "Authentication": "Автентифікація", "Before spam check": "До перевірки на спам", - "Behavior": "Поведінка", "Blocking — let this endpoint shape what happens to the message": "Блокувальний — дозволити цьому ендпоінту впливати на долю повідомлення", "Blocking lets the receiver act on this single message": "Блокувальний режим дозволяє отримувачу впливати на це конкретне повідомлення", - "Body posted to the endpoint. Envelope metadata is always sent as X-StMsg-* headers.": "Тіло запиту, що надсилається на ендпоінт. Метадані конверта завжди передаються в заголовках X-StMsg-*.", + "What we post in the request body.": "Що ми надсилаємо в тілі запиту.", "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.": "повернувши тіло JSON. Доступні дії (усі застосовуються лише до отримуваного повідомлення): відхилити / повторити доставку, перевизначити вердикт щодо спаму, додати мітки, призначити користувачів за email, позначити як обране / прочитане / видалене / архівоване, вимкнути автовідповідь, додати внутрішній коментар до ланцюжка та створити чернетку відповіді з шаблону. Використовуйте блокувальний режим лише з довіреними отримувачами.", "Create a Webhook": "Створити вебхук", "Done": "Готово", - "Each incoming message will be POSTed to this URL in the format selected below.": "Кожне вхідне повідомлення буде надіслано POST-запитом на цей URL у вибраному нижче форматі.", + "Data will be POSTed to this URL in the format selected below.": "Дані будуть надіслані POST-запитом на цей URL у вибраному нижче форматі.", "Edit Webhook": "Редагувати вебхук", "Endpoint": "Ендпоінт", "Forward every incoming message to a URL of your choice.": "Пересилайте кожне вхідне повідомлення на вибраний вами URL.", "How the receiver authenticates our requests. The credential is shown once at creation.": "Як отримувач автентифікує наші запити. Облікові дані показуються один раз під час створення.", - "JMAP Email JSON (RFC 8621)": "JMAP Email JSON (RFC 8621)", - "JMAP Email metadata (notification only)": "Метадані JMAP Email (лише сповіщення)", + "JMAP Email (full message, RFC 8621)": "JMAP Email (повне повідомлення, RFC 8621)", + "JMAP Email (metadata only, no body)": "JMAP Email (лише метадані, без тіла)", "Non-blocking (default) is the safe choice:": "Неблокувальний (за замовчуванням) — безпечний вибір:", "Outbound Webhook": "Вихідний вебхук", "Payload format": "Формат корисного навантаження", @@ -863,8 +862,14 @@ "Webhook signing secret": "Секрет підпису вебхука", "When to fire": "Коли спрацьовувати", "Whether the webhook fires before or after the message is checked for spam.": "Чи спрацьовує вебхук до або після перевірки повідомлення на спам.", - "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).": "Ваш ендпоінт отримає корисне навантаження JSON з розібраним повідомленням (відправник, отримувач, тема, тіло, заголовки, …).", "URL must start with https://": "URL має починатися з https://", "Regenerate credential": "Згенерувати нові облікові дані", - "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Генерація нових облікових даних негайно робить старі недійсними. Отримувача потрібно оновити новим значенням, перш ніж він знову зможе перевіряти вебхуки." + "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Генерація нових облікових даних негайно робить старі недійсними. Отримувача потрібно оновити новим значенням, перш ніж він знову зможе перевіряти вебхуки.", + "Trigger": "Тригер", + "Which point in the message's lifecycle fires this webhook, and whether it can influence delivery.": "У який момент життєвого циклу повідомлення спрацьовує цей вебхук і чи може він впливати на доставку.", + "Message inbound — blocking, before the spam check; can shape the message before it is scanned": "Вхідне повідомлення — блокувальний, до перевірки на спам; може змінити повідомлення до його сканування", + "Message delivering — blocking, after the spam check; can shape the message and sees the verdict": "Доставка повідомлення — блокувальний, після перевірки на спам; може змінити повідомлення та бачить вердикт", + "Message delivered (recommended) — fire after delivery, response ignored": "Повідомлення доставлено (рекомендовано) — спрацьовує після доставки, відповідь ігнорується", + "Read the webhook documentation for all technical details": "Докладні технічні відомості див. у документації щодо вебхуків", + "Method": "Метод" } diff --git a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/_index.scss b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/_index.scss index e4a0a1dc5..5c291390d 100644 --- a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/_index.scss +++ b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/_index.scss @@ -220,6 +220,12 @@ border-top: 1px solid var(--c--contextuals--border--semantic--neutral--default); } +// The regenerate button sits alone in a flex-column section, which would +// stretch it full width — keep it sized to its label. +.webhook-integration-form__regenerate { + align-self: flex-start; +} + // Credential label (webhook-only: the one-time secret display) .webhook-integration-form__credential-label { font-size: 0.9rem; diff --git a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx index 04718e817..e26c888d1 100644 --- a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx +++ b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx @@ -1,5 +1,5 @@ import { Button } from "@gouvfr-lasuite/cunningham-react"; -import { Icon, IconType } from "@gouvfr-lasuite/ui-kit"; +import { Icon, IconSize, IconType } from "@gouvfr-lasuite/ui-kit"; import { useTranslation } from "react-i18next"; import { useForm, FormProvider } from "react-hook-form"; import { zodResolver } from "@hookform/resolvers/zod"; @@ -294,38 +294,12 @@ export const WebhookIntegrationForm = ({ text={ errors.url?.message || t( - "Each incoming message will be POSTed to this URL in the format selected below.", + "Data will be POSTed to this URL in the format selected below.", ) } state={errors.url ? "error" : "default"} fullWidth /> - -
    - -
    -

    {t("Behavior")}

    {/* One flat trigger: a lifecycle event whose name says both when it fires and whether it blocks delivery. */} +
    + +
    +

    {t("Authentication")}

    + - -

    - + {isEditing && ( + <> + {t( - "Message delivered (default) is the safe choice:", + "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.", )} - {" "} - {t( - "we POST and ignore the response. Receivers cannot affect the message.", - )} -

    -

    - - {t( - "Blocking triggers let the receiver act on this single message", - )} - {" "} - {t( - "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.", - )} -

    -     + + + + )}
    - {isEditing && ( -
    -

    {t("Authentication")}

    - - {t( - "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.", - )} - - -
    - )} +

    + + {t( + "Read the webhook documentation for all technical details", + )} + + +

    {error && {error}} @@ -438,17 +430,6 @@ export const WebhookIntegrationForm = ({ : t("Create integration")}
    - - {!isEditing && ( -
    - -

    - {t( - "Your endpoint will receive a JSON payload containing the parsed message (from, to, subject, body, headers, …).", - )} -

    -
    - )} ); From 7ce2fbaa6f4cc7df7f50d9f6dac1b2d4defaac90 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Mon, 29 Jun 2026 23:28:15 +0200 Subject: [PATCH 15/21] fix lint --- src/backend/core/mda/dispatch_webhooks.py | 2 +- src/backend/core/tests/api/test_messages_delivery_statuses.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/core/mda/dispatch_webhooks.py b/src/backend/core/mda/dispatch_webhooks.py index fade9f5e5..20ec0c9f7 100644 --- a/src/backend/core/mda/dispatch_webhooks.py +++ b/src/backend/core/mda/dispatch_webhooks.py @@ -29,7 +29,7 @@ same envelope metadata in ``X-StMsg-*`` headers. """ -# pylint: disable=broad-exception-caught +# pylint: disable=broad-exception-caught,too-many-lines from __future__ import annotations diff --git a/src/backend/core/tests/api/test_messages_delivery_statuses.py b/src/backend/core/tests/api/test_messages_delivery_statuses.py index 188d0cf9b..7aa506b8b 100644 --- a/src/backend/core/tests/api/test_messages_delivery_statuses.py +++ b/src/backend/core/tests/api/test_messages_delivery_statuses.py @@ -1,5 +1,7 @@ """Test messages delivery statuses endpoint.""" +# pylint: disable=too-many-lines + from datetime import timedelta from unittest.mock import patch From 9bd69601b0f4be2e5ffa849b60a3041ac35255d7 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Tue, 30 Jun 2026 01:09:36 +0200 Subject: [PATCH 16/21] document more vars --- docs/env.md | 106 +++++++++++++++++- docs/webhooks.md | 12 +- env.d/development/backend.defaults | 1 - src/backend/core/mda/dispatch_webhooks.py | 7 ++ .../core/tests/mda/test_dispatch_webhooks.py | 32 ++++++ src/backend/messages/settings.py | 16 ++- 6 files changed, 167 insertions(+), 7 deletions(-) diff --git a/docs/env.md b/docs/env.md index 38a7b7457..c0a255af9 100644 --- a/docs/env.md +++ b/docs/env.md @@ -35,6 +35,9 @@ The application uses a new environment file structure with `.defaults` and `.loc | `DJANGO_SUPERUSER_PASSWORD` | `admin` | Default superuser password for development | Dev | | `DJANGO_DATA_DIR` | `/data` | Base directory for data storage | Optional | | `DJANGO_ADMIN_URL` | `admin` | admin route (must not be ended by `/`) | Optional | +| `INSTANCE_URL` | None | Public base URL of this instance — the scheme+host that serves both the API and the web app (e.g. `https://messages-public-url.example.com`). Used to build absolute links back into the product; currently emitted as the `X-StMsg-Instance` header on outbound webhooks (omitted when unset). | Optional | +| `USE_X_FORWARDED_FOR` | `False` | Trust the `X-Forwarded-For` header to determine the client IP (enable only behind a trusted proxy that sets it). | Optional | +| `DATA_UPLOAD_MAX_MEMORY_SIZE` | `2621440` | Django's max request body (bytes) buffered in memory before rejecting (2.5MB). | Optional | ### Database Configuration @@ -71,6 +74,7 @@ The application uses a new environment file structure with `.defaults` and `.loc | Variable | Default | Description | Required | |----------|---------|-------------|----------| | `OPENSEARCH_URL` | `["http://opensearch:9200"]` | OpenSearch hosts list | Optional | +| `OPENSEARCH_CA_CERTS` | None | Path to a CA bundle for verifying the OpenSearch TLS certificate (for `https://` hosts with a private CA). | Optional | | `OPENSEARCH_TIMEOUT` | `20` | OpenSearch query timeout (seconds) for unitary requests | Optional | | `OPENSEARCH_BULK_TIMEOUT` | `60` | OpenSearch request timeout (seconds) applied to bulk indexation calls. Raise it if full reindex (`make search-index`) hits timeouts on large payloads. | Optional | | `OPENSEARCH_BULK_MAX_BYTES` | `26_214_400` | Flush threshold (bytes) for bulk indexation payloads; default 25 MiB. Once accumulated actions exceed this, `opensearch-py` emits a sub-chunk HTTP request. Note: this is a batching threshold, not a per-document cap — a single oversized document is still sent as its own chunk. Keep well under the OpenSearch server `http.max_content_length` | Optional | @@ -110,9 +114,12 @@ The application uses a new environment file structure with `.defaults` and `.loc | Variable | Default | Description | Required | |----------|---------|-------------|----------| | `MESSAGES_DKIM_SELECTOR` | `default` | DKIM selector | Optional | +| `MESSAGES_DKIM_DEFAULT_SELECTOR` | `stmessages` | Default DKIM selector applied to managed domains that don't override it. | Optional | | `MESSAGES_DKIM_DOMAINS` | `[]` | List of domains for DKIM signing | Optional | | `MESSAGES_DKIM_PRIVATE_KEY_B64` | None | Base64 encoded DKIM private key | Optional | | `MESSAGES_DKIM_PRIVATE_KEY_FILE` | None | Path to DKIM private key file | Optional | +| `MESSAGES_DKIM_VERIFY_OUTGOING` | `False` | Verify the DKIM signature on outgoing messages before sending. | Optional | +| `MESSAGES_SPF_CHECK_OUTGOING` | `False` | Block outgoing messages when the sending domain's SPF includes are not correctly set up. | Optional | ## Storage Configuration @@ -123,10 +130,11 @@ The application uses a new environment file structure with `.defaults` and `.loc | `AWS_S3_ENDPOINT_URL` | `http://objectstorage:9000` | S3 endpoint URL | Optional | | `AWS_S3_ACCESS_KEY_ID` | `messages` | S3 access key | Optional | | `AWS_S3_SECRET_ACCESS_KEY` | `password` | S3 secret key | Optional | +| `AWS_S3_SIGNATURE_VERSION` | `s3v4` | S3 request signature version | Optional | +| `AWS_S3_DOMAIN_REPLACE` | None | If set, rewrites the host of generated S3 URLs to this value — e.g. map an internal endpoint to a public one for presigned/download links. | Optional | | `AWS_S3_REGION_NAME` | None | S3 region | Optional | | `AWS_STORAGE_BUCKET_NAME` | `st-messages-media-storage` | S3 bucket name | Optional | | `AWS_S3_UPLOAD_POLICY_EXPIRATION` | `86400` | Upload policy expiration (24h) | Optional | -| `MEDIA_BASE_URL` | `http://localhost:8902` | Base URL for media files | Optional | | `ITEM_FILE_MAX_SIZE` | `5368709120` | Max file size (5GB) | Optional | ### Message Imports Storage @@ -236,7 +244,9 @@ _Those settings are deprecated and will be removed in the future._ | `CORS_ALLOWED_ORIGINS` | `[]` | Specific allowed CORS origins | Optional | | `CORS_ALLOWED_ORIGIN_REGEXES` | `[]` | Regex patterns for allowed origins | Optional | | `CSRF_TRUSTED_ORIGINS` | `["http://localhost:8900", "http://localhost:8901"]` | Trusted origins for CSRF | Optional | +| `ALLOWED_HOSTS` | `[]` | Host/domain allow-list (Production configuration; the Development class sets this from `DJANGO_ALLOWED_HOSTS`). | Optional | | `SERVER_TO_SERVER_API_TOKENS` | `[]` | API tokens for server-to-server auth | Optional | +| `SALT_KEY` | `[]` | Key(s) for Django Fernet-encrypted model fields. Accepts a list for rotation (`["new_key", "old_key"]`); the first is used to encrypt, all are tried to decrypt. | Optional | ## Monitoring & Observability @@ -325,6 +335,7 @@ without redeploying the frontend (the flag is pulled from | `FEATURE_MAILDOMAIN_MANAGE_ACCESSES` | `True` | Allows managing mail domain accesses (create/delete). When `False`, those actions return 403. | Optional | | `FEATURE_MESSAGE_TEMPLATES` | `True` | Enables the "message templates" feature. When `False`, mailbox admins lose the `CAN_MANAGE_MESSAGE_TEMPLATES` ability and the related UI is hidden. | Optional | | `FEATURE_THREAD_SPLIT` | `True` | Enables "split thread" feature. When `False`, the split API action returns 404 and the frontend hides the menu entry. | Optional | +| `FEATURE_MAILDOMAIN_MANAGE_TOTP` | `False` | Enables the "Mandatory 2FA" (TOTP) toggle for mail domains. Requires the Keycloak identity-provider settings and `KEYCLOAK_TOTP_ROLE_ID`. | Optional | ### Business Logic @@ -339,6 +350,8 @@ without redeploying the frontend (the flag is pulled from | `MAX_TEMPLATE_IMAGE_SIZE` | `2097152` | Maximum size in bytes for images embedded in templates and signatures (2MB) | Optional | | `MAX_RECIPIENTS_PER_MESSAGE` | `500` | Maximum number of recipients per message (to + cc + bcc) | Optional | | `MAX_THREAD_EVENT_EDIT_DELAY` | `3600` | Time window in seconds during which a ThreadEvent (internal comment) can still be edited or deleted after creation. Set to `0` to disable the restriction. | Optional | +| `MESSAGES_ALLOW_INTERNAL_DELIVERY` | `True` | Deliver mailbox-to-mailbox mail through the internal inbound pipeline (fast path). Set `False` to force same-instance mail out through the external MTA so it passes the same scanning/archiving as outbound. | Optional | +| `MESSAGES_MAILBOX_LOCALPART_DENYLIST_PERSONAL` | `[]` | Local parts rejected for personal mailboxes (case-insensitive exact match). | Optional | ### Model custom attributes schema @@ -376,6 +389,11 @@ Outbound message throttling limits the number of **external recipients** (recipi | `THROTTLE_MAILBOX_OUTBOUND_EXTERNAL_RECIPIENTS` | None | Rate limit per mailbox. Format: `count/period` where period is `minute`, `hour`, or `day`. Example: `1000/day` limits each mailbox to 1000 external recipients per day. | Optional | | `THROTTLE_MAILDOMAIN_OUTBOUND_EXTERNAL_RECIPIENTS` | None | Rate limit per maildomain. Format: `count/period`. Example: `10000/day` limits each domain to 10000 external recipients per day. | Optional | | `THROTTLE_AUTOREPLY_PER_SENDER` | `1/day` | Rate limit for autoreplies per sender per mailbox. Format: `count/period`. Example: `1/day` limits each sender to 1 autoreply per day per mailbox. | Optional | +| `API_USERS_LIST_THROTTLE_RATE_SUSTAINED` | `180/hour` | Sustained rate limit on the users-list API (per user). | Optional | +| `API_USERS_LIST_THROTTLE_RATE_BURST` | `30/minute` | Burst rate limit on the users-list API (per user). | Optional | +| `API_CALDAV_CONFLICTS_THROTTLE_RATE` | `30/minute` | Rate limit on the CalDAV conflict-check API. | Optional | +| `API_WIDGET_INBOUND_CHANNEL_THROTTLE_RATE` | `30/minute` | Rate limit on inbound widget submissions, per widget channel. | Optional | +| `API_WIDGET_INBOUND_IP_THROTTLE_RATE` | `10/minute` | Per-IP burst limit on inbound widget submissions. | Optional | ### Image Proxy @@ -404,6 +422,92 @@ it can lead to memory exhaustion, increase at your own risk. | `DRIVE_BASE_URL` | None | Base URL to access Drive endpoints | Optional | | `DRIVE_APP_NAME` | `Drive` | Name of the Drive application used in the frontend | Optional | +### Identity Provider (Keycloak) + +Used for provisioning-side operations against Keycloak (e.g. toggling +mandatory 2FA on a mail domain). Distinct from the OIDC login settings +above, which handle end-user authentication. + +| Variable | Default | Description | Required | +|----------|---------|-------------|----------| +| `IDENTITY_PROVIDER` | None | Identity-provider integration to enable (e.g. `keycloak`). Unset disables provisioning-side IdP calls. | Optional | +| `KEYCLOAK_URL` | None | Base URL of the Keycloak server. | Optional | +| `KEYCLOAK_REALM` | None | Keycloak realm. | Optional | +| `KEYCLOAK_CLIENT_ID` | None | Service-account client id used for admin/provisioning calls. | Optional | +| `KEYCLOAK_CLIENT_SECRET` | None | Service-account client secret. | Optional | +| `KEYCLOAK_GROUP_PATH_PREFIX` | None | Prefix for Keycloak group paths mapped to mail domains. | Optional | +| `KEYCLOAK_TOTP_ROLE_ID` | None | Realm role id assigned in Keycloak when "Mandatory 2FA" is enabled for a mailbox (see `FEATURE_MAILDOMAIN_MANAGE_TOTP`). | Optional | + +### Domain DNS Provisioning + +Hosting of MX/SPF/DKIM records for managed mail domains, optionally +automated through a DNS provider. + +| Variable | Default | Description | Required | +|----------|---------|-------------|----------| +| `MESSAGES_TECHNICAL_DOMAIN` | `localhost` | Technical domain that MX/SPF/DKIM records point at. | Optional | +| `MESSAGES_DNS_RECORDS` | MX/SPF/DKIM template | JSON template of expected DNS records, with `{technical_domain}` placeholders. | Optional | +| `DNS_DEFAULT_PROVIDER` | None | DNS provider used to auto-create records (e.g. `scaleway`). Unset = manual DNS. | Optional | +| `DNS_SCALEWAY_API_TOKEN` | None | Scaleway API token (when `DNS_DEFAULT_PROVIDER=scaleway`). | Optional | +| `DNS_SCALEWAY_PROJECT_ID` | None | Scaleway project id. | Optional | +| `DNS_SCALEWAY_TTL` | `3600` | TTL (seconds) for records created via Scaleway. | Optional | + +### Calendar (CalDAV) + +| Variable | Default | Description | Required | +|----------|---------|-------------|----------| +| `CALDAV_DEFAULT_URL` | None | Base URL of the external CalDAV server. | Optional | +| `CALDAV_DEFAULT_WEB_URL` | None | URL of the calendar web UI surfaced to the frontend. | Optional | +| `CALDAV_DEFAULT_PASSWORD` | None | Credential for the default CalDAV account. | Optional | + +### Entitlements + +Pluggable backend deciding what a user/domain is entitled to. + +| Variable | Default | Description | Required | +|----------|---------|-------------|----------| +| `ENTITLEMENTS_BACKEND` | `core.entitlements.backends.local.LocalEntitlementsBackend` | Dotted path to the entitlements backend class. | Optional | +| `ENTITLEMENTS_BACKEND_PARAMETERS` | `{}` | JSON parameters passed to the backend. | Optional | +| `ENTITLEMENTS_CACHE_TIMEOUT` | `300` | Cache TTL (seconds) for entitlement lookups. | Optional | + +### Message Import (IMAP) + +Tuning for the IMAP-based message importer (see also `FEATURE_IMPORT_MESSAGES`). + +| Variable | Default | Description | Required | +|----------|---------|-------------|----------| +| `IMAP_TIMEOUT` | `60` | Socket timeout (seconds) for IMAP connections during import. | Optional | +| `IMAP_MAX_RETRIES` | `3` | Retry budget for transient IMAP failures during import. | Optional | + +### Spam Filtering + +| Variable | Default | Description | Required | +|----------|---------|-------------|----------| +| `SPAM_CONFIG` | `{}` | JSON config for the spam checker. Empty `{}` disables it. Example: `{"rspamd_url": "http://mpa:8010/_api", "rspamd_auth": ""}`. | Optional | + +### Celery / Task Queue + +| Variable | Default | Description | Required | +|----------|---------|-------------|----------| +| `DISABLE_CELERY_BEAT_SCHEDULE` | `False` | Disable the periodic Beat schedule (search indexing, offload, selfcheck, …). | Optional | +| `CELERY_TASK_SEND_SENT_EVENT` | `True` | Emit Celery `task-sent` events (monitoring/Flower). | Optional | +| `CELERY_WORKER_SEND_TASK_EVENTS` | `True` | Workers emit task events (monitoring/Flower). | Optional | + +### Metrics + +| Variable | Default | Description | Required | +|----------|---------|-------------|----------| +| `METRICS_STORAGE_USED_OVERHEAD_BY_MESSAGE` | `1024` | Per-message overhead (bytes) added when computing reported storage usage. | Optional | + +### Deprecated + +_Set only to receive a startup deprecation warning; otherwise ignored._ + +| Variable | Default | Description | Required | ⚠️ Deprecated | +|----------|---------|-------------|----------|----------| +| `PROVISIONING_API_KEY` | None | Ignored since global `api_key` Channels landed. Migrate to a global `api_key` Channel. | Optional | Removed in a future release | +| `METRICS_API_KEY` | None | Ignored since global `api_key` Channels landed. Migrate to a global `api_key` Channel. | Optional | Removed in a future release | + ## Legend - **Required**: Must be set for the application to function diff --git a/docs/webhooks.md b/docs/webhooks.md index 671a30880..c5fea330d 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -146,8 +146,10 @@ message lands. | --------------------- | ---------------------------------------------------------------- | | `Content-Type` | `message/rfc822` for `eml`, `application/json` for both JMAP variants | | `X-StMsg-Trigger` | The lifecycle event that fired (`message.inbound` / `message.delivering` / `message.delivered`). Route on this — it says what happened and, implicitly, whether the webhook blocked. | +| `X-StMsg-Instance` | Public base URL of the originating instance (e.g. `https://messages-public-url.example.com`). **Only present when the instance sets `INSTANCE_URL`.** Combine with the `*-Id` headers to build callback API URLs. | | `X-StMsg-Channel-Id` | UUID of the firing webhook Channel | | `X-StMsg-Mailbox` | Destination mailbox address | +| `X-StMsg-Mailbox-Id` | UUID of the destination `Mailbox` (the API is keyed by this, not the address) | | `X-StMsg-Recipient` | Envelope `RCPT TO` (usually the same as `X-StMsg-Mailbox`) | | `X-StMsg-Is-Spam` | `true`, `false`, or `pending` (`pending` for `message.inbound`, which fires before the spam check) | | `X-StMsg-Message-Id` | UUID of the stored `Message` — **non-blocking only** (see note) | @@ -157,8 +159,12 @@ The MIME message-id is **not** sent as a header — every body format already carries it (`messageId` in the JMAP variants, the raw `Message-ID:` header in `eml`). -`X-StMsg-Message-Id` / `X-StMsg-Thread-Id` are the platform's own ids -(for calling back into the API). They're only present on **non-blocking** +`X-StMsg-Mailbox-Id`, `X-StMsg-Message-Id` and `X-StMsg-Thread-Id` are the +platform's own ids (the API is keyed by UUID, not by address). To call back +into the API, join them to the instance base URL — sent as `X-StMsg-Instance` +when configured — e.g. +`GET {X-StMsg-Instance}/api/v1.0/mailboxes/{X-StMsg-Mailbox-Id}/…`. +`X-StMsg-Message-Id` / `X-StMsg-Thread-Id` are only present on **non-blocking** webhooks, which fire after the `Message` is persisted; blocking webhooks run before it exists, so they can't carry them. @@ -381,8 +387,10 @@ MTA received them. POST /inbox-hook HTTP/1.1 Content-Type: message/rfc822 X-StMsg-Trigger: message.delivered +X-StMsg-Instance: https://messages-public-url.example.com X-StMsg-Channel-Id: 05f1f991-c2e9-4fa7-8a78-98c3aa904c7c X-StMsg-Mailbox: alice@example.com +X-StMsg-Mailbox-Id: 3c2e0b1a-9d4f-4e8c-bf2a-1a2b3c4d5e6f X-StMsg-Recipient: alice@example.com X-StMsg-Is-Spam: false diff --git a/env.d/development/backend.defaults b/env.d/development/backend.defaults index c43633b79..c20243fb9 100644 --- a/env.d/development/backend.defaults +++ b/env.d/development/backend.defaults @@ -39,7 +39,6 @@ AWS_S3_ACCESS_KEY_ID=messages AWS_S3_SECRET_ACCESS_KEY=password AWS_S3_SIGNATURE_VERSION=s3v4 AWS_S3_DOMAIN_REPLACE=http://localhost:8906 -MEDIA_BASE_URL=http://localhost:8902 # Message imports storage STORAGE_MESSAGE_IMPORTS_ENDPOINT_URL=http://objectstorage:9000 diff --git a/src/backend/core/mda/dispatch_webhooks.py b/src/backend/core/mda/dispatch_webhooks.py index 20ec0c9f7..ba7c5da78 100644 --- a/src/backend/core/mda/dispatch_webhooks.py +++ b/src/backend/core/mda/dispatch_webhooks.py @@ -44,6 +44,7 @@ from typing import Any, Dict, List, Optional, Set, Tuple from urllib.parse import urlparse +from django.conf import settings from django.db.models import Q import jwt @@ -494,9 +495,15 @@ def _envelope_headers( "X-StMsg-Trigger": (channel.settings or {}).get("trigger", ""), "X-StMsg-Channel-Id": str(channel.id), "X-StMsg-Mailbox": str(mailbox), + "X-StMsg-Mailbox-Id": str(mailbox.id), "X-StMsg-Recipient": recipient_email, "X-StMsg-Is-Spam": spam_value, } + # The instance's own public URL, so a receiver (especially a shared one + # serving several instances) knows who fired and can turn the ``*-Id`` + # headers into callback API URLs. Optional — omitted when unconfigured. + if settings.INSTANCE_URL: + headers["X-StMsg-Instance"] = settings.INSTANCE_URL if message is not None: headers["X-StMsg-Message-Id"] = str(message.id) headers["X-StMsg-Thread-Id"] = str(message.thread_id) diff --git a/src/backend/core/tests/mda/test_dispatch_webhooks.py b/src/backend/core/tests/mda/test_dispatch_webhooks.py index bc09e93df..0cdf93813 100644 --- a/src/backend/core/tests/mda/test_dispatch_webhooks.py +++ b/src/backend/core/tests/mda/test_dispatch_webhooks.py @@ -13,6 +13,7 @@ from typing import Optional, Set from unittest.mock import Mock, patch +from django.test import override_settings from django.utils import timezone as dj_timezone import pytest @@ -808,11 +809,42 @@ def test_envelope_headers_set_for_both_formats( headers = mock_session.return_value.post.call_args.kwargs["headers"] assert headers["X-StMsg-Trigger"] == "message.delivering" assert headers["X-StMsg-Mailbox"] == str(mailbox) + assert headers["X-StMsg-Mailbox-Id"] == str(mailbox.id) assert headers["X-StMsg-Recipient"] == str(mailbox) assert headers["X-StMsg-Is-Spam"] == "true" # The message-id is NOT a header — every body format already # carries it (messageId in jmap, raw Message-ID: in eml). assert "X-StMsg-Message-Mime-Id" not in headers + # The instance URL header is opt-in — absent unless configured. + assert "X-StMsg-Instance" not in headers + + @override_settings(INSTANCE_URL="https://messages-public-url.example.com") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_instance_header_set_when_configured( + self, mock_session, mailbox, parsed_email + ): + # message.delivering is the blocking, after-spam trigger that fires + # in PHASE_AFTER_SPAM (message.delivered fires post-persist, not here). + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "trigger": "message.delivering", + "auth_method": "jwt", + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + headers = mock_session.return_value.post.call_args.kwargs["headers"] + assert headers["X-StMsg-Instance"] == "https://messages-public-url.example.com" @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_is_spam_header_unknown_when_none( diff --git a/src/backend/messages/settings.py b/src/backend/messages/settings.py index 228278e9a..ebe363dbc 100644 --- a/src/backend/messages/settings.py +++ b/src/backend/messages/settings.py @@ -288,9 +288,6 @@ class Base(Configuration): STATIC_ROOT = os.path.join(DATA_DIR, "static") MEDIA_URL = "/media/" MEDIA_ROOT = os.path.join(DATA_DIR, "media") - MEDIA_BASE_URL = values.Value( - None, environ_name="MEDIA_BASE_URL", environ_prefix=None - ) SITE_ID = 1 @@ -535,6 +532,19 @@ class Base(Configuration): environ_prefix=None, ) + # Public base URL of this Messages instance, i.e. the origin that serves + # both the API and the web app (e.g. ``https://messages-public-url.example.com``). + # Instance-wide, not feature-specific: anything that has to emit an + # absolute link back to ourselves should read this. Today the only + # consumer is outbound webhooks, which send it as the ``X-StMsg-Instance`` + # header so a receiver can build callback API URLs from the ``*-Id`` + # headers; the header is omitted when this is unset. + INSTANCE_URL = values.Value( + None, + environ_name="INSTANCE_URL", + environ_prefix=None, + ) + # Manual retry settings MESSAGES_MANUAL_RETRY_MAX_AGE = values.PositiveIntegerValue( 7 * 24 * 60 * 60, # 7 days in seconds From e54278895a4e5bf1b02cab8e67a659fa135dcbe3 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Tue, 30 Jun 2026 20:52:13 +0200 Subject: [PATCH 17/21] harden implem --- docs/webhooks.md | 2 +- src/backend/core/admin.py | 43 +- src/backend/core/mda/dispatch_webhooks.py | 199 +++++- src/backend/core/mda/inbound_create.py | 16 + src/backend/core/mda/inbound_pipeline.py | 12 + src/backend/core/mda/inbound_tasks.py | 221 +++++- src/backend/core/mda/outbound.py | 22 + ...lob_inboundmessage_is_internal_and_more.py | 5 + src/backend/core/models.py | 18 + src/backend/core/tests/api/test_channels.py | 49 ++ .../core/tests/mda/test_dispatch_webhooks.py | 666 +++++++++++++++++- src/backend/core/tests/services/test_ssrf.py | 116 +++ src/backend/messages/celery_app.py | 8 + src/frontend/public/locales/common/en-US.json | 9 - src/frontend/public/locales/common/fr-FR.json | 9 - src/frontend/public/locales/common/nl-NL.json | 9 - src/frontend/public/locales/common/ru-RU.json | 9 - src/frontend/public/locales/common/uk-UA.json | 9 - 18 files changed, 1318 insertions(+), 104 deletions(-) diff --git a/docs/webhooks.md b/docs/webhooks.md index c5fea330d..b8483f418 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -258,7 +258,7 @@ optional; unknown keys are ignored. | Key | Type | Meaning | | ---------------- | -------------- | ---------------------------------------------------------------------------------------------------------------- | -| `action` | `"drop"` / `"retry"` | `"drop"` drops the message at this phase; `"retry"` re-queues the inbound task (bounded by the 48h quarantine window). Any other value (or omission) is treated as accept. Case-insensitive. | +| `action` | `"drop"` | `"drop"` drops the message at this phase. Any other value (or omission) is treated as accept. Case-insensitive. There is no body-driven `"retry"`: a 2xx is a successful response. If you need the message redelivered later, return a non-2xx status (e.g. `429`/`503`) — it is held for retry, bounded by the 48h quarantine window. | | `is_spam` | bool | Override the spam verdict. Acts as a full antispam: for a `message.inbound` webhook this **skips rspamd**. | | `add_labels` | string[] | UUIDs of `Label` rows in the destination mailbox to attach to the thread once it is created. | | `assign_to` | string[] | OIDC emails of users to assign to the resulting thread (one `ThreadEvent ASSIGN` per webhook, channel-attributed). | diff --git a/src/backend/core/admin.py b/src/backend/core/admin.py index becd8303b..b802dc3df 100644 --- a/src/backend/core/admin.py +++ b/src/backend/core/admin.py @@ -1552,17 +1552,26 @@ class InboundMessageAdmin(admin.ModelAdmin): "mailbox", "channel", "has_error", + "is_abandoned", "created_at", ) - list_filter = ("created_at",) + list_filter = ("created_at", "abandoned_at") search_fields = ( "mailbox__local_part", "mailbox__domain__name", "error_message", ) autocomplete_fields = ("mailbox", "channel") - readonly_fields = ("created_at", "updated_at") - fields = ("mailbox", "channel", "error_message", "created_at", "updated_at") + readonly_fields = ("created_at", "updated_at", "abandoned_at") + fields = ( + "mailbox", + "channel", + "error_message", + "abandoned_at", + "created_at", + "updated_at", + ) + actions = ("replay_abandoned",) def has_error(self, obj): """Return whether the message has an error.""" @@ -1571,6 +1580,34 @@ def has_error(self, obj): has_error.boolean = True has_error.short_description = "Error" + def is_abandoned(self, obj): + """Return whether processing was permanently abandoned.""" + return obj.abandoned_at is not None + + is_abandoned.boolean = True + is_abandoned.short_description = "Abandoned" + + @admin.action(description="Replay abandoned messages (clear + re-queue)") + def replay_abandoned(self, request, queryset): + """Re-queue abandoned inbound messages for processing. + + Clears the terminal ``abandoned_at`` marker (and the stale + ``error_message``) and re-dispatches the per-message task, so an + operator can retry a poison message after fixing whatever caused it + to fail. No-op for rows that aren't abandoned. + """ + # pylint: disable-next=import-outside-toplevel + from core.mda.inbound_tasks import process_inbound_message_task + + replayed = 0 + for inbound_message in queryset.filter(abandoned_at__isnull=False): + inbound_message.abandoned_at = None + inbound_message.error_message = "" + inbound_message.save(update_fields=["abandoned_at", "error_message"]) + process_inbound_message_task.delay(str(inbound_message.id)) + replayed += 1 + self.message_user(request, f"Re-queued {replayed} abandoned message(s).") + def get_queryset(self, request): """Optimize queryset with select_related for better performance.""" return ( diff --git a/src/backend/core/mda/dispatch_webhooks.py b/src/backend/core/mda/dispatch_webhooks.py index ba7c5da78..eee1b2025 100644 --- a/src/backend/core/mda/dispatch_webhooks.py +++ b/src/backend/core/mda/dispatch_webhooks.py @@ -7,8 +7,9 @@ (``scope_level=GLOBAL``). A single ``settings.trigger`` lifecycle event (see ``enums.WebhookTrigger`` and ``docs/webhooks.md``) describes a webhook: ``message.inbound`` (before spam) and ``message.delivering`` (after spam) -run inline and can abort delivery (DROP), request retry (RETRY), override -the spam verdict, or attach labels via their JSON response body; +run inline and can abort delivery (DROP), override the spam verdict, or +attach labels via their JSON response body (a transient failure / non-2xx +status holds the message for RETRY, but that is not a body-driven action); ``message.delivered`` is fire-and-forget after the message is created. This file is webhook-specific: HTTP plumbing, signing (HMAC + JWT or @@ -45,13 +46,19 @@ from urllib.parse import urlparse from django.conf import settings +from django.core.cache import cache from django.db.models import Q import jwt from jmap_email import JmapEmail, parse_email from core import enums, models -from core.mda.inbound_pipeline import Decision, InboundContext, Step +from core.mda.inbound_pipeline import ( + QUARANTINE_AFTER, + Decision, + InboundContext, + Step, +) from core.mda.webhook_payload import build_jmap_email from core.services.ssrf import SSRFSafeSession, SSRFValidationError @@ -121,6 +128,131 @@ class _HttpResult: # pylint: disable=too-many-instance-attributes # scope-checked + drafted in the pipeline finalize step. reply_draft_template_id: Optional[str] = None + def to_cache(self) -> Dict[str, Any]: + """Plain JSON-able dict for the cross-retry result cache. + + Explicit (not pickle) so a deploy that changes this dataclass mid- + flight can't fail to load 48h-old entries — ``from_cache`` simply + ignores unknown/missing keys and the webhook re-fires. ``decision`` + is stored as its int value; ``labels`` as a sorted list. + """ + return { + "decision": int(self.decision), + "is_spam_override": self.is_spam_override, + "labels": sorted(self.labels), + "assign_to": list(self.assign_to), + "mark_starred": self.mark_starred, + "mark_read": self.mark_read, + "mark_trashed": self.mark_trashed, + "mark_archived": self.mark_archived, + "skip_autoreply": self.skip_autoreply, + "events": list(self.events), + "reply_draft_template_id": self.reply_draft_template_id, + } + + @classmethod + def from_cache(cls, data: Dict[str, Any]) -> "_HttpResult": + """Rebuild from ``to_cache`` output. Tolerant of partial/old data.""" + return cls( + decision=Decision(data.get("decision", int(Decision.CONTINUE))), + is_spam_override=data.get("is_spam_override"), + labels=set(data.get("labels") or []), + assign_to=list(data.get("assign_to") or []), + mark_starred=bool(data.get("mark_starred")), + mark_read=bool(data.get("mark_read")), + mark_trashed=bool(data.get("mark_trashed")), + mark_archived=bool(data.get("mark_archived")), + skip_autoreply=bool(data.get("skip_autoreply")), + events=list(data.get("events") or []), + reply_draft_template_id=data.get("reply_draft_template_id"), + ) + + +# --- cross-retry result cache --- # +# +# A blocking webhook runs *before* the spam step, so a step that later RETRYs +# (rspamd outage, a different blocking webhook failing) re-runs the whole +# pipeline on every 5-min sweep — re-POSTing every already-succeeded blocking +# webhook hundreds of times over the 48h window. To avoid that we memoise each +# successful blocking result in Redis and replay it on the next attempt. +# +# Deliberately lossy: the cache is only read on a retry attempt and only +# written when the task decides to RETRY (never on the happy path), and any +# miss / eviction / deploy-time schema drift simply re-fires the webhook. The +# webhook contract is already at-least-once (receivers dedupe on ``jti``), so a +# rare extra delivery is fine — we only need to turn "hundreds" into "a few". + +_WEBHOOK_RESULT_CACHE_VERSION = 1 +# Cover the whole quarantine window so a result cached on attempt 1 is still +# served on the last retry before the message is delivered/abandoned. +_WEBHOOK_RESULT_CACHE_TTL = int(QUARANTINE_AFTER.total_seconds()) + + +def _webhook_results_cache_key(inbound_message_id: str) -> str: + return f"inbound_webhook_results:{inbound_message_id}" + + +def load_cached_webhook_results( + inbound_message_id: str, +) -> Dict[Tuple[str, str], _HttpResult]: + """Load memoised blocking-webhook results for one inbound message. + + Returns ``{(channel_id, phase): _HttpResult}``, or ``{}`` on a miss or + ANY deserialisation problem — the cache is a best-effort optimisation, + never a source of truth, so every error path degrades to "re-fire". + """ + try: + blob = cache.get(_webhook_results_cache_key(inbound_message_id)) + if ( + not isinstance(blob, dict) + or blob.get("version") != _WEBHOOK_RESULT_CACHE_VERSION + ): + return {} + out: Dict[Tuple[str, str], _HttpResult] = {} + for entry in blob.get("results") or []: + channel_id = entry.get("channel") + phase = entry.get("phase") + if not channel_id or not phase: + continue + out[(str(channel_id), str(phase))] = _HttpResult.from_cache( + entry.get("result") or {} + ) + return out + except Exception: # pylint: disable=broad-exception-caught + # Corrupt entry, schema drift across a deploy, cache backend hiccup — + # all degrade to "re-fire", never an error on the inbound path. + return {} + + +def persist_cached_webhook_results( + inbound_message_id: str, + results: Dict[Tuple[str, str], _HttpResult], +) -> None: + """Persist blocking-webhook results so the next retry replays them. + + Best-effort: a write failure just means the webhooks re-fire next attempt. + Called only when the task decides to RETRY, so the happy path never writes. + """ + if not results: + return + try: + blob = { + "version": _WEBHOOK_RESULT_CACHE_VERSION, + "results": [ + {"channel": channel_id, "phase": phase, "result": result.to_cache()} + for (channel_id, phase), result in results.items() + ], + } + cache.set( + _webhook_results_cache_key(inbound_message_id), + blob, + timeout=_WEBHOOK_RESULT_CACHE_TTL, + ) + except Exception: # pylint: disable=broad-exception-caught + logger.warning( + "Failed to persist webhook result cache for %s", inbound_message_id + ) + def _read_capped_body(response, deadline: Optional[float] = None) -> bytes: """Read at most ``MAX_RESPONSE_BODY`` bytes from a streaming response. @@ -200,9 +332,11 @@ def _classify_response_body(body_bytes: bytes) -> _HttpResult: Empty body or non-JSON body → plain CONTINUE. JSON shape (all keys optional): - - ``action``: ``"drop"`` short-circuits delivery; ``"retry"`` asks - us to re-queue the inbound task; anything else (``"accept"``, - missing) → CONTINUE. + - ``action``: ``"drop"`` short-circuits delivery; anything else + (``"accept"``, missing, or an unknown value) → CONTINUE. A 2xx is + a *successful* response, so it can't ask to be retried — a receiver + that needs redelivery returns a non-2xx status (e.g. 429/503), + handled as a transient failure → RETRY. - ``is_spam``: bool; overrides the pipeline's spam verdict. - ``add_labels``: list of label UUID strings; the pipeline validates them against the destination mailbox. @@ -224,12 +358,12 @@ def _classify_response_body(body_bytes: bytes) -> _HttpResult: result = _HttpResult() action = payload.get("action") - if isinstance(action, str): - action = action.lower() - if action == "drop": - result.decision = Decision.DROP - elif action == "retry": - result.decision = Decision.RETRY + if isinstance(action, str) and action.lower() == "drop": + # ``drop`` is the only receiver-chosen decision on a 2xx: discard the + # mail. There is deliberately no body-driven "retry" — a successful + # (2xx) response can't ask to be retried; redelivery is signalled by + # a non-2xx status (handled as a transient failure → RETRY). + result.decision = Decision.DROP is_spam = payload.get("is_spam") if isinstance(is_spam, bool): @@ -526,7 +660,6 @@ class UserWebhookStep: only logged) - blocking: * 2xx + ``{"action":"drop"}`` → DROP (the *only* path to DROP) - * 2xx + ``{"action":"retry"}`` → RETRY * 2xx + anything else → CONTINUE (with optional side effects) * any non-2xx (4xx / 5xx / 3xx) → RETRY * SSRF / missing secret / unknown auth_method → RETRY @@ -539,6 +672,7 @@ class UserWebhookStep: def __init__(self, channel: models.Channel, phase: str): self.channel = channel + self.phase = phase # Phase suffix in the name lets the task return value carry # "which phase did this drop happen at" without a separate field. self.name = f"webhook[{channel.id}]:{phase}" @@ -570,20 +704,31 @@ def __call__(self, ctx: InboundContext) -> Decision: ctx.pending_webhooks.append((self.channel.id, ctx.is_spam)) return Decision.CONTINUE - body_format = cfg.get("format", DEFAULT_FORMAT) - content_type, body_bytes = _resolve_body( - body_format, ctx.raw_data, ctx.parsed_email - ) - - result = _dispatch_webhook( - channel=self.channel, - mailbox=ctx.mailbox, - is_spam=ctx.is_spam, - recipient_email=ctx.recipient_email, - content_type=content_type, - body_bytes=body_bytes, - blocking=True, - ) + # Replay a memoised result from a previous attempt instead of + # re-POSTing (see the cross-retry result cache above). The map is only + # pre-loaded on a retry attempt, so on the happy path this is always a + # miss and we fire normally. + cache_key = (str(self.channel.id), self.phase) + result = ctx.blocking_webhook_results.get(cache_key) + if result is None: + body_format = cfg.get("format", DEFAULT_FORMAT) + content_type, body_bytes = _resolve_body( + body_format, ctx.raw_data, ctx.parsed_email + ) + result = _dispatch_webhook( + channel=self.channel, + mailbox=ctx.mailbox, + is_spam=ctx.is_spam, + recipient_email=ctx.recipient_email, + content_type=content_type, + body_bytes=body_bytes, + blocking=True, + ) + # Memoise only terminal (non-RETRY) outcomes — a webhook that + # failed must re-fire next attempt, not be served from cache. The + # task persists this map to Redis iff it ends up RETRYing. + if result.decision != Decision.RETRY: + ctx.blocking_webhook_results[cache_key] = result if result.is_spam_override is not None: ctx.is_spam = result.is_spam_override ctx.labels |= result.labels diff --git a/src/backend/core/mda/inbound_create.py b/src/backend/core/mda/inbound_create.py index 5ef5e0460..f05a228d9 100644 --- a/src/backend/core/mda/inbound_create.py +++ b/src/backend/core/mda/inbound_create.py @@ -259,6 +259,18 @@ def _create_message_from_inbound( # pylint: disable=too-many-arguments mime_id, mailbox.id, ) + # Dedup hit on ``(mailbox, mime_id)``: this call did NOT create + # the row. The common cause is a DUPLICATE INBOUND EMAIL — an + # upstream MTA redelivered the same Message-ID (SMTP retry / + # greylisting / relay double-send), so the original ``Message`` + # already exists; a concurrent reprocess could land here too, + # but the prefork ``time_limit`` / lock-TTL coupling makes that + # structurally rare. Signal the caller so it skips the + # non-idempotent finalize side effects (events, draft replies, + # autoreply, the ``message.delivered`` webhook) that already + # ran for the original create — re-running would duplicate them. + # pylint: disable-next=protected-access + existing_message._created_now = False # noqa: SLF001 return existing_message # --- 3. Find or Create Thread --- # @@ -640,6 +652,10 @@ def _create_message_from_inbound( # pylint: disable=too-many-arguments mailbox.id, thread.id, ) + # Freshly created this call — the caller may run the one-shot finalize + # side effects (see the dedup branch above for why this matters). + # pylint: disable-next=protected-access + message._created_now = True # noqa: SLF001 return message # Return created Message on success (truthy), None on failure diff --git a/src/backend/core/mda/inbound_pipeline.py b/src/backend/core/mda/inbound_pipeline.py index e83b668b4..f190ad942 100644 --- a/src/backend/core/mda/inbound_pipeline.py +++ b/src/backend/core/mda/inbound_pipeline.py @@ -135,6 +135,18 @@ class InboundContext: # pylint: disable=too-many-instance-attributes # the symbols (DKIM/DMARC verdicts) without a second HTTP call. rspamd_result: Optional[Dict[str, Any]] = None + # Memoised results of blocking webhook steps, keyed by + # ``(channel_id, phase)``. Pre-loaded from Redis at the start of a + # *retry* attempt so an already-succeeded blocking webhook is replayed + # from cache instead of re-POSTed — without this, a sustained rspamd + # outage (which RETRYs after the before-spam webhooks have run) would + # re-fire every before-spam webhook on each 5-min sweep, hundreds of + # times. Empty on the happy path; the task persists it back only when it + # decides to RETRY. Values are opaque ``_HttpResult`` objects owned by + # ``dispatch_webhooks`` — carried here, not interpreted, to avoid an + # import cycle. + blocking_webhook_results: Dict[Tuple[str, str], Any] = field(default_factory=dict) + # A Step is just a callable. It MUST have a ``.name`` attribute so # logs and the task return value can report which step aborted. diff --git a/src/backend/core/mda/inbound_tasks.py b/src/backend/core/mda/inbound_tasks.py index 589cccabe..2f0bef664 100644 --- a/src/backend/core/mda/inbound_tasks.py +++ b/src/backend/core/mda/inbound_tasks.py @@ -12,13 +12,19 @@ from django.conf import settings from django.core.cache import cache +from django.db import transaction from django.utils import timezone +from celery.exceptions import SoftTimeLimitExceeded from celery.utils.log import get_task_logger from jmap_email import JmapEmail, first_address_email, parse_email from core import models -from core.mda.dispatch_webhooks import dispatch_recorded_webhooks +from core.mda.dispatch_webhooks import ( + dispatch_recorded_webhooks, + load_cached_webhook_results, + persist_cached_webhook_results, +) from core.mda.inbound_create import _create_message_from_inbound from core.mda.inbound_pipeline import ( QUARANTINE_AFTER, @@ -38,6 +44,24 @@ logger = get_task_logger(__name__) +# Hard ceiling on one inbound task's wall-clock (Celery kills the task here). +# A deliberately non-configurable constant: it's an internal safety bound +# sized to the worst-case blocking-webhook budget (each up to 30s, fired for +# every matching channel across both pipeline phases), not an operator knob. +# The soft limit fires 60s earlier, raising ``SoftTimeLimitExceeded`` inside +# the task so it bails out gracefully (releases the lock, holds for retry) +# instead of being hard-killed mid-flight. +_INBOUND_TASK_TIME_LIMIT = 600 # seconds (10 min) +_INBOUND_TASK_SOFT_TIME_LIMIT = max(_INBOUND_TASK_TIME_LIMIT - 60, 1) +# The per-message lock must outlive the hard limit. On a clean (or soft-limit) +# exit the ``finally`` releases it immediately; on a hard-kill / worker OOM the +# lock is freed only by this TTL. Setting it just past the hard limit means a +# *live* task can never have its lock stolen (Celery kills the task before the +# lock expires), while a *dead* task's lock frees ~a minute later so the 5-min +# sweep can retry. +_INBOUND_TASK_LOCK_TTL = _INBOUND_TASK_TIME_LIMIT + 60 + + def _is_selfcheck(parsed_email: JmapEmail, recipient_email: str) -> bool: """Strict envelope match for the configured self-probe. @@ -120,10 +144,13 @@ def _retry_or_abandon( Within ``QUARANTINE_AFTER`` the row is kept so the 5-min sweep retries it (a transient DB error or constraint hiccup clears on its own). Past - the window the attempt is abandoned and the row deleted: otherwise a - poison message (one that parses but can never be created) loops - forever, re-running the whole pipeline — and re-firing every user - webhook — on each sweep. + the window the attempt is abandoned: ``abandoned_at`` is stamped so the + sweep skips the row and stops re-running the whole pipeline — and + re-firing every user webhook — on it, but the row is NOT deleted. The + raw bytes (``raw_data`` or the referenced ``blob``) are the only copy of + the message, so deleting would silently lose mail; instead an operator + can inspect and replay the row from the Django admin, and ``logger.error`` + raises a Sentry alert. ``error_message`` keeps the human-readable reason. """ age = timezone.now() - inbound_message.created_at if age <= QUARANTINE_AFTER: @@ -141,7 +168,11 @@ def _retry_or_abandon( age, reason, ) - inbound_message.delete() + # Keep the row (and its bytes) — stamp it terminally failed so the sweep + # skips it instead of deleting and losing the only copy of the mail. + inbound_message.error_message = reason + inbound_message.abandoned_at = timezone.now() + inbound_message.save(update_fields=["error_message", "abandoned_at"]) return { "success": False, "inbound_message_id": str(inbound_message.id), @@ -172,7 +203,11 @@ def _stamp_processing_failed(ctx: InboundContext) -> None: logger.warning("Failed to re-parse after prepending X-StMsg-Processing-Failed") -@celery_app.task(bind=True) +@celery_app.task( + bind=True, + time_limit=_INBOUND_TASK_TIME_LIMIT, + soft_time_limit=_INBOUND_TASK_SOFT_TIME_LIMIT, +) def process_inbound_message_task(self, inbound_message_id: str): """Process an inbound message: run the pipeline, persist the result. @@ -180,11 +215,13 @@ def process_inbound_message_task(self, inbound_message_id: str): messages still need work. On DROP, the ``InboundMessage`` row is deleted (we're done with it) and the task reports success. """ - # Redis lock keyed on the message id prevents two workers from - # racing on the same row. Auto-expires after 5 min so a hung worker - # doesn't block the next sweep. + # Redis lock keyed on the message id prevents two workers from racing on + # the same row. Its TTL is the task's hard time limit + 60s, so a live + # task (which Celery kills at the hard limit) can never have its lock + # stolen, while a crashed/OOM'd worker's lock still auto-frees for the + # next sweep. lock_key = f"process_inbound_message_lock:{inbound_message_id}" - if not cache.add(lock_key, "locked", 300): + if not cache.add(lock_key, "locked", _INBOUND_TASK_LOCK_TTL): logger.warning( "InboundMessage %s is already being processed — skipping", inbound_message_id, @@ -200,6 +237,16 @@ def process_inbound_message_task(self, inbound_message_id: str): logger.error(error_msg) return {"success": False, "error": error_msg} + if inbound_message.abandoned_at is not None: + # Terminally failed on an earlier attempt. The sweep already + # excludes these; this guards a direct re-dispatch so a poison + # message can never resume looping the pipeline. + return { + "success": False, + "inbound_message_id": str(inbound_message_id), + "error": "abandoned", + } + raw_data_bytes = inbound_message.get_raw_bytes() parsed_email = parse_email(raw_data_bytes) if parsed_email is None: @@ -232,6 +279,16 @@ def process_inbound_message_task(self, inbound_message_id: str): inbound_message_id, ) + # On a retry attempt (the row has been processed before, so it carries + # an ``error_message``) replay the blocking-webhook results memoised on + # the previous attempt — so a sustained downstream failure (e.g. rspamd + # down) doesn't re-POST every already-succeeded blocking webhook on + # each 5-min sweep. The happy path (first attempt) skips this read. + if inbound_message.error_message: + ctx.blocking_webhook_results = load_cached_webhook_results( + str(inbound_message.id) + ) + decision, aborted_by = run_inbound_pipeline(build_inbound_pipeline(ctx), ctx) if decision == Decision.DROP: @@ -250,6 +307,13 @@ def process_inbound_message_task(self, inbound_message_id: str): if decision == Decision.RETRY: age = timezone.now() - inbound_message.created_at if age <= QUARANTINE_AFTER: + # About to hold for retry: persist the blocking webhooks that + # DID succeed this round so the next attempt replays them + # instead of re-POSTing. Written only here, on the retry path — + # the happy path never touches Redis. + persist_cached_webhook_results( + str(inbound_message.id), ctx.blocking_webhook_results + ) return _handle_retry(inbound_message, aborted_by) # Past the quarantine window: a processing step (blocking # webhook, rspamd, …) has failed persistently. Stop holding — @@ -278,21 +342,53 @@ def process_inbound_message_task(self, inbound_message_id: str): # vet — suppress it for quarantined messages. ctx.skip_autoreply = True - inbound_msg = _create_message_from_inbound( - recipient_email=ctx.recipient_email, - parsed_email=ctx.parsed_email, - raw_data=ctx.raw_data, - mailbox=mailbox, - channel=inbound_message.channel, - is_spam=False if quarantined else bool(ctx.is_spam), - is_trashed=ctx.mark_trashed, - is_archived=ctx.mark_archived, - ) + # Create the Message and drop the queue row as one unit: either the + # message persists and the InboundMessage is gone, or neither is. This + # closes the crash window where the message committed but the queue row + # survived, leaving the 5-min sweep to reprocess and re-run the + # one-shot finalize side effects below. + with transaction.atomic(): + inbound_msg = _create_message_from_inbound( + recipient_email=ctx.recipient_email, + parsed_email=ctx.parsed_email, + raw_data=ctx.raw_data, + mailbox=mailbox, + channel=inbound_message.channel, + is_spam=False if quarantined else bool(ctx.is_spam), + is_trashed=ctx.mark_trashed, + is_archived=ctx.mark_archived, + ) + if inbound_msg: + inbound_message.delete() if inbound_msg: - inbound_message.delete() + # Run the finalize side effects only when THIS call created the + # message. ``_create_message_from_inbound`` returns the existing + # row with ``_created_now=False`` whenever it dedups on + # ``(mailbox, mime_id)`` — most commonly a DUPLICATE INBOUND EMAIL: + # an upstream MTA redelivers the same Message-ID (SMTP retry, + # greylisting, a relay double-sending), so we get a second + # ``InboundMessage`` and process it later. (A concurrent second + # task could also land here, but is structurally prevented in + # practice — the prefork hard ``time_limit`` kills a task before + # its lock TTL frees; see ``process_inbound_message_task``.) Either + # way the side effects already ran for the original create, so + # repeating them here would duplicate them. + # + # The gate (not any inherent idempotency) is what makes this safe: + # events create a ThreadEvent, drafts create a Message, the + # autoreply SENDS an email, and the non-blocking webhook POSTs + # ``message.delivered`` to the receiver — all external, none + # idempotent. (Labels / assigns / flags happen to be idempotent and + # could run unconditionally, but are gated with the rest for + # simplicity — there is nothing new to apply on a dedup hit anyway.) + # NB: ``message.delivered`` is independently at-least-once at the + # Celery layer; this only stops a duplicate *enqueue* on reprocess. + created_now = isinstance(inbound_msg, models.Message) and getattr( + inbound_msg, "_created_now", False + ) - if isinstance(inbound_msg, models.Message): + if created_now: # Each finalize step is isolated — a failure in one # (DB hiccup, race with admin deletion) must not skip # the others. The message has landed; best effort. @@ -348,7 +444,7 @@ def process_inbound_message_task(self, inbound_message_id: str): ), ) - if isinstance(inbound_msg, models.Message) and not ctx.skip_autoreply: + if created_now and not ctx.skip_autoreply: from core.mda.autoreply import ( # pylint: disable=import-outside-toplevel try_send_autoreply, ) @@ -392,6 +488,26 @@ def process_inbound_message_task(self, inbound_message_id: str): inbound_message, "Failed to create message from inbound message" ) + except SoftTimeLimitExceeded: + # The task ran past its soft time limit (almost always a slow chain of + # blocking webhooks). Bail out gracefully while we still can — before + # the hard limit SIGKILLs us — so the ``finally`` below releases the + # lock cleanly. Hold for retry: a message that *always* overruns + # (e.g. far too many slow blocking webhooks) is bounded by the same + # 48h quarantine and ends up abandoned (kept + marked) rather than + # looping forever. + logger.warning( + "Inbound message %s exceeded the %ss soft time limit — holding for retry", + inbound_message_id, + _INBOUND_TASK_SOFT_TIME_LIMIT, + ) + if inbound_message: + return _retry_or_abandon( + inbound_message, + f"Processing exceeded the {_INBOUND_TASK_SOFT_TIME_LIMIT}s " + "soft time limit", + ) + return {"success": False, "error": "soft_time_limit"} except Exception as e: logger.exception( "Error processing inbound message %s: %s", inbound_message_id, e @@ -422,7 +538,11 @@ def process_inbound_messages_queue_task(self, batch_size: int = 10): # Only retry messages older than 5 minutes retry_threshold = timezone.now() - timezone.timedelta(minutes=5) old_messages = models.InboundMessage.objects.filter( - created_at__lt=retry_threshold + created_at__lt=retry_threshold, + # Terminally-failed rows are kept for inspection/replay but must + # not be retried — otherwise the poison message loops the pipeline + # (and re-fires every user webhook) every 5 minutes forever. + abandoned_at__isnull=True, ).order_by("created_at")[:batch_size] total = len(old_messages) @@ -455,3 +575,54 @@ def process_inbound_messages_queue_task(self, batch_size: int = 10): "errors": errors, "total": total, } + + +# How long an abandoned InboundMessage (``abandoned_at`` set) is kept before +# the purge sweep reclaims it: long enough for an operator to act on the +# Sentry alert (inspect / replay from the admin), short enough that a stream +# of poison mail can't grow the transient queue table without bound. +_ABANDONED_RETENTION = timezone.timedelta(days=7) + + +@celery_app.task(bind=True) +def purge_abandoned_inbound_messages_task( + self, batch_size: int = 500, max_batches: int = 200 +): + """Reclaim inbound messages abandoned more than ``_ABANDONED_RETENTION`` ago. + + Abandoned rows are deliberately kept (never deleted at abandon time) so the + mail stays inspectable / replayable — see ``_retry_or_abandon``. But they + must not accumulate forever: a sustained stream of unparseable / uncreatable + mail would otherwise grow this transient queue table (and its inline + ``raw_data`` bytes) without bound. This daily sweep deletes rows past the + retention window. + + Deletes in batches through ``QuerySet.delete()`` (not ``_raw_delete``) so + the ``post_delete`` signal fires per row and any referenced blob is + scheduled for GC. ``max_batches`` caps a single run, so a large backlog + (e.g. after an abuse spike) drains over a few days instead of one giant + locking transaction. + """ + cutoff = timezone.now() - _ABANDONED_RETENTION + purged = 0 + for _ in range(max_batches): + ids = list( + models.InboundMessage.objects.filter( + abandoned_at__isnull=False, + abandoned_at__lt=cutoff, + ) + .order_by("abandoned_at") + .values_list("id", flat=True)[:batch_size] + ) + if not ids: + break + models.InboundMessage.objects.filter(id__in=ids).delete() + purged += len(ids) + + if purged: + logger.info( + "Purged %s abandoned inbound message(s) older than %s", + purged, + _ABANDONED_RETENTION, + ) + return {"success": True, "purged": purged} diff --git a/src/backend/core/mda/outbound.py b/src/backend/core/mda/outbound.py index 6fdec5f15..0ed65e958 100644 --- a/src/backend/core/mda/outbound.py +++ b/src/backend/core/mda/outbound.py @@ -669,6 +669,28 @@ def _mark_delivered( # "handed off to the recipient's inbound queue" — # the recipient's webhook outcome plays out on their # side and never feeds back into the sender's status. + # + # DESIGN TRADEOFF (intentional): the sender's recipient + # is marked SENT_INTERNAL on handoff, before the + # recipient pipeline actually creates (or DROPs) the + # Message. So "delivered" is really "accepted for + # delivery" — it can be a white lie if a recipient-side + # blocking webhook DROPs the message or creation fails. + # We deliberately do NOT add a QUEUED status that later + # resolves to SENT_INTERNAL / FAILED, because doing it + # honestly needs a cross-mailbox callback we don't have: + # the recipient's inbound finalizer would have to reach + # back and update THIS sender's MessageRecipient. That + # means threading the sender's MessageRecipient id + # through InboundMessage (a new nullable FK), new + # QUEUED->SENT_INTERNAL/FAILED transitions in the + # delivery-status state machine, a frontend "pending" + # rendering, and a new metrics label. Same-instance + # delivery lands in milliseconds, so the gain is a + # sub-second "queued" flicker that only matters in the + # rare recipient-DROP/failure case. If that accuracy is + # ever needed, build it as a focused follow-up with the + # callback above — not as a status flip here. delivered = deliver_inbound_message( recipient_email, parsed_email, diff --git a/src/backend/core/migrations/0032_inboundmessage_blob_inboundmessage_is_internal_and_more.py b/src/backend/core/migrations/0032_inboundmessage_blob_inboundmessage_is_internal_and_more.py index 5db9b6967..019daff4d 100644 --- a/src/backend/core/migrations/0032_inboundmessage_blob_inboundmessage_is_internal_and_more.py +++ b/src/backend/core/migrations/0032_inboundmessage_blob_inboundmessage_is_internal_and_more.py @@ -11,6 +11,11 @@ class Migration(migrations.Migration): ] operations = [ + migrations.AddField( + model_name='inboundmessage', + name='abandoned_at', + field=models.DateTimeField(blank=True, help_text='Set when processing was permanently abandoned after the retry window; the row is kept for inspection/replay and skipped by the retry sweep.', null=True, verbose_name='abandoned at'), + ), migrations.AddField( model_name='inboundmessage', name='blob', diff --git a/src/backend/core/models.py b/src/backend/core/models.py index bc97d7bbb..9339046b0 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -2288,6 +2288,24 @@ class InboundMessage(BaseModel): blank=True, help_text="Error message if processing failed", ) + # Terminal-failure marker. Set when processing is permanently given up + # after the quarantine window (a poison message that can never be + # parsed/created). The row and its bytes are KEPT — never deleted at + # abandon time — so the mail stays recoverable; the retry sweep skips + # rows with this set, so the pipeline (and user webhooks) stop re-firing. + # Doubles as the audit timestamp ("when did it die") and the cutoff key + # for ``purge_abandoned_inbound_messages_task``, which reclaims rows once + # they pass the retention window. NULL = still live (pending or retrying). + abandoned_at = models.DateTimeField( + "abandoned at", + null=True, + blank=True, + help_text=( + "Set when processing was permanently abandoned after the retry " + "window; the row is kept for inspection/replay and skipped by the " + "retry sweep." + ), + ) class Meta: db_table = "messages_inboundmessage" diff --git a/src/backend/core/tests/api/test_channels.py b/src/backend/core/tests/api/test_channels.py index fccf2a17c..c2c531cd8 100644 --- a/src/backend/core/tests/api/test_channels.py +++ b/src/backend/core/tests/api/test_channels.py @@ -846,3 +846,52 @@ def test_regenerate_secret_rejects_unknown_auth_method(self, api_client, mailbox channel.refresh_from_db() # Old secret untouched — rotation never ran. assert channel.encrypted_settings["secret"] == "whsec_original" + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + def test_rejects_missing_url(self, api_client, mailbox): + """A webhook channel without settings.url is rejected with 400.""" + response = self._post( + api_client, + mailbox, + {"trigger": "message.delivered", "auth_method": "jwt"}, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + @pytest.mark.parametrize( + "bad_url", + [ + "ftp://host/x", # non-http(s) scheme + "javascript:alert(1)", # script scheme + "https:///no-host", # no host + ], + ) + def test_rejects_invalid_url_scheme_or_host(self, api_client, mailbox, bad_url): + """Non-http(s) schemes, script URLs and host-less URLs are rejected + up front — the create-time check the SSRF guard backstops at + dispatch.""" + response = self._post( + api_client, + mailbox, + { + "url": bad_url, + "trigger": "message.delivered", + "auth_method": "jwt", + }, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.content + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=False) + def test_rejects_plain_http_outside_debug(self, api_client, mailbox): + """Plain http would leak the HMAC/JWT signing headers in transit, so + it is rejected unless running under DEBUG (local-dev escape hatch).""" + response = self._post( + api_client, + mailbox, + { + "url": "http://hook.example.com/in", + "trigger": "message.delivered", + "auth_method": "jwt", + }, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.content diff --git a/src/backend/core/tests/mda/test_dispatch_webhooks.py b/src/backend/core/tests/mda/test_dispatch_webhooks.py index 0cdf93813..f6cd53398 100644 --- a/src/backend/core/tests/mda/test_dispatch_webhooks.py +++ b/src/backend/core/tests/mda/test_dispatch_webhooks.py @@ -16,6 +16,7 @@ from django.test import override_settings from django.utils import timezone as dj_timezone +import jwt import pytest import requests as requests_lib @@ -27,10 +28,16 @@ FORMAT_JMAP, PHASE_AFTER_SPAM, PHASE_BEFORE_SPAM, + WEBHOOK_CONNECT_TIMEOUT, + WEBHOOK_TIMEOUT, + UserWebhookStep, _classify_response_body, + _HttpResult, build_jmap_email, dispatch_webhook_task, find_webhook_channels_for_mailbox, + load_cached_webhook_results, + persist_cached_webhook_results, webhook_steps_for_mailbox, ) from core.mda.inbound_pipeline import ( @@ -38,7 +45,11 @@ Decision, InboundContext, ) -from core.mda.inbound_tasks import process_inbound_message_task +from core.mda.inbound_tasks import ( + process_inbound_message_task, + process_inbound_messages_queue_task, + purge_abandoned_inbound_messages_task, +) from core.services.ssrf import SSRFValidationError @@ -1185,6 +1196,112 @@ def test_missing_secret_retries_when_blocking( assert outcome.decision == Decision.RETRY mock_session.return_value.post.assert_not_called() + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_jwt_bearer_token_verifies_and_binds_body( + self, mock_session, mailbox, parsed_email + ): + """The ``Authorization: Bearer`` JWT must actually verify with the + channel secret (HS256) and carry the documented claims, including + ``body_sha256`` bound to the exact bytes POSTed — a receiver + relying on the standard JWT verify path depends on this.""" + channel = factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "trigger": "message.delivering", + "auth_method": "jwt", + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"From: a\r\n\r\nbody", + is_spam=False, + ) + kwargs = mock_session.return_value.post.call_args.kwargs + body_bytes = kwargs["data"] + auth = kwargs["headers"]["Authorization"] + assert auth.startswith("Bearer ") + token = auth.split(" ", 1)[1] + + # Decoding with the right secret succeeds (signature + exp valid). + claims = jwt.decode(token, self.SECRET, algorithms=["HS256"]) + assert claims["iss"] == "messages-webhook" + assert claims["exp"] == claims["iat"] + 300 + assert claims["cid"] == str(channel.id) + assert claims["jti"] # replay nonce present + # The JWT binds to the exact posted bytes. + assert claims["body_sha256"] == hashlib.sha256(body_bytes).hexdigest() + + # A wrong secret must fail the signature check — the token is + # genuinely keyed, not merely well-formed. + with pytest.raises(jwt.InvalidSignatureError): + jwt.decode(token, "wrong-secret", algorithms=["HS256"]) + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_rotation_changes_signing_secret(self, mock_session, mailbox, parsed_email): + """``rotate_secret`` invalidates the old secret immediately: the + next dispatch signs with the NEW secret, and a signature recomputed + with the OLD secret no longer matches.""" + channel = factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "trigger": "message.delivering", + "auth_method": "jwt", + "format": "eml", + }, + ) + raw = b"From: a\r\n\r\nbody" + old_secret = channel.encrypted_settings["secret"] + + mock_session.return_value.post.return_value = _make_response(200) + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=raw, + is_spam=False, + ) + + # Rotate: a fresh secret is persisted; the dispatcher re-reads the + # channel from the DB on the next sweep. + new_secret = channel.rotate_secret() + assert new_secret != old_secret + + mock_session.return_value.post.reset_mock() + mock_session.return_value.post.return_value = _make_response(200) + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=raw, + is_spam=False, + ) + headers = mock_session.return_value.post.call_args.kwargs["headers"] + ts = headers["X-StMsg-Webhook-Timestamp"] + sig = headers["X-StMsg-Webhook-Signature"].split("=", 1)[1] + + expected_new = hmac.new( + new_secret.encode("utf-8"), + ts.encode("ascii") + b"." + raw, + hashlib.sha256, + ).hexdigest() + expected_old = hmac.new( + old_secret.encode("utf-8"), + ts.encode("ascii") + b"." + raw, + hashlib.sha256, + ).hexdigest() + assert hmac.compare_digest(sig, expected_new) + assert not hmac.compare_digest(sig, expected_old) + # --- integration with process_inbound_message_task --- # @@ -1325,14 +1442,20 @@ def test_creation_failure_retries_then_abandons(self, mock_rspamd, mock_create): assert result["error"] == "retry" assert models.InboundMessage.objects.filter(id=inbound_message.id).exists() - # Aged past the window → abandoned, row deleted (loop stops). + # Aged past the window → abandoned. The row is KEPT (its raw bytes + # are the only copy of the mail) but stamped terminally failed so + # the sweep stops re-running the pipeline + webhooks on it. models.InboundMessage.objects.filter(id=inbound_message.id).update( created_at=dj_timezone.now() - QUARANTINE_AFTER - QUARANTINE_AFTER ) with patch.object(process_inbound_message_task, "update_state", Mock()): result = process_inbound_message_task.run(str(inbound_message.id)) assert result["error"] == "abandoned" - assert not models.InboundMessage.objects.filter(id=inbound_message.id).exists() + assert models.InboundMessage.objects.filter(id=inbound_message.id).exists() + inbound_message.refresh_from_db() + # Terminally marked via the typed field; error_message keeps the reason. + assert inbound_message.abandoned_at is not None + assert inbound_message.error_message # --- non-blocking dispatch isolation --- # @@ -1752,6 +1875,14 @@ def test_action_unknown_is_continue(self): outcome = _classify_response_body(b'{"action": "quarantine"}') assert outcome.decision == Decision.CONTINUE + def test_action_retry_is_continue(self): + """A 2xx body can no longer request a retry: ``action == "retry"`` + is treated like any other unknown action → CONTINUE. A 2xx is + success; redelivery is signalled only by a non-2xx status, so the + body cannot ask a successful response to be re-POSTed.""" + outcome = _classify_response_body(b'{"action": "retry"}') + assert outcome.decision == Decision.CONTINUE + def test_is_spam_true_sets_override(self): outcome = _classify_response_body(b'{"is_spam": true}') assert outcome.decision == Decision.CONTINUE @@ -2140,6 +2271,71 @@ def _counting_iter(*_args, **_kwargs): # And the connection was returned to the pool. response.close.assert_called_once() + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_blocking_retries_on_response_read_timeout( + self, mock_session, mailbox, parsed_email + ): + """A receiver that drip-feeds the body just under the per-read + socket timeout trips the total-exchange deadline mid-read. That + must surface as RETRY (transport failure), NOT a benign empty-body + CONTINUE — otherwise a slow-loris receiver could silently deliver + every message as accept.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "trigger": "message.delivering", + "auth_method": "jwt", + }, + ) + response = _make_response(200) + # Reading the streamed body exceeds the time budget. + response.iter_content = Mock(side_effect=TimeoutError("slow drip")) + mock_session.return_value.post.return_value = response + + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + assert outcome.decision == Decision.RETRY + # The streamed response is still released back to the pool. + response.close.assert_called_once() + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_post_uses_bounded_timeout_and_streaming( + self, mock_session, mailbox, parsed_email + ): + """The POST must pass the (connect, read) timeout tuple and + ``stream=True`` so a hostile receiver can't pin a worker on connect + or OOM it on a giant unread body. Asserted explicitly so neither + bound can be dropped silently.""" + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "trigger": "message.delivering", + "auth_method": "jwt", + }, + ) + mock_session.return_value.post.return_value = _make_response(200) + dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + kwargs = mock_session.return_value.post.call_args.kwargs + assert kwargs["timeout"] == (WEBHOOK_CONNECT_TIMEOUT, WEBHOOK_TIMEOUT) + assert kwargs["stream"] is True + # --- pipeline integration: RETRY, label apply, antispam override --- # @@ -3120,3 +3316,467 @@ def test_quarantine_force_is_spam_false( blob_content = message.blob.get_content() assert b"X-StMsg-Processing-Failed: true" in blob_content mock_autoreply.assert_not_called() + + +@pytest.mark.django_db +class TestAbandonedRowHandling: + """A row that fails to be CREATED past the quarantine window is + abandoned: kept (its raw bytes are the only copy of the mail) but + stamped via ``abandoned_at`` so neither the 5-min sweep nor a direct + re-dispatch ever re-runs the pipeline (and re-fires every webhook) + on it again.""" + + def test_sweep_skips_abandoned_rows(self): + """The retry sweep must not re-queue an abandoned row.""" + mailbox = factories.MailboxFactory() + abandoned = models.InboundMessage.objects.create( + mailbox=mailbox, + raw_data=b"From: s@example.com\r\nSubject: t\r\n\r\nbody", + error_message="persistent failure", + abandoned_at=dj_timezone.now(), + ) + # Age it past the 5-minute retry threshold so it would otherwise + # be picked up by the sweep. + models.InboundMessage.objects.filter(id=abandoned.id).update( + created_at=dj_timezone.now() - timedelta(minutes=10) + ) + + with patch( + "core.mda.inbound_tasks.process_inbound_message_task.delay" + ) as mock_delay: + process_inbound_messages_queue_task.run() + + dispatched = [call.args[0] for call in mock_delay.call_args_list] + assert str(abandoned.id) not in dispatched + + @patch("core.mda.inbound_tasks._create_message_from_inbound") + @patch("core.mda.inbound_pipeline._call_rspamd") + def test_redispatch_of_abandoned_row_early_returns(self, mock_rspamd, mock_create): + """A direct re-dispatch of an abandoned row early-returns + ``{"error": "abandoned"}`` without ever running the pipeline — + rspamd is never consulted and no creation is attempted.""" + mailbox = factories.MailboxFactory() + abandoned = models.InboundMessage.objects.create( + mailbox=mailbox, + raw_data=b"From: s@example.com\r\nSubject: t\r\n\r\nbody", + error_message="persistent failure", + abandoned_at=dj_timezone.now(), + ) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + result = process_inbound_message_task.run(str(abandoned.id)) + + assert result["error"] == "abandoned" + mock_rspamd.assert_not_called() + mock_create.assert_not_called() + # Row preserved for operator inspection / replay. + assert models.InboundMessage.objects.filter(id=abandoned.id).exists() + + def test_purge_deletes_old_abandoned_rows(self): + """The retention purge reclaims rows abandoned past the window.""" + mailbox = factories.MailboxFactory() + old = models.InboundMessage.objects.create( + mailbox=mailbox, + raw_data=b"x", + error_message="persistent failure", + abandoned_at=dj_timezone.now() - timedelta(days=8), + ) + + result = purge_abandoned_inbound_messages_task.run() + + assert result["purged"] == 1 + assert not models.InboundMessage.objects.filter(id=old.id).exists() + + def test_purge_keeps_recent_abandoned_rows(self): + """Rows abandoned within the retention window are kept.""" + mailbox = factories.MailboxFactory() + recent = models.InboundMessage.objects.create( + mailbox=mailbox, + raw_data=b"x", + error_message="persistent failure", + abandoned_at=dj_timezone.now() - timedelta(days=1), + ) + + result = purge_abandoned_inbound_messages_task.run() + + assert result["purged"] == 0 + assert models.InboundMessage.objects.filter(id=recent.id).exists() + + def test_purge_ignores_live_rows(self): + """A non-abandoned row is never touched by the purge, however old.""" + mailbox = factories.MailboxFactory() + live = models.InboundMessage.objects.create( + mailbox=mailbox, + raw_data=b"x", + ) + models.InboundMessage.objects.filter(id=live.id).update( + created_at=dj_timezone.now() - timedelta(days=30) + ) + + result = purge_abandoned_inbound_messages_task.run() + + assert result["purged"] == 0 + assert models.InboundMessage.objects.filter(id=live.id).exists() + + +@pytest.mark.django_db +class TestPipelineIdempotency: + """A duplicate inbound email — most often an upstream MTA redelivering + the same Message-ID (SMTP retry / greylisting / relay double-send), so a + second ``InboundMessage`` is processed later — must hit the + ``(mailbox, mime_id)`` dedup branch in ``_create_message_from_inbound`` + (``_created_now=False``) and skip the one-shot finalize side effects (IM + events, draft replies, autoreply, the non-blocking ``message.delivered`` + webhook) that already ran for the original create — re-running them would + duplicate them.""" + + @patch("core.mda.autoreply.try_send_autoreply") + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_dedup_hit_skips_finalize_side_effects( + self, mock_session, mock_rspamd, mock_autoreply + ): + mailbox = factories.MailboxFactory() + template = factories.MessageTemplateFactory( + mailbox=mailbox, + type=enums.MessageTemplateTypeChoices.MESSAGE, + is_active=True, + html_body="

    Thanks!

    ", + text_body="Thanks!", + raw_body={"type": "doc", "content": [{"type": "paragraph"}]}, + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "trigger": "message.delivering", + "auth_method": "jwt", + }, + ) + mock_rspamd.return_value = (False, None, None) + action_body = json.dumps( + { + "add_event": [{"type": "im", "content": "AI: urgent"}], + "reply_draft": {"template": str(template.id)}, + } + ).encode("utf-8") + mock_session.return_value.post.return_value = _make_response( + 200, body=action_body + ) + + mime = "idem-1@example.com" + raw_data = ( + b"From: customer@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: help\r\n" + b"Message-ID: <" + mime.encode() + b">\r\n\r\nbody" + ) + + # --- First pass: creates the message + all side effects. --- + im1 = models.InboundMessage.objects.create(mailbox=mailbox, raw_data=raw_data) + with patch.object(process_inbound_message_task, "update_state", Mock()): + r1 = process_inbound_message_task.run(str(im1.id)) + assert r1["success"] is True + + message = models.Message.objects.get(mime_id=mime) + thread = message.thread + + def im_event_count(): + return models.ThreadEvent.objects.filter( + thread=thread, type=enums.ThreadEventTypeChoices.IM + ).count() + + def draft_count(): + return models.Message.objects.filter(thread=thread, is_draft=True).count() + + assert im_event_count() == 1 + assert draft_count() == 1 + assert mock_autoreply.call_count == 1 # fired once, on create + + # --- Second pass: SAME Message-ID → dedup hit (_created_now=False). --- + im2 = models.InboundMessage.objects.create(mailbox=mailbox, raw_data=raw_data) + with patch.object(process_inbound_message_task, "update_state", Mock()): + process_inbound_message_task.run(str(im2.id)) + + # The pipeline DID run again (the blocking webhook fired a 2nd time), + # proving the skip is gated on _created_now, not on the pipeline + # being short-circuited. + assert mock_session.return_value.post.call_count == 2 + # ...but NO finalize side effect was repeated. + assert im_event_count() == 1 + assert draft_count() == 1 + assert mock_autoreply.call_count == 1 # still just the original + # The duplicate queue row was still consumed (one Message exists). + assert models.Message.objects.filter(mime_id=mime).count() == 1 + assert not models.InboundMessage.objects.filter(id=im2.id).exists() + + @patch("core.mda.dispatch_webhooks.dispatch_webhook_task.delay") + @patch("core.mda.inbound_pipeline._call_rspamd") + def test_dedup_hit_does_not_refire_nonblocking_webhook( + self, mock_rspamd, mock_delay + ): + """A duplicate delivery must not re-enqueue the non-blocking + ``message.delivered`` webhook — its dispatch is one of the finalize + side effects gated on ``_created_now`` (and ``message.delivered`` is + already at-least-once at the Celery layer, so a duplicate *enqueue* + here would compound that).""" + mailbox = factories.MailboxFactory() + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "trigger": "message.delivered", + "auth_method": "jwt", + }, + ) + mock_rspamd.return_value = (False, None, None) + + mime = "idem-nb@example.com" + raw_data = ( + b"From: customer@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: help\r\n" + b"Message-ID: <" + mime.encode() + b">\r\n\r\nbody" + ) + + im1 = models.InboundMessage.objects.create(mailbox=mailbox, raw_data=raw_data) + with patch.object(process_inbound_message_task, "update_state", Mock()): + process_inbound_message_task.run(str(im1.id)) + # Enqueued exactly once, on the original create. + assert mock_delay.call_count == 1 + + # Duplicate delivery: same Message-ID, separate queue row. + im2 = models.InboundMessage.objects.create(mailbox=mailbox, raw_data=raw_data) + with patch.object(process_inbound_message_task, "update_state", Mock()): + process_inbound_message_task.run(str(im2.id)) + + # Still 1 — the dedup hit skipped the non-blocking dispatch. + assert mock_delay.call_count == 1 + assert models.Message.objects.filter(mime_id=mime).count() == 1 + + +# --- cross-retry blocking-webhook result cache --- # + + +class TestHttpResultCacheSerialization: + """``_HttpResult.to_cache`` / ``from_cache`` are the JSON-able bridge + for the cross-retry result cache. They must round-trip every field and + tolerate partial/old data (a deploy that changes the dataclass must not + fail to load 48h-old entries).""" + + def test_roundtrip_preserves_every_field(self): + original = _HttpResult( + decision=Decision.DROP, + is_spam_override=True, + labels={str(uuid.uuid4()), str(uuid.uuid4())}, + assign_to=["alice@example.org", "bob@example.org"], + mark_starred=True, + mark_read=True, + mark_trashed=True, + mark_archived=True, + skip_autoreply=True, + events=[{"type": "im", "content": "x", "mentions": []}], + reply_draft_template_id=str(uuid.uuid4()), + ) + # The intermediate form is a plain JSON-able dict (no enums/sets). + cached = original.to_cache() + assert isinstance(cached, dict) + assert cached["decision"] == int(Decision.DROP) + assert isinstance(cached["labels"], list) + + restored = _HttpResult.from_cache(cached) + assert restored.decision == original.decision + assert restored.is_spam_override == original.is_spam_override + assert restored.labels == original.labels + assert restored.assign_to == original.assign_to + assert restored.mark_starred == original.mark_starred + assert restored.mark_read == original.mark_read + assert restored.mark_trashed == original.mark_trashed + assert restored.mark_archived == original.mark_archived + assert restored.skip_autoreply == original.skip_autoreply + assert restored.events == original.events + assert restored.reply_draft_template_id == original.reply_draft_template_id + # Dataclass equality covers all fields at once. + assert restored == original + + def test_from_cache_empty_yields_defaults(self): + """Missing keys (old/partial entry) degrade to a default CONTINUE + result, never an exception.""" + restored = _HttpResult.from_cache({}) + assert restored == _HttpResult() + assert restored.decision == Decision.CONTINUE + assert restored.is_spam_override is None + assert restored.labels == set() + + +@pytest.mark.django_db +class TestWebhookResultCachePersistence: + """``load_cached_webhook_results`` / ``persist_cached_webhook_results`` + against the real Django cache. The cache is a best-effort optimisation: + every miss/error must degrade to an empty map (re-fire).""" + + def test_load_miss_returns_empty(self): + assert load_cached_webhook_results(str(uuid.uuid4())) == {} + + def test_persist_then_load_roundtrip(self): + inbound_id = str(uuid.uuid4()) + cid = str(uuid.uuid4()) + label = str(uuid.uuid4()) + results = { + (cid, PHASE_BEFORE_SPAM): _HttpResult( + decision=Decision.DROP, + is_spam_override=True, + labels={label}, + assign_to=["a@example.org"], + skip_autoreply=True, + ), + } + persist_cached_webhook_results(inbound_id, results) + + loaded = load_cached_webhook_results(inbound_id) + assert set(loaded.keys()) == {(cid, PHASE_BEFORE_SPAM)} + got = loaded[(cid, PHASE_BEFORE_SPAM)] + want = results[(cid, PHASE_BEFORE_SPAM)] + assert got.decision == want.decision + assert got.is_spam_override == want.is_spam_override + assert got.labels == want.labels + assert got.assign_to == want.assign_to + assert got.skip_autoreply == want.skip_autoreply + + def test_persist_noops_on_empty(self): + inbound_id = str(uuid.uuid4()) + persist_cached_webhook_results(inbound_id, {}) + # Nothing was written → still a clean miss. + assert load_cached_webhook_results(inbound_id) == {} + + +@pytest.mark.django_db +class TestBlockingWebhookResultCache: + """The ``UserWebhookStep`` blocking branch replays a memoised result + instead of re-POSTing, while still applying its side effects.""" + + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_cache_hit_skips_post_but_applies_side_effects( + self, mock_session, mailbox, parsed_email + ): + channel = factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "trigger": "message.inbound", # before-spam, blocking + "auth_method": "jwt", + }, + ) + label_id = str(uuid.uuid4()) + cached = _HttpResult(is_spam_override=True, labels={label_id}) + ctx = InboundContext( + mailbox=mailbox, + inbound_message=Mock(id="im", created_at=dj_timezone.now()), + recipient_email=str(mailbox), + raw_data=b"raw", + parsed_email=parsed_email, + spam_config={}, + is_spam=None, + blocking_webhook_results={ + (str(channel.id), PHASE_BEFORE_SPAM): cached, + }, + ) + + step = UserWebhookStep(channel, phase=PHASE_BEFORE_SPAM) + decision = step(ctx) + + # The memoised result short-circuits the network call entirely... + assert decision == Decision.CONTINUE + mock_session.return_value.post.assert_not_called() + # ...yet its side effects are still replayed onto the context. + assert ctx.is_spam is True + assert label_id in ctx.labels + + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_blocking_webhook_not_refired_across_retries( + self, mock_session, mock_rspamd + ): + """The core storm-fix regression guard: a before-spam blocking + webhook that succeeded on attempt 1 is NOT re-POSTed on attempt 2 + when a sustained rspamd outage keeps the message in RETRY.""" + mailbox = factories.MailboxFactory() + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: storm\r\n" + b"Message-ID: \r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "trigger": "message.inbound", # before-spam, blocking + "auth_method": "jwt", + }, + ) + # Webhook succeeds (empty 200 → no is_spam opinion, so the later + # rspamd step still runs); rspamd is erroring → pipeline RETRYs. + mock_session.return_value.post.return_value = _make_response(200) + mock_rspamd.return_value = (False, "rspamd unreachable", None) + + # Attempt 1: webhook POSTed once, message held for retry. + with patch.object(process_inbound_message_task, "update_state", Mock()): + r1 = process_inbound_message_task.run(str(inbound_message.id)) + assert r1["error"] == "retry" + assert mock_session.return_value.post.call_count == 1 + # The success was memoised to Redis on the retry path. + inbound_message.refresh_from_db() + assert inbound_message.error_message # marks this as a retry attempt + + # Attempt 2 on the SAME row: the cached result is replayed, so the + # webhook is NOT POSTed again — still RETRYs (rspamd still down). + with patch.object(process_inbound_message_task, "update_state", Mock()): + r2 = process_inbound_message_task.run(str(inbound_message.id)) + assert r2["error"] == "retry" + assert mock_session.return_value.post.call_count == 1 # served from cache + + @patch("core.mda.inbound_tasks.persist_cached_webhook_results") + @patch("core.mda.inbound_tasks.load_cached_webhook_results") + @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_happy_path_does_not_touch_cache( + self, mock_session, mock_rspamd, mock_load, mock_persist + ): + """A normal single-pass delivery never reads (first attempt → no + ``error_message``) nor writes (no RETRY) the result cache.""" + mailbox = factories.MailboxFactory() + raw_data = ( + b"From: sender@example.com\r\n" + b"To: " + str(mailbox).encode() + b"\r\n" + b"Subject: happy\r\n" + b"Message-ID: \r\n\r\nbody" + ) + inbound_message = models.InboundMessage.objects.create( + mailbox=mailbox, raw_data=raw_data + ) + factories.ChannelFactory( + type=enums.ChannelTypes.WEBHOOK, + mailbox=mailbox, + settings={ + "url": "https://hook.example.com", + "trigger": "message.delivering", # after-spam, blocking + "auth_method": "jwt", + }, + ) + mock_rspamd.return_value = (False, None, None) + mock_session.return_value.post.return_value = _make_response(200) + + with patch.object(process_inbound_message_task, "update_state", Mock()): + result = process_inbound_message_task.run(str(inbound_message.id)) + + assert result["success"] is True + mock_load.assert_not_called() + mock_persist.assert_not_called() diff --git a/src/backend/core/tests/services/test_ssrf.py b/src/backend/core/tests/services/test_ssrf.py index 362314660..498107f2e 100644 --- a/src/backend/core/tests/services/test_ssrf.py +++ b/src/backend/core/tests/services/test_ssrf.py @@ -10,9 +10,11 @@ from unittest.mock import MagicMock, patch import pytest +import requests from core.services.ssrf import ( MAX_REDIRECTS, + SSRFProtectedAdapter, SSRFSafeSession, SSRFValidationError, assert_public_ip, @@ -375,3 +377,117 @@ def dns_side_effect(host, *_args, **_kwargs): SSRFSafeSession().post("https://hook.legit.com/in", timeout=10, data=b"x") assert mock_post.call_count == 1 + + +class TestSSRFProtectedAdapterPinning: + """The IP-pinning enforcement in ``SSRFProtectedAdapter``. + + This is the load-bearing TOCTOU / DNS-rebinding defense: after a + hostname is validated to a concrete IP, the adapter must dial *that + exact IP* (never re-resolve the hostname at connect time) while still + presenting the original hostname for ``Host:`` routing and TLS + certificate verification. The redirect tests above prove the + *decision* to validate each hop; these prove the *enforcement* — + that the request actually goes to the pinned IP. + """ + + def _prepared(self, url: str) -> requests.PreparedRequest: + req = requests.PreparedRequest() + req.prepare(method="POST", url=url, headers={}, data=b"payload") + return req + + @patch("requests.adapters.HTTPAdapter.send") + def test_send_rewrites_url_to_pinned_ipv4_and_keeps_host(self, mock_super_send): + """The request URL is rewritten to the validated IPv4 (with port), + not the hostname, and the Host header is set to the original + hostname so virtual-hosted receivers still route correctly.""" + adapter = SSRFProtectedAdapter( + dest_ip="93.184.216.34", + dest_port=443, + original_hostname="example.com", + original_scheme="https", + ) + request = self._prepared("https://example.com/path?q=1") + + adapter.send(request) + + # The parent adapter actually dials the rewritten request. + sent_request = mock_super_send.call_args.args[0] + assert sent_request.url == "https://93.184.216.34:443/path?q=1" + # Hostname preserved for routing + TLS SNI/cert verification. + assert sent_request.headers["Host"] == "example.com" + + @patch("requests.adapters.HTTPAdapter.send") + def test_send_rewrites_url_to_bracketed_ipv6(self, mock_super_send): + """An IPv6 destination is rewritten using the ``[addr]:port`` + netloc form so the URL stays well-formed.""" + adapter = SSRFProtectedAdapter( + dest_ip="2606:2800:220:1:248:1893:25c8:1946", + dest_port=8443, + original_hostname="example.com", + original_scheme="https", + ) + request = self._prepared("https://example.com/path?q=1") + + adapter.send(request) + + sent_request = mock_super_send.call_args.args[0] + assert sent_request.url == ( + "https://[2606:2800:220:1:248:1893:25c8:1946]:8443/path?q=1" + ) + # The Host header reflects the ORIGINAL request URL (no explicit + # port → bare hostname); the pinned dest_port only steers the + # socket, it doesn't appear in Host. + assert sent_request.headers["Host"] == "example.com" + + @patch("requests.adapters.HTTPAdapter.send") + def test_send_host_header_carries_original_nondefault_port(self, mock_super_send): + """When the ORIGINAL URL names a non-default port, that port rides + in the Host header so the receiver routes to the right vhost:port.""" + adapter = SSRFProtectedAdapter( + dest_ip="93.184.216.34", + dest_port=8443, + original_hostname="example.com", + original_scheme="https", + ) + request = self._prepared("https://example.com:8443/path?q=1") + + adapter.send(request) + + sent_request = mock_super_send.call_args.args[0] + assert sent_request.url == "https://93.184.216.34:8443/path?q=1" + assert sent_request.headers["Host"] == "example.com:8443" + + @patch("requests.adapters.HTTPAdapter.init_poolmanager") + def test_init_poolmanager_pins_tls_hostname_for_https(self, mock_super_init): + """For https, the pool manager is configured to verify the cert + against (and send SNI for) the ORIGINAL hostname, even though the + socket connects to the pinned IP.""" + SSRFProtectedAdapter( + dest_ip="93.184.216.34", + dest_port=443, + original_hostname="example.com", + original_scheme="https", + ) + + # __init__ calls init_poolmanager once during HTTPAdapter setup. + assert mock_super_init.called + pool_kwargs = mock_super_init.call_args.kwargs + assert pool_kwargs["assert_hostname"] == "example.com" + assert pool_kwargs["server_hostname"] == "example.com" + + @patch("requests.adapters.HTTPAdapter.init_poolmanager") + def test_init_poolmanager_does_not_pin_tls_for_http(self, mock_super_init): + """Plain http has no TLS handshake, so no hostname pinning kwargs + are injected (they'd be meaningless / could error).""" + SSRFProtectedAdapter( + dest_ip="93.184.216.34", + dest_port=80, + original_hostname="example.com", + original_scheme="http", + ) + + assert mock_super_init.called + pool_kwargs = mock_super_init.call_args.kwargs + assert "assert_hostname" not in pool_kwargs + assert "server_hostname" not in pool_kwargs diff --git a/src/backend/messages/celery_app.py b/src/backend/messages/celery_app.py index 5800e5437..ed9954f66 100644 --- a/src/backend/messages/celery_app.py +++ b/src/backend/messages/celery_app.py @@ -45,6 +45,14 @@ "schedule": 300.0, # Every 5 minutes "options": {"queue": "inbound"}, }, + "purge-abandoned-inbound-messages": { + # Reclaim inbound rows abandoned past the 7-day retention window + # (kept until then for inspection/replay). Daily; housekeeping, so + # the default queue like the other GC sweeps. + "task": "core.mda.inbound_tasks.purge_abandoned_inbound_messages_task", + "schedule": 86400.0, # daily + "options": {"queue": "default"}, + }, "process-pending-reindex": { "task": "core.services.search.tasks.process_pending_reindex_task", "schedule": settings.SEARCH_REINDEX_TASKS_INTERVAL, diff --git a/src/frontend/public/locales/common/en-US.json b/src/frontend/public/locales/common/en-US.json index edcb2e1a7..e05e231ef 100755 --- a/src/frontend/public/locales/common/en-US.json +++ b/src/frontend/public/locales/common/en-US.json @@ -176,7 +176,6 @@ "Address shared between {{count}} members_other": "Address shared between {{count}} members", "Addresses": "Addresses", "After creating the widget, you will receive the installation code to add to your website.": "After creating the widget, you will receive the installation code to add to your website.", - "After spam check (recommended)": "After spam check (recommended)", "All messages": "All messages", "All settings": "All settings", "All users with access to the mailbox \"{{mailboxName}}\" will no longer see this thread.": "All users with access to the mailbox \"{{mailboxName}}\" will no longer see this thread.", @@ -239,12 +238,8 @@ "Back to your inbox": "Back to your inbox", "bcc": "bcc", "BCC: ": "BCC: ", - "Before spam check": "Before spam check", "Blind copy: ": "Blind copy: ", - "Blocking — let this endpoint shape what happens to the message": "Blocking — let this endpoint shape what happens to the message", - "Blocking lets the receiver act on this single message": "Blocking lets the receiver act on this single message", "What we post in the request body.": "What we post in the request body.", - "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.": "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.", "Calendar invite": "Calendar invite", "Calendar service unavailable": "Calendar service unavailable", "Cancel": "Cancel", @@ -600,7 +595,6 @@ "No thread could be starred.": "No thread could be starred.", "No threads": "No threads", "No threads match the active filters": "No threads match the active filters", - "Non-blocking (default) is the safe choice:": "Non-blocking (default) is the safe choice:", "OK": "OK", "Older": "Older", "On going": "On going", @@ -851,15 +845,12 @@ "Variables": "Variables", "View full documentation": "View full documentation", "Visit the Help center": "Visit the Help center", - "we POST and ignore the response. Receivers cannot affect the message.": "we POST and ignore the response. Receivers cannot affect the message.", "Webhook": "Webhook", "Webhook API key": "Webhook API key", "Webhook signing secret": "Webhook signing secret", "Website Widget": "Website Widget", "Wednesday": "Wednesday", "Weekly": "Weekly", - "When to fire": "When to fire", - "Whether the webhook fires before or after the message is checked for spam.": "Whether the webhook fires before or after the message is checked for spam.", "While the auto-reply is disabled, it will not be sent.": "While the auto-reply is disabled, it will not be sent.", "While the signature is disabled, it will not be available to the users.": "While the signature is disabled, it will not be available to the users.", "Widget": "Widget", diff --git a/src/frontend/public/locales/common/fr-FR.json b/src/frontend/public/locales/common/fr-FR.json index 99a5b8a2e..601acfe89 100755 --- a/src/frontend/public/locales/common/fr-FR.json +++ b/src/frontend/public/locales/common/fr-FR.json @@ -246,7 +246,6 @@ "Address shared between {{count}} members_other": "Adresse partagée entre {{count}} membres", "Addresses": "Adresses", "After creating the widget, you will receive the installation code to add to your website.": "Après avoir créé le widget, vous recevrez le code d'installation à ajouter à votre site web.", - "After spam check (recommended)": "Après le contrôle anti-spam (recommandé)", "All messages": "Tous les messages", "All settings": "Tous les paramètres", "All users with access to the mailbox \"{{mailboxName}}\" will no longer see this thread.": "Tous les utilisateurs avec un accès à la boîte \"{{mailboxName}}\" ne verront plus ce thread.", @@ -311,12 +310,8 @@ "Back to your inbox": "Retour à votre messagerie", "bcc": "cci", "BCC: ": "CCI : ", - "Before spam check": "Avant le contrôle anti-spam", "Blind copy: ": "Copie cachée : ", - "Blocking — let this endpoint shape what happens to the message": "Bloquant — laissez ce point de terminaison déterminer le sort du message", - "Blocking lets the receiver act on this single message": "Le mode bloquant permet au destinataire d'agir sur ce message précis", "What we post in the request body.": "Ce qui est envoyé dans le corps de la requête.", - "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.": "en renvoyant un corps JSON. Actions disponibles (toutes limitées au message en cours de réception) : abandonner / réessayer la distribution, remplacer le verdict anti-spam, ajouter des libellés, assigner des utilisateurs par e-mail, marquer comme suivi / lu / supprimé / archivé, désactiver la réponse automatique, ajouter un commentaire interne au fil de discussion et créer un brouillon de réponse à partir d'un modèle. N'utilisez le mode bloquant qu'avec des destinataires de confiance.", "Calendar invite": "Invitation calendrier", "Calendar service unavailable": "Service d'agenda indisponible", "Cancel": "Annuler", @@ -682,7 +677,6 @@ "No thread could be starred.": "Aucune conversation n'a pu être suivie.", "No threads": "Aucune conversation", "No threads match the active filters": "Aucune conversation ne correspond aux filtres actifs", - "Non-blocking (default) is the safe choice:": "Le mode non bloquant (par défaut) est le choix sûr :", "OK": "OK", "Older": "Plus ancien", "On going": "En cours", @@ -938,15 +932,12 @@ "Variables": "Variables", "View full documentation": "Voir la documentation complète", "Visit the Help center": "Consulter le centre d'aide", - "we POST and ignore the response. Receivers cannot affect the message.": "nous envoyons un POST et ignorons la réponse. Les destinataires ne peuvent pas agir sur le message.", "Webhook": "Webhook", "Webhook API key": "Clé API du Webhook", "Webhook signing secret": "Secret de signature du Webhook", "Website Widget": "Widget de site web", "Wednesday": "Mercredi", "Weekly": "Hebdomadaire", - "When to fire": "Quand déclencher", - "Whether the webhook fires before or after the message is checked for spam.": "Indique si le Webhook se déclenche avant ou après le contrôle anti-spam du message.", "While the auto-reply is disabled, it will not be sent.": "Tant que la réponse automatique est désactivée, elle ne sera pas envoyée.", "While the signature is disabled, it will not be available to the users.": "Tant que la signature est désactivée, elle ne sera pas disponible pour les utilisateurs.", "Widget": "Widget", diff --git a/src/frontend/public/locales/common/nl-NL.json b/src/frontend/public/locales/common/nl-NL.json index b66b6b5c6..aa90449b7 100644 --- a/src/frontend/public/locales/common/nl-NL.json +++ b/src/frontend/public/locales/common/nl-NL.json @@ -1,12 +1,7 @@ { - "After spam check (recommended)": "Na spamcontrole (aanbevolen)", "API key in header — for receivers that can only check a static header value": "API-sleutel in header — voor ontvangers die alleen een statische headerwaarde kunnen controleren", "Authentication": "Authenticatie", - "Before spam check": "Voor spamcontrole", - "Blocking lets the receiver act on this single message": "Blokkerend laat de ontvanger ingrijpen op dit ene bericht", - "Blocking — let this endpoint shape what happens to the message": "Blokkerend — laat dit endpoint bepalen wat er met het bericht gebeurt", "What we post in the request body.": "Wat we in de request-body verzenden.", - "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.": "door een JSON-body terug te sturen. Beschikbare acties (allemaal beperkt tot het ontvangen bericht): de bezorging weigeren / opnieuw proberen, het spamoordeel overschrijven, labels toevoegen, gebruikers toewijzen op e-mailadres, markeren als met ster / gelezen / verwijderd / gearchiveerd, het automatisch antwoord onderdrukken, een interne opmerking aan het gesprek toevoegen en een conceptantwoord vanuit een sjabloon maken. Gebruik blokkerend alleen bij ontvangers die je vertrouwt.", "Create a Webhook": "Een Webhook maken", "Done": "Klaar", "Data will be POSTed to this URL in the format selected below.": "De gegevens worden via POST naar deze URL verzonden in het hieronder geselecteerde formaat.", @@ -16,7 +11,6 @@ "How the receiver authenticates our requests. The credential is shown once at creation.": "Hoe de ontvanger onze verzoeken authenticeert. De toegangsgegevens worden eenmalig getoond bij het aanmaken.", "JMAP Email (full message, RFC 8621)": "JMAP Email (volledig bericht, RFC 8621)", "JMAP Email (metadata only, no body)": "JMAP Email (alleen metadata, geen body)", - "Non-blocking (default) is the safe choice:": "Niet-blokkerend (standaard) is de veilige keuze:", "Outbound Webhook": "Uitgaande webhook", "Payload format": "Payloadformaat", "Raw .eml (message/rfc822)": "Onbewerkte .eml (message/rfc822)", @@ -26,12 +20,9 @@ "URL": "URL", "URL is required.": "URL is vereist.", "URL must start with http:// or https://": "URL moet beginnen met http:// of https://", - "we POST and ignore the response. Receivers cannot affect the message.": "verzenden we via POST en negeren we het antwoord. Ontvangers kunnen het bericht niet beïnvloeden.", "Webhook": "Webhook", "Webhook API key": "Webhook API-sleutel", "Webhook signing secret": "Webhook-ondertekeningsgeheim", - "When to fire": "Wanneer activeren", - "Whether the webhook fires before or after the message is checked for spam.": "Of de webhook wordt geactiveerd voordat of nadat het bericht op spam is gecontroleerd.", "{{count}} attachments_one": "{{count}} bijlage", "{{count}} attachments_other": "{{count}} bijlagen", "{{count}} days ago_one": "{{count}} dag geleden", diff --git a/src/frontend/public/locales/common/ru-RU.json b/src/frontend/public/locales/common/ru-RU.json index 3061153f9..20c88dc3a 100644 --- a/src/frontend/public/locales/common/ru-RU.json +++ b/src/frontend/public/locales/common/ru-RU.json @@ -829,14 +829,9 @@ "You were unassigned": "Вы исключены из числа участников", "Your email...": "Ваш e-mail...", "Your session has expired. Please log in again.": "Время сеанса истекло. Пожалуйста, войдите снова.", - "After spam check (recommended)": "После проверки на спам (рекомендуется)", "API key in header — for receivers that can only check a static header value": "API-ключ в заголовке — для получателей, которые могут проверять только статическое значение заголовка", "Authentication": "Аутентификация", - "Before spam check": "До проверки на спам", - "Blocking — let this endpoint shape what happens to the message": "Блокирующий — позволить этому эндпоинту влиять на судьбу сообщения", - "Blocking lets the receiver act on this single message": "Блокирующий режим позволяет получателю воздействовать на это конкретное сообщение", "What we post in the request body.": "Что мы отправляем в теле запроса.", - "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.": "вернув тело JSON. Доступные действия (все применяются только к получаемому сообщению): отклонить / повторить доставку, переопределить вердикт о спаме, добавить метки, назначить пользователей по email, пометить как избранное / прочитанное / удалённое / архивированное, отключить автоответ, добавить внутренний комментарий к цепочке и создать черновик ответа из шаблона. Используйте блокирующий режим только с доверенными получателями.", "Create a Webhook": "Создать вебхук", "Done": "Готово", "Data will be POSTed to this URL in the format selected below.": "Данные будут отправлены POST-запросом на этот URL в выбранном ниже формате.", @@ -846,7 +841,6 @@ "How the receiver authenticates our requests. The credential is shown once at creation.": "Как получатель аутентифицирует наши запросы. Учётные данные показываются один раз при создании.", "JMAP Email (full message, RFC 8621)": "JMAP Email (полное сообщение, RFC 8621)", "JMAP Email (metadata only, no body)": "JMAP Email (только метаданные, без тела)", - "Non-blocking (default) is the safe choice:": "Неблокирующий (по умолчанию) — безопасный выбор:", "Outbound Webhook": "Исходящий вебхук", "Payload format": "Формат полезной нагрузки", "Raw .eml (message/rfc822)": "Исходный .eml (message/rfc822)", @@ -856,12 +850,9 @@ "URL": "URL", "URL is required.": "URL обязателен.", "URL must start with http:// or https://": "URL должен начинаться с http:// или https://", - "we POST and ignore the response. Receivers cannot affect the message.": "мы отправляем POST-запрос и игнорируем ответ. Получатели не могут повлиять на сообщение.", "Webhook": "Вебхук", "Webhook API key": "API-ключ вебхука", "Webhook signing secret": "Секрет подписи вебхука", - "When to fire": "Когда срабатывать", - "Whether the webhook fires before or after the message is checked for spam.": "Срабатывает ли вебхук до или после проверки сообщения на спам.", "URL must start with https://": "URL должен начинаться с https://", "Regenerate credential": "Сгенерировать новые учётные данные", "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Генерация новых учётных данных немедленно делает старые недействительными. Получателя необходимо обновить новым значением, прежде чем он снова сможет проверять вебхуки.", diff --git a/src/frontend/public/locales/common/uk-UA.json b/src/frontend/public/locales/common/uk-UA.json index 51ff7a23b..c370cd33f 100644 --- a/src/frontend/public/locales/common/uk-UA.json +++ b/src/frontend/public/locales/common/uk-UA.json @@ -829,14 +829,9 @@ "You were unassigned": "Ви виключені з учасників", "Your email...": "Ваша ел. адреса...", "Your session has expired. Please log in again.": "Ваш сеанс закінчився. Будь ласка, увійдіть знову.", - "After spam check (recommended)": "Після перевірки на спам (рекомендовано)", "API key in header — for receivers that can only check a static header value": "API-ключ у заголовку — для отримувачів, які можуть перевіряти лише статичне значення заголовка", "Authentication": "Автентифікація", - "Before spam check": "До перевірки на спам", - "Blocking — let this endpoint shape what happens to the message": "Блокувальний — дозволити цьому ендпоінту впливати на долю повідомлення", - "Blocking lets the receiver act on this single message": "Блокувальний режим дозволяє отримувачу впливати на це конкретне повідомлення", "What we post in the request body.": "Що ми надсилаємо в тілі запиту.", - "by returning a JSON body. Available actions (all scoped to the message being received): drop / retry the delivery, override the spam verdict, attach labels, assign users by email, mark starred / read / trashed / archived, suppress the autoreply, add an internal comment to the thread, and create a draft reply from a template. Use blocking only with receivers you trust.": "повернувши тіло JSON. Доступні дії (усі застосовуються лише до отримуваного повідомлення): відхилити / повторити доставку, перевизначити вердикт щодо спаму, додати мітки, призначити користувачів за email, позначити як обране / прочитане / видалене / архівоване, вимкнути автовідповідь, додати внутрішній коментар до ланцюжка та створити чернетку відповіді з шаблону. Використовуйте блокувальний режим лише з довіреними отримувачами.", "Create a Webhook": "Створити вебхук", "Done": "Готово", "Data will be POSTed to this URL in the format selected below.": "Дані будуть надіслані POST-запитом на цей URL у вибраному нижче форматі.", @@ -846,7 +841,6 @@ "How the receiver authenticates our requests. The credential is shown once at creation.": "Як отримувач автентифікує наші запити. Облікові дані показуються один раз під час створення.", "JMAP Email (full message, RFC 8621)": "JMAP Email (повне повідомлення, RFC 8621)", "JMAP Email (metadata only, no body)": "JMAP Email (лише метадані, без тіла)", - "Non-blocking (default) is the safe choice:": "Неблокувальний (за замовчуванням) — безпечний вибір:", "Outbound Webhook": "Вихідний вебхук", "Payload format": "Формат корисного навантаження", "Raw .eml (message/rfc822)": "Необроблений .eml (message/rfc822)", @@ -856,12 +850,9 @@ "URL": "URL", "URL is required.": "URL є обов'язковим.", "URL must start with http:// or https://": "URL має починатися з http:// або https://", - "we POST and ignore the response. Receivers cannot affect the message.": "ми надсилаємо POST-запит та ігноруємо відповідь. Отримувачі не можуть вплинути на повідомлення.", "Webhook": "Вебхук", "Webhook API key": "API-ключ вебхука", "Webhook signing secret": "Секрет підпису вебхука", - "When to fire": "Коли спрацьовувати", - "Whether the webhook fires before or after the message is checked for spam.": "Чи спрацьовує вебхук до або після перевірки повідомлення на спам.", "URL must start with https://": "URL має починатися з https://", "Regenerate credential": "Згенерувати нові облікові дані", "Regenerating the credential invalidates the old one immediately. The receiver must be updated with the new value before it can verify webhooks again.": "Генерація нових облікових даних негайно робить старі недійсними. Отримувача потрібно оновити новим значенням, перш ніж він знову зможе перевіряти вебхуки.", From a750e6e0ef56dca14c414d085efc87168d4f75f0 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Wed, 1 Jul 2026 12:44:06 +0200 Subject: [PATCH 18/21] refactor spam processing, add postmark --- docs/webhooks.md | 117 ++-- src/backend/core/api/openapi.json | 10 +- src/backend/core/api/serializers.py | 25 +- src/backend/core/api/viewsets/channel.py | 19 +- src/backend/core/api/viewsets/inbound/mta.py | 57 +- .../core/api/viewsets/inbound/widget.py | 13 +- src/backend/core/enums.py | 8 +- .../management/commands/backfill_postmark.py | 128 +++++ src/backend/core/mda/autoreply.py | 36 +- src/backend/core/mda/dispatch_webhooks.py | 194 ++++--- src/backend/core/mda/inbound.py | 45 +- src/backend/core/mda/inbound_create.py | 41 +- src/backend/core/mda/inbound_pipeline.py | 209 ++++--- src/backend/core/mda/inbound_tasks.py | 80 ++- src/backend/core/mda/outbound.py | 6 +- src/backend/core/mda/webhook_payload.py | 24 +- .../0032_inboundmessage_blob_envelope.py | 66 +++ ...lob_inboundmessage_is_internal_and_more.py | 38 -- src/backend/core/models.py | 144 +++-- .../tests/api/test_channel_scope_level.py | 8 +- src/backend/core/tests/api/test_channels.py | 156 +++++- .../core/tests/api/test_inbound_mta.py | 100 +++- .../tests/commands/test_backfill_postmark.py | 72 +++ src/backend/core/tests/mda/test_autoreply.py | 45 +- .../core/tests/mda/test_dispatch_webhooks.py | 523 ++++++++++-------- src/backend/core/tests/mda/test_inbound.py | 51 +- .../core/tests/mda/test_inbound_auth.py | 127 ++--- .../core/tests/mda/test_inbound_e2e.py | 12 +- .../tests/mda/test_inbound_spoofed_sender.py | 11 +- .../core/tests/mda/test_spam_processing.py | 162 ++++-- src/backend/core/tests/models/test_blob.py | 16 +- .../models/test_message_get_parsed_data.py | 83 +++ src/backend/messages/settings.py | 16 + src/frontend/public/locales/common/en-US.json | 2 + src/frontend/public/locales/common/fr-FR.json | 2 + src/frontend/public/locales/common/nl-NL.json | 2 + src/frontend/public/locales/common/ru-RU.json | 2 + src/frontend/public/locales/common/uk-UA.json | 2 + .../api/gen/models/channel_create_response.ts | 2 - .../gen/models/regenerated_secret_response.ts | 2 +- .../webhook-integration-form.tsx | 7 +- .../thread-message/thread-message-header.tsx | 20 +- 42 files changed, 1858 insertions(+), 825 deletions(-) create mode 100644 src/backend/core/management/commands/backfill_postmark.py create mode 100644 src/backend/core/migrations/0032_inboundmessage_blob_envelope.py delete mode 100644 src/backend/core/migrations/0032_inboundmessage_blob_inboundmessage_is_internal_and_more.py create mode 100644 src/backend/core/tests/commands/test_backfill_postmark.py diff --git a/docs/webhooks.md b/docs/webhooks.md index b8483f418..700713949 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -22,15 +22,16 @@ so invalid combinations can't be expressed: | `message.delivering` | After the spam verdict, while delivery is in flight | yes (sync) | known | | `message.delivered` | After the message has landed in the mailbox | no (async) | final | -This flat list replaces the older `(events, phase, blocking)` triple. -Future lifecycle events (e.g. `message.sent`) extend it with new +Future lifecycle events (e.g. `message.sent`) are added as new `trigger` values. **`message.inbound`** and **`message.delivering`** are *synchronous*: they run **inline** on the pipeline worker and get to shape delivery — drop the -message, ask to be retried later, or return a small JSON body that -overrides the spam verdict and/or attaches labels to the resulting thread -(see [Response contract](#response-contract)). +message, or return a small JSON body that overrides the spam verdict +and/or attaches labels to the resulting thread (see +[Response contract](#response-contract)). They can't *ask* to be retried +through the body: a webhook that needs the message redelivered returns a +non-2xx status, which is treated as a transient failure → RETRY. **`message.delivered`** is *asynchronous* — fire-and-forget; failures are logged and the pipeline continues unchanged. Because it can't influence @@ -76,7 +77,7 @@ A webhook channel stores its configuration in `Channel.settings` | Key | Type | Default | Description | | ------------- | -------- | -------------- | --------------------------------------------------------------------------- | -| `url` | string | **required** | `https://` endpoint, validated by the SSRF guard at each call. `http://` is accepted only when Django `DEBUG` is on (the local-dev escape hatch). | +| `url` | string | **required** | `https://` endpoint. **Rejected at create/update** if it resolves to an internal address or doesn't resolve, and re-validated by the SSRF guard (with IP pinning) at each call. `http://` is accepted only when Django `DEBUG` is on (the local-dev escape hatch). | | `trigger` | string | **required** | `message.inbound`, `message.delivering`, or `message.delivered` (see [When does it fire?](#when-does-it-fire)). | | `format` | string | `eml` | `eml`, `jmap`, or `jmap_metadata` (see [Payload formats](#payload-formats)). | | `auth_method` | string | **required** | `jwt` or `api_key` (see [Authentication](#authentication)). | @@ -109,25 +110,26 @@ returned exactly once at create time and rotatable via setting picks how that root is presented on each POST. The root itself never travels on the wire. -| `auth_method` | Headers sent | Wire value | Receiver verifies | -| ------------- | ----------------------------------------------------------------------------------------------------------- | --------------------------------------------------------- | ------------------------------------------------------------------------------ | -| `jwt` | `X-StMsg-Webhook-Timestamp`, `X-StMsg-Webhook-Signature: v1=`, `Authorization: Bearer ` | HMAC sig + JWT, both **keyed** by the root | HMAC-SHA256 of `f"{timestamp}.{body}"` with the root, **or** the HS256 JWT. | -| `api_key` | `X-StMsg-Api-Key: ` | `whk_` + `HMAC-SHA256(root, "messages.webhook.api_key.v1").hex()` | Constant-time compare of the header against the receiver's stored copy. | +| `auth_method` | Headers sent | Wire value | Receiver verifies | +| ------------- | -------------------------------------------------- | ---------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | +| `jwt` | `Authorization: Bearer ` | HS256 JWT **keyed** by the root | Verify the JWT with the root using any JWT library; it binds the exact body via the `body_sha256` claim and expires (`exp`, 5 min) with a `jti` nonce. | +| `api_key` | `Authorization: Bearer ` | `whk_` + `HMAC-SHA256(root, "messages.webhook.api_key.v1").hex()` | Constant-time compare of the Bearer token against the receiver's stored copy (an opaque static key, not a JWT). | A channel sends **only** the headers for its configured method — the unused presentation never rides on the wire, so it can't leak through receiver-side proxies or debug panes. The API-key value is a **one-way derivation** of the root, so a receiver-side leak of the API -key reveals nothing about the root: HMAC/JWT verification on other -receivers stays unforgeable. +key reveals nothing about the root: JWT verification (and the API-key +derivation) on other receivers stays unforgeable. #### Picking a method - `jwt` — best when the receiver controls a server (n8n, your own - Lambda, a Flask/Express app, Cloudflare Worker). Body integrity is - proven by the HMAC; JWT lets receivers verify with stock libraries. -- `api_key` — for low-code receivers that can only check a header - (Zapier "API key in header" trigger, IFTTT, a Zap webhook step). + Lambda, a Flask/Express app, Cloudflare Worker). Verify with stock JWT + libraries; the token binds the exact body (`body_sha256`) and expires. +- `api_key` — for low-code receivers that can only do a static + header-equals-value check (Zapier, IFTTT, a Zap webhook step): they just + compare `Authorization` against the stored `Bearer ` value. #### Switching methods on an existing channel @@ -185,27 +187,42 @@ The three possible **decisions** are about the inbound email itself: delivery, so "the `InboundMessage` is deleted" is not what makes DROP special; **the email never landing** is. * **RETRY** — keep the email queued and re-fire the webhook on the next - 5-minute sweep (bounded by the **quarantine window** below). + 5-minute sweep (bounded by the **deferral window** below). **A webhook error never drops the email.** The *only* way a blocking webhook discards a message is by **explicitly** returning `{"action": "drop"}` with an HTTP `2xx` (see -[JSON action body](#json-action-body)). Every transport- or status-level -failure is held for RETRY — a receiver bug that answers `4xx`, an -endpoint that is down, or a config error on our side must never cost the -user their mail. - -| Outcome | Decision | What happens | -| ------------------------------------- | ---------- | ------------------------------------------------------------------------------- | -| HTTP `2xx`, empty / non-JSON body | CONTINUE | Email delivered normally. | -| HTTP `2xx` + `{"action": "drop"}` | DROP | The **only** path to DROP — the receiver deliberately discards the email. | -| HTTP `2xx` + other JSON action body | see below | Body parsed for `action` / `is_spam` / `add_labels`; default is CONTINUE. | -| Any non-2xx (`4xx`, `5xx`, `3xx`) | RETRY | The email stays queued; the 5-min sweep re-fires the webhook. | -| Connection error, timeout, DNS, etc. | RETRY | Transient. | -| SSRF rejection | RETRY | Config error on our side (bad URL) — held until fixed, never dropped. | -| Missing signing secret (misconfig) | RETRY | Can't sign the POST — held until the channel is fixed, never dropped. | - -`RETRY` is bounded by a **quarantine window** so a persistently-failing +[JSON action body](#json-action-body)). Other failures split two ways: + +* A **transient** failure — a `4xx`/`5xx`, a timeout, a connection error, + a receiver that is temporarily down — is held for **RETRY** and re-fired + by the sweep until it recovers (bounded by the deferral window). +* A **config** failure that waiting cannot fix — the URL is refused by the + [SSRF guard](#security-notes) (resolves to an internal address, or won't + resolve), or the channel is missing its secret / url / auth_method — + delivers the mail **past** the broken webhook (**CONTINUE**) and logs an + `ERROR` so an admin fixes or disables the channel. We never POST the + internal target (the guard already blocked it); we just don't stall a + whole scope's inbound — instance-wide for a GLOBAL blocking webhook — for + 48h on a config that has never worked. Create/update validation rejects + internal/unresolvable URLs up front, so at dispatch this is only a DNS + rebinding or a hand-edited row. + +Either way the user never loses mail. Note the config path is **fail-open** +for that one webhook: a hard spam/security gate is bypassed during the +failure (the rest of the pipeline, incl. rspamd, still runs) — a conscious +trade against stalling all inbound; the `ERROR` log is your signal to fix it. + +| Outcome | Decision | What happens | +| ----------------------------------------------- | --------------- | ---------------------------------------------------------------------------- | +| HTTP `2xx`, empty / non-JSON body | CONTINUE | Email delivered normally. | +| HTTP `2xx` + `{"action": "drop"}` | DROP | The **only** path to DROP — the receiver deliberately discards the email. | +| HTTP `2xx` + other JSON action body | see below | Body parsed for `action` / `is_spam` / `add_labels`; default is CONTINUE. | +| Any non-2xx (`4xx`/`5xx`/`3xx`), timeout, conn. | RETRY | Transient — held, re-fired by the sweep, bounded by the deferral window. | +| SSRF rejection (internal / unresolvable URL) | CONTINUE + alert | Config error retry can't fix; deliver past it, `log.error`. URL never dialed.| +| Missing secret / url / auth_method (misconfig) | CONTINUE + alert | Non-DRF misconfig retry can't fix; deliver past it, `log.error`. | + +`RETRY` is bounded by a **deferral window** so a persistently-failing processing step can neither pin a row forever nor lose mail. If a step is still failing **48 hours** after the message arrived, we stop holding and **deliver the message anyway** — landed in the inbox (`is_spam=False` @@ -214,11 +231,14 @@ marker. The web UI reads that marker and shows a prominent warning banner (the same surface as the unverified-sender warning), so the recipient knows the message bypassed a processing step and can review it with caution. Nothing is ever silently dropped; if the step recovers within -the window, the next sweep delivers normally with no marker. +the window, the next sweep delivers normally with no marker. (That +`X-StMsg-Processing-Failed` marker rides in the stored MIME as an +`X-StMsg-*` header; sender-supplied `X-StMsg-*` headers are stripped at +ingest, so a malicious **sender** can't forge it to fake a bypassed check.) The mechanism is generic: a blocking webhook is the trigger today, but any step that returns `RETRY` (e.g. a persistently-unreachable spam -checker) is quarantined the same way. +checker) is deferred the same way. > **Delivery is at-least-once — make your receiver idempotent.** A > `RETRY` re-fires the webhook on the next sweep, and a worker crash @@ -228,10 +248,6 @@ checker) is quarantined the same way. > message more than once. Key on the `Message-ID` (or the > `X-StMsg-*` envelope headers) and treat repeats as no-ops. -(The marker rides in the stored MIME as an `X-StMsg-*` header. Sender- -supplied `X-StMsg-*` headers are stripped at ingest, so receivers can't -forge it.) - #### JSON action body When a blocking webhook returns `HTTP 2xx` with `Content-Type: @@ -258,11 +274,11 @@ optional; unknown keys are ignored. | Key | Type | Meaning | | ---------------- | -------------- | ---------------------------------------------------------------------------------------------------------------- | -| `action` | `"drop"` | `"drop"` drops the message at this phase. Any other value (or omission) is treated as accept. Case-insensitive. There is no body-driven `"retry"`: a 2xx is a successful response. If you need the message redelivered later, return a non-2xx status (e.g. `429`/`503`) — it is held for retry, bounded by the 48h quarantine window. | +| `action` | `"drop"` | `"drop"` drops the message at this phase. Any other value (or omission) is treated as accept. Case-insensitive. There is no body-driven `"retry"`: a 2xx is a successful response. If you need the message redelivered later, return a non-2xx status (e.g. `429`/`503`) — it is held for retry, bounded by the 48h deferral window. | | `is_spam` | bool | Override the spam verdict. Acts as a full antispam: for a `message.inbound` webhook this **skips rspamd**. | | `add_labels` | string[] | UUIDs of `Label` rows in the destination mailbox to attach to the thread once it is created. | | `assign_to` | string[] | OIDC emails of users to assign to the resulting thread (one `ThreadEvent ASSIGN` per webhook, channel-attributed). | -| `mark_starred` | bool (true only) | Star the resulting thread for the destination mailbox. | +| `mark_starred` | bool (true only) | Star the resulting thread for the destination mailbox. | | `mark_read` | bool (true only) | Mark the resulting thread as read for the destination mailbox. | | `mark_trashed` | bool (true only) | Land the message with `is_trashed=true`. (Distinct from `action: "drop"` — the row stays, just hidden.) | | `mark_archived` | bool (true only) | Land the message with `is_archived=true`. | @@ -316,6 +332,10 @@ Supported types: | ------ | ------------------ | ----------------------------------------------------------------------------------- | | `"im"` | `content` (string) | Persists as an internal-message ThreadEvent — the same surface humans post into. | +The `im` `content` is stored on every inbound, so it is capped at +**32 KiB** (UTF-8) — longer content is truncated. At most **20** +`add_event` entries are processed per response; extras are dropped. + Unknown types are silently skipped at the classifier — the contract stays forward-compatible so receivers can begin emitting new types (e.g. `"iframe"`) before the server learns them, with no churn for @@ -449,11 +469,16 @@ metadata lives in the headers above. #### Fields omitted on purpose JMAP defines a few `Email` properties that only make sense once the -message is **stored** in a JMAP server. The webhook fires *before* the -`Message` row exists (and may abort delivery in the blocking case), so -these are intentionally absent: - -* `id`, `blobId`, `threadId`, `mailboxIds`, `keywords`. +message is **stored**: + +* `id` and `threadId` are included **only** on `message.delivered` — it + fires *after* the `Message` row exists, so we stamp them (the same values + also ride in the `X-StMsg-Message-Id` / `X-StMsg-Thread-Id` headers). The + blocking triggers (`message.inbound` / `message.delivering`) fire *before* + the row exists — there is no id yet, so both are absent. +* `blobId`, `mailboxIds`, `keywords` are absent on **every** trigger: + `blobId` would imply a JMAP blob-download endpoint we don't expose, and + `mailboxIds` / `keywords` need a folder/flag mapping we haven't designed. Attachment **bytes** are also intentionally omitted: JMAP keeps attachment content behind a `blobId` and a separate fetch, which has no diff --git a/src/backend/core/api/openapi.json b/src/backend/core/api/openapi.json index e2da56ec1..32f2b9355 100644 --- a/src/backend/core/api/openapi.json +++ b/src/backend/core/api/openapi.json @@ -2982,7 +2982,7 @@ } } }, - "description": "Channel created successfully. The response carries the one-time plaintext credentials (api_key / secret / password) which are never returned again." + "description": "Channel created successfully. The response carries the one-time plaintext credential (api_key / secret) which is never returned again." }, "400": { "description": "Invalid input data" @@ -6944,7 +6944,7 @@ } } }, - "description": "Channel created successfully. The response carries the one-time plaintext credentials (api_key / secret / password) which are never returned again." + "description": "Channel created successfully. The response carries the one-time plaintext credential (api_key / secret) which is never returned again." }, "400": { "description": "Invalid input data" @@ -7582,10 +7582,6 @@ "type": "string", "description": "Plaintext API key — api_key channels and webhook channels with auth_method=api_key." }, - "password": { - "type": "string", - "description": "Plaintext password, when the channel type mints one." - }, "secret": { "type": "string", "description": "webhook channels with auth_method=jwt — the HMAC/JWT signing secret." @@ -9812,7 +9808,7 @@ }, "api_key": { "type": "string", - "description": "Present for ``api_key`` channels and webhook channels with ``auth_method='api_key'`` — the plaintext API key presented in a request header (X-API-Key / X-StMsg-Api-Key). Returned ONCE; for api_key webhooks it changes whenever the root rotates." + "description": "Present for ``api_key`` channels and webhook channels with ``auth_method='api_key'`` — the plaintext API key. api_key channels send it as ``X-API-Key`` on inbound API calls; api_key webhooks present it as ``Authorization: Bearer``. Returned ONCE; for api_key webhooks it changes whenever the root rotates." }, "secret": { "type": "string", diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index 4c422aebd..82ccd995b 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -22,6 +22,7 @@ from core.mda.inline_images import extract_inline_images_html from core.services.blob_gc import schedule_for_gc from core.services.identity import keycloak as keycloak_service +from core.services.ssrf import SSRFValidationError, validate_hostname class CreateOnlyFieldsMixin: @@ -2259,7 +2260,7 @@ def _validate_webhook_settings(self, attrs): Runs on CREATE and on every PATCH/PUT touching ``type`` or ``settings``. Same airtight rule as ``_validate_api_key_scopes``: a settings-only PATCH on an existing webhook channel must hit the - URL/events validators, otherwise a mailbox admin could PATCH + URL/trigger validators, otherwise a mailbox admin could PATCH ``{"settings": {"url": "javascript:..."}}`` past the create-time check. """ @@ -2300,6 +2301,23 @@ def _validate_webhook_settings(self, attrs): {"settings": "webhook settings.url must use https://."} ) + # Reject internal / unresolvable webhook URLs up front, so a misconfig + # is a clear 400 at config time instead of silently-stalled mail later. + # This is a fail-fast UX layer, NOT the security boundary: a hostname + # can resolve public now and be rebound to internal at dispatch, so + # ``SSRFSafeSession`` re-validates with IP pinning on every POST + # regardless. ``validate_hostname`` does a DNS lookup (a config + # endpoint, not a hot path) and rejects IP literals, private/internal + # targets, and names that don't resolve. Skipped under DEBUG, the + # local-dev escape hatch where operators point at local receivers. + if not settings.DEBUG: + try: + validate_hostname(host) + except SSRFValidationError as exc: + raise serializers.ValidationError( + {"settings": f"webhook settings.url is not allowed: {exc}"} + ) from exc + # A single ``trigger`` lifecycle event describes the webhook — it # encodes the event, the pipeline phase, and whether it blocks # delivery, so invalid combinations (e.g. non-blocking before-spam) @@ -2435,10 +2453,6 @@ class ChannelCreateResponseSerializer(ChannelSerializer): "with auth_method=api_key." ), ) - password = serializers.CharField( - required=False, - help_text="Plaintext password, when the channel type mints one.", - ) secret = serializers.CharField( required=False, help_text="webhook channels with auth_method=jwt — the HMAC/JWT signing secret.", @@ -2447,7 +2461,6 @@ class ChannelCreateResponseSerializer(ChannelSerializer): class Meta(ChannelSerializer.Meta): fields = ChannelSerializer.Meta.fields + [ "api_key", - "password", "secret", ] diff --git a/src/backend/core/api/viewsets/channel.py b/src/backend/core/api/viewsets/channel.py index 0590a3ac7..669aae272 100644 --- a/src/backend/core/api/viewsets/channel.py +++ b/src/backend/core/api/viewsets/channel.py @@ -118,8 +118,8 @@ def get_save_kwargs(self): response=serializers.ChannelCreateResponseSerializer, description=( "Channel created successfully. The response carries the " - "one-time plaintext credentials (api_key / secret / " - "password) which are never returned again." + "one-time plaintext credential (api_key / secret) which " + "is never returned again." ), ), 400: OpenApiResponse(description="Invalid input data"), @@ -143,10 +143,8 @@ def create(self, request, *args, **kwargs): instance = serializer.save(**self.get_save_kwargs()) data = serializer.data - # Surface plaintext credentials exactly once on creation — - # subsequent GETs never return them. - if password := getattr(instance, "_generated_password", None): - data["password"] = password + # Surface the freshly-minted plaintext credential exactly once on + # creation — subsequent GETs never return it. _attach_credential(data, instance) return Response(data, status=status.HTTP_201_CREATED) @@ -198,10 +196,11 @@ def destroy(self, request, *args, **kwargs): "Present for ``api_key`` channels and " "webhook channels with " "``auth_method='api_key'`` — the plaintext " - "API key presented in a request header " - "(X-API-Key / X-StMsg-Api-Key). Returned " - "ONCE; for api_key webhooks it changes " - "whenever the root rotates." + "API key. api_key channels send it as " + "``X-API-Key`` on inbound API calls; api_key " + "webhooks present it as ``Authorization: " + "Bearer``. Returned ONCE; for api_key webhooks " + "it changes whenever the root rotates." ), ), "secret": drf_serializers.CharField( diff --git a/src/backend/core/api/viewsets/inbound/mta.py b/src/backend/core/api/viewsets/inbound/mta.py index 0551736d2..6a18eba4e 100644 --- a/src/backend/core/api/viewsets/inbound/mta.py +++ b/src/backend/core/api/viewsets/inbound/mta.py @@ -184,27 +184,42 @@ def deliver(self, request): mta_metadata["original_recipients"], # Log all intended recipients ) - # Drop any sender-supplied X-StMsg-* headers so only values we prepend - # further down the pipeline (sender-auth verdict, widget-referer, ...) - # are visible to storage and the frontend. - raw_data = remove_mime_headers(raw_data, prefixes=["x-stmsg-"]) + # Strip sender-supplied headers we own or authoritatively rewrite: + # - ``X-StMsg-*``: our pipeline namespace (sender-auth etc.) — must not + # be forgeable. + # - ``Return-Path``: only the delivering MTA may write it (RFC 5321 + # §4.4); we rewrite it below from the real envelope MAIL FROM, so any + # inbound copy is forged and must go first. + raw_data = remove_mime_headers( + raw_data, prefixes=["x-stmsg-"], names=["return-path"] + ) def sanitize_header(header: str) -> str: return header.replace("\r", "").replace("\n", "")[0:255] + # Bake the immutable envelope facts as standard headers at ingest, on + # top of the received bytes and BEFORE the blob is created, so they ride + # in the single stored blob: + # - ``Return-Path``: the envelope MAIL FROM (``<>`` for a null sender / + # bounce). Durable home for what the autoreply/bounce logic reads. + # - ``Received``: the SMTP trace (HELO / rDNS / IP), when the MTA + # forwarded it. Shared across recipients (one prepend), so identical + # bytes still dedup to one blob. + sender = mta_metadata.get("sender") or "" + prepend_headers = [("Return-Path", f"<{sender}>" if sender else "<>")] if "client_helo" in mta_metadata: - prepend_headers = [ + prepend_headers.append( ( "Received", f"from {mta_metadata['client_helo']} (" + f"{mta_metadata['client_hostname']} [{mta_metadata['client_address']}]);", - ), - ] + ) + ) - raw_data = ( - "\r\n".join([f"{k}: {sanitize_header(v)}" for k, v in prepend_headers]) - + "\r\n" - ).encode("utf-8") + raw_data + raw_data = ( + "\r\n".join([f"{k}: {sanitize_header(v)}" for k, v in prepend_headers]) + + "\r\n" + ).encode("utf-8") + raw_data # Parse the email message once parsed_email = parse_email(raw_data) @@ -217,6 +232,19 @@ def sanitize_header(header: str) -> str: status=status.HTTP_400_BAD_REQUEST, # Bad request as email is malformed ) + # Structured SMTP envelope carried alongside the bytes (never injected + # into them, so the blob stays byte-identical across recipients and + # dedups to one row). ``sender`` is the MAIL FROM; the client_* fields + # are the connecting SMTP peer (forwarded by the MTA). ``rcpt_to`` is + # the actual RCPT TO and is set per recipient inside the loop. + base_envelope = { + "origin": "mta", + "mail_from": mta_metadata.get("sender", ""), + "ip": mta_metadata.get("client_address", ""), + "helo": mta_metadata.get("client_helo", ""), + "hostname": mta_metadata.get("client_hostname", ""), + } + # Deliver the parsed email to each original recipient success_count = 0 failure_count = 0 @@ -225,7 +253,12 @@ def sanitize_header(header: str) -> str: for recipient in mta_metadata["original_recipients"]: try: # Call the refactored delivery function which returns True/False - delivered = deliver_inbound_message(recipient, parsed_email, raw_data) + delivered = deliver_inbound_message( + recipient, + parsed_email, + raw_data, + envelope={**base_envelope, "rcpt_to": recipient}, + ) if delivered: success_count += 1 delivery_results[recipient] = "Success" diff --git a/src/backend/core/api/viewsets/inbound/widget.py b/src/backend/core/api/viewsets/inbound/widget.py index b2acc3ce2..db696f68e 100644 --- a/src/backend/core/api/viewsets/inbound/widget.py +++ b/src/backend/core/api/viewsets/inbound/widget.py @@ -188,7 +188,12 @@ def deliver(self, request): def sanitize_header(header: str) -> str: return header.replace("\r", "").replace("\n", "")[0:1000] - prepend_headers = [("X-StMsg-Sender-Auth", "none")] + # ``Return-Path`` (envelope MAIL FROM) and the widget ``Received`` are + # immutable ingest facts → baked as headers in the one blob. The + # sender-auth "none" baseline for widget mail is set structurally in the + # pipeline (``postmark["auth"]``), not baked here. ``X-StMsg-Widget- + # Referer`` stays a header (immutable ingest provenance). + prepend_headers = [("Return-Path", f"<{sender_email}>")] source_name = "widget" if request.META.get("HTTP_REFERER"): referer = sanitize_header(request.META.get("HTTP_REFERER")) @@ -236,6 +241,12 @@ def sanitize_header(header: str) -> str: parsed_email, compose_email(parsed_email, prepend_headers=prepend_headers), channel=channel, + envelope={ + "origin": "widget", + "mail_from": sender_email, + "rcpt_to": target_email, + "ip": request.META.get("REMOTE_ADDR", ""), + }, ) if not delivered: diff --git a/src/backend/core/enums.py b/src/backend/core/enums.py index add59c126..c64bc2fbb 100644 --- a/src/backend/core/enums.py +++ b/src/backend/core/enums.py @@ -246,11 +246,11 @@ class WebhookTrigger(StrEnum): delivery is the spec in ``docs/webhooks.md``: - ``MESSAGE_INBOUND`` — the message just arrived, **before** the spam - check. Synchronous (blocking): the receiver can DROP / RETRY / - mutate it and sees no spam verdict yet. + check. Synchronous (blocking): the receiver can DROP / mutate it + and sees no spam verdict yet. - ``MESSAGE_DELIVERING`` — **after** the spam check, while delivery - is still in flight. Synchronous (blocking): can DROP / RETRY / - mutate and sees the verdict. + is still in flight. Synchronous (blocking): can DROP / mutate + and sees the verdict. - ``MESSAGE_DELIVERED`` — the message has landed in the mailbox. Asynchronous (fire-and-forget): a notification that can't influence delivery, always reflecting the final spam verdict. diff --git a/src/backend/core/management/commands/backfill_postmark.py b/src/backend/core/management/commands/backfill_postmark.py new file mode 100644 index 000000000..047494de8 --- /dev/null +++ b/src/backend/core/management/commands/backfill_postmark.py @@ -0,0 +1,128 @@ +"""Progressively backfill ``Message.postmark`` from legacy ``X-StMsg-*`` bytes. + +Messages created before ``postmark`` existed carry their sender-auth / +processing-failed verdicts as ``X-StMsg-*`` headers baked into the stored MIME. +``Message.get_stmsg_headers()`` reads both sources during the transition, so +nothing is broken meanwhile — but to eventually drop the byte-reading branch we +need those verdicts moved into the structured field. + +This command does that in bounded batches so it can be run repeatedly (e.g. from +cron) instead of one job that reads 100% of bodies at once. Each run scans up to +``--limit`` messages whose ``postmark`` is still NULL, oldest first, and sets it: +the extracted verdicts, or ``{}`` for a message that had none (which both marks +it scanned so it isn't re-read and reads back identically to NULL). + +Usage: + python manage.py backfill_postmark # one bounded run + python manage.py backfill_postmark --limit 50000 --batch-size 1000 + python manage.py backfill_postmark --before 2026-07-01 --dry-run +""" + +import datetime +import logging + +from django.core.management.base import BaseCommand +from django.utils.dateparse import parse_datetime + +from core import models + +logger = logging.getLogger(__name__) + + +def _postmark_from_stmsg(headers: dict) -> dict: + """Project the legacy ``X-StMsg-*`` header dict onto ``postmark`` keys.""" + postmark = {} + sender_auth = headers.get("sender-auth") + if sender_auth in ("none", "fail"): + postmark["auth"] = sender_auth + if headers.get("processing-failed"): + # Legacy value was the literal "true"; normalise to the new "fail". + postmark["processing"] = "fail" + return postmark + + +class Command(BaseCommand): + """Backfill Message.postmark from legacy X-StMsg-* headers, in batches.""" + + help = "Populate Message.postmark from legacy X-StMsg-* MIME headers." + + def add_arguments(self, parser): + parser.add_argument( + "--limit", + type=int, + default=10000, + help="Max messages to process this run (default: 10000).", + ) + parser.add_argument( + "--batch-size", + type=int, + default=500, + help="Rows fetched (and bodies read) per batch (default: 500).", + ) + parser.add_argument( + "--before", + type=str, + default=None, + help=( + "Only backfill messages created before this ISO date/datetime. " + "Use to target the pre-deploy backlog and skip fresh mail." + ), + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Report what would change without writing.", + ) + + def handle(self, *args, **options): + limit = options["limit"] + batch_size = options["batch_size"] + dry_run = options["dry_run"] + + qs = models.Message.objects.filter(postmark__isnull=True) + if options["before"]: + cutoff = parse_datetime(options["before"]) + if cutoff is None: + # Bare date → midnight UTC. + cutoff = datetime.datetime.fromisoformat(options["before"]).replace( + tzinfo=datetime.UTC + ) + qs = qs.filter(created_at__lt=cutoff) + + scanned = 0 + populated = 0 + errors = 0 + + while scanned < limit: + take = min(batch_size, limit - scanned) + # Processed rows leave the ``postmark__isnull=True`` set, so the next + # slice is always fresh work — no cursor/offset needed. + batch = list(qs.order_by("created_at")[:take]) + if not batch: + break + + for message in batch: + scanned += 1 + try: + postmark = _postmark_from_stmsg(message.get_stmsg_headers()) + except Exception: # pylint: disable=broad-exception-caught + # A single unreadable/corrupt blob must not abort the run. + errors += 1 + logger.exception("backfill_postmark: failed to read %s", message.id) + continue + + if postmark: + populated += 1 + if dry_run: + continue + # ``{}`` marks the row scanned (won't be re-read) and reads back + # the same as NULL through ``get_stmsg_headers``. + message.postmark = postmark + message.save(update_fields=["postmark"]) + + self.stdout.write( + self.style.SUCCESS( + f"backfill_postmark: scanned={scanned} populated={populated} " + f"errors={errors}{' (dry-run)' if dry_run else ''}" + ) + ) diff --git a/src/backend/core/mda/autoreply.py b/src/backend/core/mda/autoreply.py index 5cad2597c..d3a9d40ba 100644 --- a/src/backend/core/mda/autoreply.py +++ b/src/backend/core/mda/autoreply.py @@ -77,17 +77,25 @@ def _is_recipient_explicit(mailbox_email: str, parsed_email: JmapEmail) -> bool: return False -def _is_auto_reply_message(parsed_email: JmapEmail) -> bool: +def _is_auto_reply_message( + parsed_email: JmapEmail, envelope: Optional[dict] = None +) -> bool: """Detect whether the inbound message is itself an automatic reply. - Checks Auto-Submitted, Precedence, List-Id, X-Auto-Response-Suppress, - X-Autoreply, X-Autorespond, and Return-Path bounce indicators. + Checks the SMTP envelope MAIL FROM (bounce indicator) plus the + Auto-Submitted, Precedence, List-Id, X-Auto-Response-Suppress, + X-Autoreply, and X-Autorespond headers. """ - # Return-Path empty or <> means bounce (RFC 3834). Walk every - # occurrence so a benign duplicate can't mask a bounce indicator. - for return_path in find_headers(parsed_email, "Return-Path"): - if return_path.strip() in ("", "<>"): - return True + # A null envelope sender (MAIL FROM ``<>`` or empty) marks a bounce / + # notification we must never reply to (RFC 3834 §2). We read the + # authoritative SMTP envelope rather than a ``Return-Path`` header: the + # delivering MTA never writes one on our inbound path, and any header in + # the body is sender-forgeable. Only an explicitly-present null value + # counts — an absent ``mail_from`` key means "no envelope info supplied" + # (e.g. a caller that doesn't carry one), not "null sender". + mail_from = (envelope or {}).get("mail_from") + if mail_from is not None and mail_from.strip() in ("", "<>"): + return True # Auto-Submitted (max=1 per RFC 3834 §5). Parameters after ``;`` # (e.g. ``auto-replied; owner-email=...``) are stripped before @@ -111,6 +119,7 @@ def should_send_autoreply( mailbox: models.Mailbox, parsed_email: JmapEmail, is_spam: bool = False, + envelope: Optional[dict] = None, ) -> Optional[models.MessageTemplate]: """Determine whether we should send an autoreply and return the template. @@ -121,8 +130,8 @@ def should_send_autoreply( if is_spam: return None - # 2. Skip auto-generated messages (loop prevention) - if _is_auto_reply_message(parsed_email): + # 2. Skip auto-generated messages and bounces (loop prevention) + if _is_auto_reply_message(parsed_email, envelope): return None # 3. Self-reply prevention: skip if sender == mailbox email @@ -362,14 +371,19 @@ def try_send_autoreply( parsed_email: JmapEmail, message: models.Message, is_spam: bool = False, + envelope: Optional[dict] = None, ): """Evaluate autoreply conditions and send if appropriate. Safe to call from any delivery path (MTA inbound, internal delivery). + ``envelope`` carries the SMTP envelope (see ``InboundMessage.envelope``) + so the null-sender bounce check reads the authoritative MAIL FROM. Exceptions are logged but never propagated. """ try: - template = should_send_autoreply(mailbox, parsed_email, is_spam=is_spam) + template = should_send_autoreply( + mailbox, parsed_email, is_spam=is_spam, envelope=envelope + ) if template: send_autoreply_for_message(template, mailbox, message) except Exception: # pylint: disable=broad-exception-caught diff --git a/src/backend/core/mda/dispatch_webhooks.py b/src/backend/core/mda/dispatch_webhooks.py index eee1b2025..972e1e831 100644 --- a/src/backend/core/mda/dispatch_webhooks.py +++ b/src/backend/core/mda/dispatch_webhooks.py @@ -12,8 +12,8 @@ status holds the message for RETRY, but that is not a body-driven action); ``message.delivered`` is fire-and-forget after the message is created. -This file is webhook-specific: HTTP plumbing, signing (HMAC + JWT or -API key), JMAP body building, SSRF-safe POST, response classification. +This file is webhook-specific: HTTP plumbing, signing (JWT or API key), +JMAP body building, SSRF-safe POST, response classification. The pipeline-side glue is ``UserWebhookStep`` + ``webhook_steps_for_mailbox``. The HTTP client is the shared ``SSRFSafeSession`` — webhook URLs are @@ -35,7 +35,6 @@ from __future__ import annotations import hashlib -import hmac import json import logging import secrets @@ -54,7 +53,7 @@ from core import enums, models from core.mda.inbound_pipeline import ( - QUARANTINE_AFTER, + DEFERRAL_MAX_AGE, Decision, InboundContext, Step, @@ -87,7 +86,7 @@ MAX_LABELS_PER_RESPONSE = 50 MAX_ASSIGN_TO_PER_RESPONSE = 50 MAX_EVENTS_PER_RESPONSE = 20 -MAX_IM_CONTENT_BYTES = 8 * 1024 # 8 KiB per internal-message comment +MAX_IM_CONTENT_BYTES = 32 * 1024 # 32 KiB per internal-message comment PHASE_BEFORE_SPAM = "before_spam" PHASE_AFTER_SPAM = "after_spam" @@ -183,9 +182,9 @@ def from_cache(cls, data: Dict[str, Any]) -> "_HttpResult": # rare extra delivery is fine — we only need to turn "hundreds" into "a few". _WEBHOOK_RESULT_CACHE_VERSION = 1 -# Cover the whole quarantine window so a result cached on attempt 1 is still +# Cover the whole deferral window so a result cached on attempt 1 is still # served on the last retry before the message is delivered/abandoned. -_WEBHOOK_RESULT_CACHE_TTL = int(QUARANTINE_AFTER.total_seconds()) +_WEBHOOK_RESULT_CACHE_TTL = int(DEFERRAL_MAX_AGE.total_seconds()) def _webhook_results_cache_key(inbound_message_id: str) -> str: @@ -326,6 +325,30 @@ def _failure(blocking: bool, decision: Decision) -> _HttpResult: return _HttpResult(decision=decision if blocking else Decision.CONTINUE) +def _config_skip() -> _HttpResult: + """Result for a misconfiguration that waiting can't fix: a missing + secret / url / auth_method, or a URL the SSRF guard refuses at dispatch + time. Create/update validation rejects internal or unresolvable URLs and + guarantees a secret + valid auth_method, so at dispatch these mean either + a hand-edited (non-DRF) row or a malicious DNS rebinding — none of which a + 48h retry would ever clear. + + So CONTINUE: deliver the mail past the broken webhook rather than stall a + whole scope's inbound (instance-wide for a GLOBAL blocking webhook) for up + to 48 hours waiting for a config that has never worked. The SSRF guard + itself is unchanged — ``SSRFSafeSession`` still refuses to POST to an + internal address — so only this one webhook's gatekeeping is skipped (the + rest of the pipeline, incl. rspamd, still runs). Every caller logs at + ERROR so an admin is paged to fix or disable the channel. (A future + ``channel.is_active`` will let an operator disable it outright.) + + NB this is fail-OPEN for that webhook: a hard spam/security gate is + bypassed during the failure. Acceptable because it's a rare edge, the + breakage is loud, and stalling all inbound is the worse outcome — see + docs/webhooks.md.""" + return _HttpResult() # Decision.CONTINUE + + def _classify_response_body(body_bytes: bytes) -> _HttpResult: """Parse a 2xx response body into an ``_HttpResult``. @@ -467,21 +490,24 @@ def _classify_response_body(body_bytes: bytes) -> _HttpResult: USER_AGENT = "Messages-Webhook/1.0" -# Signature scheme tag. Bumped when the algorithm changes so receivers -# can pin the version they accept. -SIGNATURE_SCHEME = "v1" - -# JWT in the Authorization header is a short-lived HS256 token covering -# the same envelope as the raw HMAC, intended for receivers that prefer -# a standard JWT verify path (e.g. n8n, Zapier, Make). +# ``auth_method=jwt``: a short-lived HS256 JWT in the Authorization header. +# It is HMAC-based (HS256), binds the exact request body via the +# ``body_sha256`` claim, and carries ``exp`` (replay window) + ``jti`` +# (nonce) — a complete, self-contained signature a receiver verifies with +# any standard JWT library and the channel secret. (A separate raw-HMAC +# scheme, if ever wanted, would be its own ``auth_method`` — not bolted on +# here; the JWT already subsumes it.) JWT_ISSUER = "messages-webhook" -JWT_TTL_SECONDS = 300 # 5 min — same window receivers should accept on the raw HMAC +JWT_TTL_SECONDS = 300 # 5 min — receivers SHOULD reject older tokens (exp) def _resolve_body( body_format: str, raw_data: bytes, parsed_email: JmapEmail, + *, + message_id: Optional[str] = None, + thread_id: Optional[str] = None, ) -> Tuple[str, bytes]: """Compute (Content-Type, raw bytes to sign and POST). @@ -492,11 +518,21 @@ def _resolve_body( JSON is serialised here once so the signature and the wire bytes cannot drift (``requests`` would otherwise re-serialise with different separators/ordering). + + ``message_id`` / ``thread_id`` are passed only on the post-creation + (``message.delivered``) path, where they populate the JMAP body's + ``id`` / ``threadId``; the blocking paths fire before the row exists + and omit them (see ``build_jmap_email``). """ if body_format == FORMAT_EML: return "message/rfc822", raw_data include_body = body_format == FORMAT_JMAP - payload = build_jmap_email(parsed_email, include_body=include_body) + payload = build_jmap_email( + parsed_email, + include_body=include_body, + message_id=message_id, + thread_id=thread_id, + ) # ``separators=(",", ":")`` produces the compact bytes we sign. # Hand the same bytes to ``requests`` via ``data=`` so what we sign # is exactly what we POST. @@ -506,16 +542,6 @@ def _resolve_body( return "application/json", body_bytes -def _sign(secret: str, timestamp: str, body_bytes: bytes) -> str: - """Stripe-style HMAC: HMAC-SHA256 over ``{timestamp}.{body}``. - - Returns hex digest. Receivers MUST compute the same and compare - constant-time, and SHOULD reject timestamps older than ~5 minutes. - """ - msg = timestamp.encode("ascii") + b"." + body_bytes - return hmac.new(secret.encode("utf-8"), msg, hashlib.sha256).hexdigest() - - def _sign_jwt( secret: str, *, @@ -661,13 +687,17 @@ class UserWebhookStep: - blocking: * 2xx + ``{"action":"drop"}`` → DROP (the *only* path to DROP) * 2xx + anything else → CONTINUE (with optional side effects) - * any non-2xx (4xx / 5xx / 3xx) → RETRY - * SSRF / missing secret / unknown auth_method → RETRY - * timeout / connection / generic transport → RETRY + * TRANSIENT failure — any non-2xx (4xx / 5xx / 3xx), timeout, + connection, generic transport, response-read budget → RETRY + (recoverable; bounded by the 48h deferral window) + * CONFIG failure — SSRF reject / missing secret / url / + auth_method → CONTINUE past the broken webhook + ``log.error`` + (retry can't fix it; see ``_config_skip``) A webhook error never drops the user's email — only an explicit - ``{"action": "drop"}`` does. Every failure is held for retry, - bounded by the pipeline's 48-hour quarantine window. + ``{"action": "drop"}`` does. A *transient* failure is held for retry + (bounded by the 48h deferral window); a *config* failure delivers + the mail past the broken webhook and pages an admin. """ def __init__(self, channel: models.Channel, phase: str): @@ -767,31 +797,26 @@ def _build_auth_headers( auth_method = (channel.settings or {}).get("auth_method") if auth_method == enums.WebhookAuthMethod.API_KEY: - # Derived from the root secret via HMAC. The raw root never - # touches the wire — a receiver-side log leak of this value - # reveals nothing about the root, so HMAC/JWT verification - # remains unforgeable. - return {"X-StMsg-Api-Key": channel.get_webhook_api_key()} + # Static key derived from the root via HMAC, presented as an opaque + # Bearer token (RFC 6750 — a Bearer token need not be a JWT). Sent via + # ``Authorization`` like the jwt method, so logging / proxy / APM + # tooling auto-redacts it — which matters for a long-lived static + # credential. The raw root never touches the wire; a receiver-side leak + # of this derived value reveals nothing about the root. + return {"Authorization": f"Bearer {channel.get_webhook_api_key()}"} if auth_method == enums.WebhookAuthMethod.JWT: - # HMAC signature over the body + short-TTL HS256 JWT, both keyed - # by the root secret. Signed at send time (here / in the task), - # so the JWT TTL is measured from the actual POST, not enqueue. - now = int(time.time()) - timestamp = str(now) - signature = _sign(secret, timestamp, body_bytes) + # Short-TTL HS256 JWT keyed by the root secret, binding the exact + # body via ``body_sha256``. Signed at send time (here / in the task), + # so the TTL is measured from the actual POST, not enqueue. bearer = _sign_jwt( secret, channel=channel, mailbox=mailbox, body_bytes=body_bytes, - issued_at=now, + issued_at=int(time.time()), ) - return { - "X-StMsg-Webhook-Timestamp": timestamp, - "X-StMsg-Webhook-Signature": f"{SIGNATURE_SCHEME}={signature}", - "Authorization": f"Bearer {bearer}", - } + return {"Authorization": f"Bearer {bearer}"} # Settings validator forbids creating a webhook channel without a # valid auth_method; an existing row with a missing/unknown value is @@ -822,23 +847,27 @@ def _deliver_signed_webhook( """ secret = (channel.encrypted_settings or {}).get("secret") if not secret: - # The create path always mints a secret; a row without one is - # misconfigured. We can't sign the POST, so we hold for RETRY - # rather than drop the user's mail — re-minting the secret lets - # the next sweep deliver. A webhook failure must never silently - # discard the email (only an explicit ``{"action": "drop"}`` on - # a 2xx does that). - logger.warning( - "Webhook channel %s has no secret — holding for retry", + # The create path always mints a secret; a row without one is a + # (non-DRF) misconfiguration that retry can't fix — CONTINUE past it + # rather than stall the scope's inbound (see ``_config_skip``). + logger.error( + "Webhook channel %s has no secret — delivering past it; " + "fix or disable the channel", channel.id, ) - return _failure(blocking, Decision.RETRY) + return _config_skip() auth_headers = _build_auth_headers(channel, secret, body_bytes, mailbox) if auth_headers is None: - # Unknown/misconfigured auth_method — same reasoning: hold for - # retry, never drop the email on our config error. - return _failure(blocking, Decision.RETRY) + # Unknown/missing auth_method. Create-time validation requires a valid + # one, so this is a non-DRF row; retry can't fix it — CONTINUE past it. + # ``_build_auth_headers`` logged the detail; surface it at ERROR here. + logger.error( + "Webhook channel %s has an unusable auth_method — delivering past " + "it; fix or disable the channel", + channel.id, + ) + return _config_skip() signed_headers = { **envelope_headers, @@ -860,20 +889,24 @@ def _deliver_signed_webhook( data=body_bytes, ) except SSRFValidationError as exc: - # SSRF block is a config error on our side (the URL points at a - # disallowed address). Hold for RETRY rather than drop — fixing - # the URL lets the next sweep deliver. We never discard the user's - # mail because of a webhook/config failure. - logger.warning( - "Webhook channel %s rejected by SSRF for url=%s: %s", + # The URL resolves to a disallowed (internal) address or won't + # resolve. Create/update validation already rejects internal / + # unresolvable URLs, so at dispatch this is a DNS rebinding (or a + # non-DRF row) — retry can't fix it, and the guard already (correctly) + # refused to POST to the internal target. CONTINUE past it rather than + # stall inbound. (``exc`` carries only the hostname, never the + # secret-bearing path/query, so it's safe to log.) + logger.error( + "Webhook channel %s rejected by SSRF for url=%s: %s — delivering " + "past it; fix or disable the channel", channel.id, _sanitize_url(url), exc, ) - return _failure(blocking, Decision.RETRY) + return _config_skip() except Exception as exc: # Timeout, connection refused, DNS, unknown transport-level - # failure: all transient. The 48-hour quarantine window in the + # failure: all transient. The 48-hour deferral window in the # pipeline runner bounds the retries. Log only the exception # *type*, not its message or traceback: requests/urllib3 errors # embed the full request URL (path + query), which is exactly @@ -925,7 +958,7 @@ def _deliver_signed_webhook( # webhook DROPs an email *only* when it explicitly returns # ``{"action": "drop"}`` with a 2xx (handled above). A receiver # bug that answers 4xx must never cost the user their mail — the - # 48-hour quarantine window bounds the hold. + # 48-hour deferral window bounds the hold. return _failure(blocking, Decision.RETRY) finally: response.close() @@ -952,11 +985,15 @@ def _dispatch_webhook( """ url = (channel.settings or {}).get("url") if not url: - # The serializer guarantees a url on create; a row without one is - # misconfigured. Hold for retry rather than drop (blocking); for - # the non-blocking task this collapses to a no-op CONTINUE. - logger.warning("Webhook channel %s has no url — skipping", channel.id) - return _failure(blocking, Decision.RETRY) + # The serializer guarantees a url on create; a row without one is a + # (non-DRF) misconfiguration retry can't fix — CONTINUE past it rather + # than stall inbound (see ``_config_skip``). + logger.error( + "Webhook channel %s has no url — delivering past it; " + "fix or disable the channel", + channel.id, + ) + return _config_skip() envelope_headers = _envelope_headers( channel=channel, mailbox=mailbox, @@ -1003,7 +1040,14 @@ def _resolve_body_from_message( f"cannot parse stored blob for message {message.id} into " f"webhook format {body_format!r}" ) - return _resolve_body(body_format, raw_data, parsed_email or {}) + # Post-creation path: stamp the persisted ids into the JMAP body. + return _resolve_body( + body_format, + raw_data, + parsed_email or {}, + message_id=str(message.id), + thread_id=str(message.thread_id), + ) def dispatch_recorded_webhooks( diff --git a/src/backend/core/mda/inbound.py b/src/backend/core/mda/inbound.py index fb9ee781f..5d702539c 100644 --- a/src/backend/core/mda/inbound.py +++ b/src/backend/core/mda/inbound.py @@ -139,7 +139,7 @@ def deliver_inbound_message( imap_labels: list[str] | None = None, imap_flags: list[str] | None = None, channel: models.Channel | None = None, - is_internal: bool = False, + envelope: dict | None = None, blob: "models.Blob | None" = None, ) -> bool: # Return True on success, False on failure """Deliver a parsed inbound email message. @@ -149,12 +149,18 @@ def deliver_inbound_message( Warning: messages imported here could be is_sender=True. Everything else is queued for the inbound pipeline via - ``process_inbound_message_task`` (spam steps + user webhooks). - Internal mailbox-to-mailbox delivery sets ``is_internal=True`` so the - pipeline skips spam checking while still firing webhooks, and passes - the sender's already-committed ``blob`` so the queue row references - the encrypted bytes instead of copying them. ``raw_data`` is stored - inline only when no ``blob`` is supplied (the external MTA path). + ``process_inbound_message_task`` (spam steps + user webhooks). The bytes + are committed to an encrypted, content-addressed ``Blob`` at ingest: the + caller may pass an already-committed ``blob`` (internal mail reuses the + sender's ``Message.blob``), otherwise one is created from ``raw_data`` + (external MTA / widget). Because ``create_blob`` dedups by content hash, + a message delivered to N recipients shares ONE blob and nothing sits in + plaintext. + + ``envelope`` is the structured SMTP/provenance record for this delivery + (see ``InboundMessage.envelope``); its ``origin`` key is the explicit + trust discriminator that drives ``is_internal`` — internal mail skips the + spam steps while still firing user webhooks. raw_data is not parsed again, just stored as is. """ @@ -208,28 +214,37 @@ def deliver_inbound_message( ) return bool(result) + envelope = envelope or {} + is_internal = envelope.get("origin") == "internal" + # Internal mail is expected to reference the sender's already-committed # blob — that's the whole point (no second plaintext copy). Enforce the - # contract so a future caller can't silently fall back to inline bytes. + # contract so a future caller can't silently fall back to re-ingesting. if is_internal and blob is None: raise ValueError("internal delivery requires a blob") - # External and internal messages: queue for the inbound pipeline. - # Internal mail references the sender's existing blob (no second - # plaintext copy); external mail stores its MTA bytes inline. + # External and internal messages: queue for the inbound pipeline. Commit + # the bytes to an encrypted, content-addressed blob at ingest — internal + # mail already carries the sender's committed blob; external/widget mail + # is ingested here. create_blob dedups by SHA-256, so N recipients of the + # same message end up sharing one blob. try: + if blob is None: + blob = models.Blob.objects.create_blob( + content=raw_data, + content_type="message/rfc822", + ) inbound_message = models.InboundMessage.objects.create( mailbox=mailbox, - raw_data=None if blob is not None else raw_data, blob=blob, + envelope=envelope, channel=channel, - is_internal=is_internal, ) logger.info( - "Queued inbound message %s (recipient: %s, internal: %s)", + "Queued inbound message %s (recipient: %s, origin: %s)", inbound_message.id, recipient_email, - is_internal, + envelope.get("origin"), ) # Queue the task immediately for processing (no lag) process_inbound_message_task.delay(str(inbound_message.id)) diff --git a/src/backend/core/mda/inbound_create.py b/src/backend/core/mda/inbound_create.py index f05a228d9..4b765bf83 100644 --- a/src/backend/core/mda/inbound_create.py +++ b/src/backend/core/mda/inbound_create.py @@ -200,6 +200,30 @@ def _find_thread_by_message_ids( return None +def _record_divergent_rcpt( + postmark: dict, recipient_email: str, parsed_email: JmapEmail +) -> None: + """Record the envelope RCPT TO in ``postmark`` when it diverges from the + MIME addressees. + + In the happy path the RCPT is one of the visible To/Cc addresses and we + store nothing (keeps ``postmark`` NULL). When it isn't — a BCC'd copy, an + alias/catch-all, or plus-addressed delivery — the RCPT is the only record + of how this mailbox actually received the mail, so we keep it. Matching is + case-insensitive on the address. + """ + rcpt = (recipient_email or "").strip().lower() + if not rcpt: + return + visible = { + (entry.get("email") or "").strip().lower() + for field_name in ("to", "cc") + for entry in (parsed_email.get(field_name) or []) + } + if rcpt not in visible: + postmark["rcpt_to"] = recipient_email + + def _create_message_from_inbound( # pylint: disable=too-many-arguments recipient_email: str, parsed_email: JmapEmail, @@ -214,6 +238,8 @@ def _create_message_from_inbound( # pylint: disable=too-many-arguments is_trashed: bool = False, is_archived: bool = False, is_outbound: bool = False, + blob: "models.Blob | None" = None, + postmark: dict | None = None, ) -> models.Message | None: """Create a message and thread from parsed email data. @@ -428,15 +454,26 @@ def _create_message_from_inbound( # pylint: disable=too-many-arguments # so the GC sweep never sees the Blob row without its # referencing FK on ``Message.blob``. Outbound messages have # no blob yet — ``prepare_outbound_message`` adds it later. + # Per-recipient postmark: record the envelope RCPT TO only when it + # diverges from the MIME To/Cc (BCC / alias / catch-all) — the + # "you were BCC'd / reached via " signal. Structured, never + # in the bytes, so it can't break blob dedup across recipients. + postmark = dict(postmark or {}) + _record_divergent_rcpt(postmark, recipient_email, parsed_email) + with transaction.atomic(): - blob = None - if not is_outbound: + # Reuse the ingest blob (inbound queue path) so a message has + # ONE blob from ingest through to here — no second plaintext + # copy, no re-encrypt. Imports have no ingest blob and pass + # raw bytes; outbound gets its blob later from the send path. + if blob is None and not is_outbound: blob = models.Blob.objects.create_blob( content=raw_data, content_type="message/rfc822", ) message = models.Message.objects.create( + postmark=postmark or None, thread=thread, sender=sender_contact, subject=subject, diff --git a/src/backend/core/mda/inbound_pipeline.py b/src/backend/core/mda/inbound_pipeline.py index f190ad942..4b6f5e23f 100644 --- a/src/backend/core/mda/inbound_pipeline.py +++ b/src/backend/core/mda/inbound_pipeline.py @@ -6,21 +6,12 @@ the context — set ``is_spam``, add ``labels``, cache ``rspamd_result``, prepend an authentication header, etc. - pipeline = [ - *user_webhook_steps(mailbox, phase="before_spam"), - hardcoded_rules_step(spam_config), - rspamd_step(spam_config), - inbound_auth_step(spam_config), - *user_webhook_steps(mailbox, phase="after_spam"), - ] - for step in pipeline: - d = step(ctx) - if d != Decision.CONTINUE: - break - -The orchestrator (``run_inbound_pipeline``) iterates and aborts on the -first ``DROP`` / ``RETRY``. The caller turns that decision into a -task-level return value. +``build_inbound_pipeline`` assembles the ordered step list for a message — +before-spam user webhooks, the hardcoded-rules and rspamd spam checks, the +inbound-auth (DKIM/DMARC) step, then after-spam user webhooks. +``run_inbound_pipeline`` iterates that list and aborts on the first step +that returns a non-``CONTINUE`` ``Decision`` (``DROP`` / ``RETRY``); the +caller turns that decision into a task-level return value. This file deliberately knows nothing about HTTP, JWT, or JMAP — those live in ``dispatch_webhooks.py`` behind ``UserWebhookStep``. The @@ -38,11 +29,12 @@ from enum import IntEnum from typing import Any, Callable, Dict, List, Optional, Set, Tuple +from django.conf import settings from django.db.models import Q from django.utils import timezone import requests -from jmap_email import JmapEmail, has_header, parse_email +from jmap_email import JmapEmail, has_header from core import enums, models from core.mda.inbound_auth import ( @@ -88,6 +80,12 @@ class InboundContext: # pylint: disable=too-many-instance-attributes # - True/False: the last decisive step wins is_spam: Optional[bool] = None + # Sparse pipeline record written to ``Message.postmark`` at finalize time. + # Steps add flat keys ("auth", "processing", ...) rather than prepending + # X-StMsg-* to the bytes, so the ingest blob is reused untouched. Empty on + # the happy path (finalize stores NULL, not {}). + postmark: Dict[str, Any] = field(default_factory=dict) + # Labels webhook receivers have asked us to attach to the thread. # Validated against the destination mailbox at finalize time; # unknown UUIDs are dropped silently. @@ -154,19 +152,21 @@ class InboundContext: # pylint: disable=too-many-instance-attributes # Inbound messages held by a transient RETRY get one more chance every -# 5 minutes via ``process_inbound_messages_queue_task``. The webhook step -# is the only producer of a RETRY, and it is bounded by the quarantine -# window below — so a held message is never dropped, only delivered -# (flagged) once it gives up. +# 5 minutes via ``process_inbound_messages_queue_task``. RETRY is produced +# by a blocking webhook step (transport failure / non-2xx) or by the rspamd +# step (spam-check outage), and is bounded by the deferral window below — +# so a held message is never dropped, only delivered (flagged) once it +# gives up. # -# A processing step that keeps failing (a blocking webhook today; rspamd -# or any future RETRY-returning step tomorrow) must not hold a message +# A processing step that keeps failing (a blocking webhook or rspamd today, +# any future RETRY-returning step tomorrow) must not hold a message # forever *or* silently lose it. After this window the task stops holding # and delivers the message anyway, stamped with ``X-StMsg-Processing- # Failed`` so the UI warns the recipient it bypassed a processing step. # Generic on purpose — see the RETRY branch in -# ``process_inbound_message_task``. -QUARANTINE_AFTER = timedelta(hours=48) +# ``process_inbound_message_task``. Operator-tunable via +# ``MESSAGES_INBOUND_DEFERRAL_MAX_AGE`` (seconds; default 48h). +DEFERRAL_MAX_AGE = timedelta(seconds=settings.MESSAGES_INBOUND_DEFERRAL_MAX_AGE) # --------------------------------------------------------------------------- @@ -237,14 +237,26 @@ def _check_hardcoded_rules( def _call_rspamd( - raw_data: bytes, spam_config: Dict[str, Any] -) -> Tuple[Optional[bool], Optional[str], Optional[Dict[str, Any]]]: + raw_data: bytes, + spam_config: Dict[str, Any], + envelope: Optional[Dict[str, Any]] = None, +) -> Tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]: """POST raw RFC-822 bytes to rspamd's ``/checkv2``. - Returns ``(is_spam_or_None, error_message, result_dict)``. is_spam - is ``None`` only when rspamd is not configured. Errors are - swallowed and surfaced via the error_message channel so a flaky - rspamd never blocks delivery (mirroring the old behaviour). + The SMTP ``envelope`` (MAIL FROM, RCPT TO, connecting IP / HELO / rDNS) + is forwarded via rspamd's documented scan headers + (``From``/``Rcpt``/``IP``/``Helo``/``Hostname``) so envelope-based checks + — SPF, DNS RBLs, the Return-Path-vs-From mismatch symbols — score against + the real peer. Without them rspamd sees an empty envelope and SPF/RBL + degrade to no-ops. + + Returns ``(action, error_message, result_dict)``. ``action`` is the raw + rspamd action string (e.g. "no action", "add header", "reject", …) — this + function does NOT interpret it; the whole action → verdict/decision mapping + lives in a single place, ``_make_rspamd_step``. ``action`` is ``None`` when + rspamd is not configured or on error (the ``error_message`` channel + distinguishes the two); errors are swallowed so a flaky rspamd never blocks + delivery. """ url = spam_config.get("rspamd_url") if not url: @@ -256,6 +268,22 @@ def _call_rspamd( if auth: headers["Authorization"] = auth + # Forward the SMTP envelope as rspamd scan headers. Only send the fields + # we actually have (widget/internal mail carry no HELO/rDNS); an empty + # value would read as a genuinely empty envelope and skew scoring. HELO + # and hostname are attacker-influenced, so strip CR/LF to prevent header + # injection into the rspamd request. + for header_name, envelope_key in ( + ("From", "mail_from"), + ("Rcpt", "rcpt_to"), + ("IP", "ip"), + ("Helo", "helo"), + ("Hostname", "hostname"), + ): + value = (envelope or {}).get(envelope_key) + if value: + headers[header_name] = str(value).replace("\r", "").replace("\n", "") + try: response = requests.post( f"{url}/checkv2", data=raw_data, headers=headers, timeout=10 @@ -269,27 +297,19 @@ def _call_rspamd( # is_spam=False so the pipeline keeps moving. The # error_message channel lets the caller log loudly. logger.exception("Error calling rspamd: %s", exc) - return False, str(exc), None + return None, str(exc), None except Exception as exc: logger.exception("Unexpected error calling rspamd: %s", exc) - return False, str(exc), None + return None, str(exc), None if not isinstance(result, dict): logger.warning("rspamd returned non-object body: %r", result) - return False, "rspamd returned non-object body", None + return None, "rspamd returned non-object body", None action = result.get("action", "") score = result.get("score", 0.0) - required = result.get("required_score", 15.0) - is_spam = action == "reject" - logger.info( - "Rspamd: action=%s score=%.2f required=%.2f is_spam=%s", - action, - score, - required, - is_spam, - ) - return is_spam, None, result + logger.info("Rspamd: action=%s score=%.2f", action, score) + return action, None, result # --------------------------------------------------------------------------- @@ -314,16 +334,27 @@ def hardcoded_rules(ctx: InboundContext) -> Decision: def _make_rspamd_step(spam_config: Dict[str, Any]) -> Step: """Rspamd as a step. - Sets ``is_spam`` if no earlier step decided. Always caches the - full ``rspamd_result`` dict on the context — ``inbound_auth_step`` - reuses the symbols (DKIM/DMARC) without a second HTTP call. - - An rspamd *error* never fails open: we don't deliver mail that - couldn't be spam-checked. The step RETRYs instead, so the message is - held and retried; if the outage lasts past ``QUARANTINE_AFTER`` the - quarantine path delivers it flagged rather than silently unchecked. - (rspamd simply not being configured is not an error — ``_call_rspamd`` - returns no opinion and the pipeline moves on.) + Maps the rspamd action onto a pipeline decision. Always caches the full + ``rspamd_result`` dict on the context — ``inbound_auth_step`` reuses the + symbols (DKIM/DMARC) without a second HTTP call. The full action set + (https://docs.rspamd.com/configuration/metrics/): + + * ``no action`` → deliver (is_spam=False). + * ``add header`` / ``rewrite subject`` / ``quarantine`` / ``reject`` → + spam verdict (is_spam=True → Junk). We can't honour ``reject`` at SMTP + time (already accepted), so it lands in Junk like the other markers. + * ``greylist`` / ``soft reject`` → temporary failures, NOT verdicts: route + onto our deferral path (RETRY). The condition (rate-limit, greylist, + transient DNS) usually clears within the 5-min sweep; a persistent one + force-delivers flagged past ``DEFERRAL_MAX_AGE`` (postmark processing). + * ``discard`` → rspamd asks us to accept-and-silently-drop (blackhole, no + bounce). Honour it as DROP — the message is consumed, nothing created. + + An rspamd *error* never fails open: we don't deliver mail that couldn't be + spam-checked. The step RETRYs, so the message is held; if the outage lasts + past ``DEFERRAL_MAX_AGE`` it is force-delivered flagged rather than + silently unchecked. (rspamd not being configured is not an error — + ``_call_rspamd`` returns no opinion and the pipeline moves on.) """ def rspamd(ctx: InboundContext) -> Decision: @@ -332,10 +363,12 @@ def rspamd(ctx: InboundContext) -> Decision: # rspamd's symbols for inbound_auth. The auth step has its # own fallback so we can cheaply skip rspamd entirely here. return Decision.CONTINUE - is_spam, err, result = _call_rspamd(ctx.raw_data, spam_config) + action, err, result = _call_rspamd( + ctx.raw_data, spam_config, envelope=ctx.inbound_message.envelope + ) if err: # Don't fail open — hold for retry rather than deliver - # unchecked. A sustained outage is bounded by the quarantine + # unchecked. A sustained outage is bounded by the deferral # window (the message is then delivered flagged). logger.warning( "rspamd error on inbound message %s: %s (holding for retry)", @@ -344,8 +377,40 @@ def rspamd(ctx: InboundContext) -> Decision: ) return Decision.RETRY ctx.rspamd_result = result - if is_spam is not None: - ctx.is_spam = is_spam + if action is None: + # rspamd not configured — no opinion, leave the verdict undecided. + return Decision.CONTINUE + + # --- The single source of truth for rspamd action → outcome. --- + if action in ("greylist", "soft reject"): + # Temporary failure, not a verdict — defer and re-evaluate on the + # sweep (converges on the same deferral-expiry force-delivery as + # any persistent processing failure). + logger.info( + "rspamd '%s' on inbound message %s — holding for retry", + action, + ctx.inbound_message.id, + ) + return Decision.RETRY + if action == "discard": + # rspamd blackhole: accept-and-drop, no delivery, no bounce. + logger.info( + "rspamd 'discard' on inbound message %s — dropping", + ctx.inbound_message.id, + ) + return Decision.DROP + # Milder flag actions deliver to the INBOX with a graded "suspected + # spam" marker for the UI (like the suspicious-sender banner) — not + # hidden in Junk. rspamd scores "add header" below "rewrite subject", + # so preserve that gradient: possible < likely. + if action == "add header": + ctx.postmark["spam"] = "possible" + elif action == "rewrite subject": + ctx.postmark["spam"] = "likely" + # Junk only for the high-confidence isolate actions; everything else + # (no action, the flagged-but-delivered actions above, unknown) is + # delivered to the inbox. + ctx.is_spam = action in ("quarantine", "reject") return Decision.CONTINUE rspamd.name = "rspamd" @@ -356,28 +421,32 @@ def _make_inbound_auth_step(spam_config: Dict[str, Any]) -> Step: """DKIM / DMARC verdict via ``check_inbound_authentication``. Reuses ``ctx.rspamd_result`` if populated; otherwise calls rspamd - itself when ``auth_mode='rspamd'``. On a verdict, prepends an - ``X-StMsg-Sender-Auth`` header to both ``raw_data`` and - ``parsed_email`` so subsequent steps + downstream consumers see it. + itself when ``auth_mode='rspamd'``. On a verdict, records it in + ``ctx.postmark["auth"]`` (structured, off the bytes) so the ingest blob + is reused untouched; ``get_stmsg_headers`` surfaces it downstream. """ def inbound_auth(ctx: InboundContext) -> Decision: if ctx.rspamd_result is None and get_inbound_auth_mode(spam_config) == "rspamd": - _, _, ctx.rspamd_result = _call_rspamd(ctx.raw_data, spam_config) + _, _, ctx.rspamd_result = _call_rspamd( + ctx.raw_data, spam_config, envelope=ctx.inbound_message.envelope + ) verdict = check_inbound_authentication( ctx.raw_data, ctx.parsed_email, spam_config, ctx.rspamd_result ) if not verdict: + # Widget submissions arrive over an unauthenticated web form, so + # they carry the "none" baseline even when DKIM/DMARC verification + # is disabled instance-wide (which is when ``verdict`` is falsy). + if (ctx.inbound_message.envelope or {}).get("origin") == "widget": + ctx.postmark["auth"] = "none" return Decision.CONTINUE - prepended = f"X-StMsg-Sender-Auth: {verdict}\r\n".encode("ascii") + ctx.raw_data - reparsed = parse_email(prepended) - if reparsed is not None: - ctx.parsed_email = reparsed - ctx.raw_data = prepended - else: - # Keep raw_data / parsed_email in lockstep: if re-parse breaks, - # we sacrifice the auth banner rather than corrupting the blob. - logger.warning("Failed to re-parse after prepending X-StMsg-Sender-Auth") + # Record the verdict structurally instead of prepending it to the + # bytes: the ingest blob stays untouched (reused as Message.blob) and + # ``get_stmsg_headers`` surfaces it. ``verdict`` is already "none" + # (unverified) or "fail" (forged); a verified message returns no + # verdict and leaves ``auth`` absent. + ctx.postmark["auth"] = verdict return Decision.CONTINUE inbound_auth.name = "inbound_auth" diff --git a/src/backend/core/mda/inbound_tasks.py b/src/backend/core/mda/inbound_tasks.py index 2f0bef664..7e98bc87a 100644 --- a/src/backend/core/mda/inbound_tasks.py +++ b/src/backend/core/mda/inbound_tasks.py @@ -27,7 +27,7 @@ ) from core.mda.inbound_create import _create_message_from_inbound from core.mda.inbound_pipeline import ( - QUARANTINE_AFTER, + DEFERRAL_MAX_AGE, Decision, InboundContext, apply_labels_to_thread, @@ -110,10 +110,11 @@ def _handle_retry( The InboundMessage row is kept in place — the 5-min sweep (``process_inbound_messages_queue_task``) re-fires the task on the - next cycle. We never drop here: a persistently-failing blocking - webhook is bounded instead by ``QUARANTINE_AFTER`` (the message is - then delivered flagged, see ``_stamp_processing_failed``), and the - webhook step is the only thing that produces a RETRY. + next cycle. We never drop here: a persistently-failing processing step + (a blocking webhook, or rspamd being unreachable) is bounded instead by + ``DEFERRAL_MAX_AGE`` (the message is then delivered flagged, see + ``_stamp_processing_failed``). The blocking webhook steps and the rspamd + step are the producers of a RETRY. """ age = timezone.now() - inbound_message.created_at logger.info( @@ -142,18 +143,18 @@ def _retry_or_abandon( ) -> Dict[str, Any]: """Bounded handling for a message that failed to be created/processed. - Within ``QUARANTINE_AFTER`` the row is kept so the 5-min sweep retries + Within ``DEFERRAL_MAX_AGE`` the row is kept so the 5-min sweep retries it (a transient DB error or constraint hiccup clears on its own). Past the window the attempt is abandoned: ``abandoned_at`` is stamped so the sweep skips the row and stops re-running the whole pipeline — and re-firing every user webhook — on it, but the row is NOT deleted. The - raw bytes (``raw_data`` or the referenced ``blob``) are the only copy of - the message, so deleting would silently lose mail; instead an operator + referenced ``blob`` is the only copy of the message, so deleting would + silently lose mail; instead an operator can inspect and replay the row from the Django admin, and ``logger.error`` raises a Sentry alert. ``error_message`` keeps the human-readable reason. """ age = timezone.now() - inbound_message.created_at - if age <= QUARANTINE_AFTER: + if age <= DEFERRAL_MAX_AGE: inbound_message.error_message = reason inbound_message.save(update_fields=["error_message"]) return { @@ -182,25 +183,15 @@ def _retry_or_abandon( def _stamp_processing_failed(ctx: InboundContext) -> None: - """Prepend the ``X-StMsg-Processing-Failed`` marker to the message. - - Mirrors the ``X-StMsg-Sender-Auth`` prepend in the pipeline: the - header rides in the stored MIME, ``Message.get_stmsg_headers()`` - surfaces it as ``processing-failed``, and the frontend renders a - warning banner. Deliberately generic — any processing step that - fails persistently (a blocking webhook, rspamd, …) lands here. - Sender-supplied ``X-StMsg-*`` headers are stripped at ingest, so this - namespace is ours alone — the flag can't be forged. + """Record the ``processing`` failure marker in ``ctx.postmark``. + + Written structurally (not prepended to the bytes), so the ingest blob is + reused untouched as ``Message.blob``. ``Message.get_stmsg_headers()`` + surfaces it as ``processing-failed`` and the frontend renders a warning + banner. Deliberately generic — any processing step that fails + persistently (a blocking webhook, rspamd, …) lands here. """ - prepended = b"X-StMsg-Processing-Failed: true\r\n" + ctx.raw_data - reparsed = parse_email(prepended) - if reparsed is not None: - ctx.parsed_email = reparsed - ctx.raw_data = prepended - else: - # Keep raw_data / parsed_email in lockstep — drop the marker - # rather than corrupt the blob (same fallback as Sender-Auth). - logger.warning("Failed to re-parse after prepending X-StMsg-Processing-Failed") + ctx.postmark["processing"] = "fail" @celery_app.task( @@ -252,7 +243,7 @@ def process_inbound_message_task(self, inbound_message_id: str): if parsed_email is None: # A deterministic parse failure never succeeds on retry — # route through ``_retry_or_abandon`` so it's bounded by the - # quarantine window instead of looping on every 5-min sweep. + # deferral window instead of looping on every 5-min sweep. return _retry_or_abandon(inbound_message, "Failed to parse email message") mailbox = inbound_message.mailbox @@ -303,10 +294,10 @@ def process_inbound_message_task(self, inbound_message_id: str): "inbound_message_id": str(inbound_message_id), "dropped_by": aborted_by, } - quarantined = False + deferral_expired = False if decision == Decision.RETRY: age = timezone.now() - inbound_message.created_at - if age <= QUARANTINE_AFTER: + if age <= DEFERRAL_MAX_AGE: # About to hold for retry: persist the blocking webhooks that # DID succeed this round so the next attempt replays them # instead of re-POSTing. Written only here, on the retry path — @@ -315,31 +306,31 @@ def process_inbound_message_task(self, inbound_message_id: str): str(inbound_message.id), ctx.blocking_webhook_results ) return _handle_retry(inbound_message, aborted_by) - # Past the quarantine window: a processing step (blocking + # The deferral window has expired: a processing step (blocking # webhook, rspamd, …) has failed persistently. Stop holding — # deliver the message anyway so it's never lost, but stamp it # so the UI warns the recipient it bypassed a processing step, # and land it in the inbox (is_spam=False) so the warning is # actually seen rather than buried in the spam folder. logger.warning( - "Inbound message %s quarantine-delivered after persistent " - "failure at step=%s (age=%s)", + "Inbound message %s force-delivered (deferral window expired) " + "after persistent failure at step=%s (age=%s)", inbound_message_id, aborted_by, age, ) _stamp_processing_failed(ctx) - quarantined = True + deferral_expired = True # The message is being forced to the inbox, so it is no longer # treated as spam. Normalize ctx.is_spam so downstream consumers # (autoreply gate, task result) agree with where it actually lands. ctx.is_spam = False - # ...but a quarantine delivery means a processing step never - # completed: the forced is_spam=False is a placement decision, - # not a real spam verdict, and a blocking step that wanted to - # suppress the reply (or classify the sender as spam) never got - # to run. Don't fire an autoreply to a sender we couldn't fully - # vet — suppress it for quarantined messages. + # ...but force-delivering past an expired deferral means a + # processing step never completed: the forced is_spam=False is a + # placement decision, not a real spam verdict, and a blocking step + # that wanted to suppress the reply (or classify the sender as + # spam) never got to run. Don't fire an autoreply to a sender we + # couldn't fully vet — suppress it when the deferral window expired. ctx.skip_autoreply = True # Create the Message and drop the queue row as one unit: either the @@ -354,9 +345,13 @@ def process_inbound_message_task(self, inbound_message_id: str): raw_data=ctx.raw_data, mailbox=mailbox, channel=inbound_message.channel, - is_spam=False if quarantined else bool(ctx.is_spam), + is_spam=False if deferral_expired else bool(ctx.is_spam), is_trashed=ctx.mark_trashed, is_archived=ctx.mark_archived, + # Reuse the ingest blob (the bytes are never mutated now that + # verdicts go to postmark) and carry the pipeline's postmark. + blob=inbound_message.blob, + postmark=ctx.postmark, ) if inbound_msg: inbound_message.delete() @@ -464,6 +459,7 @@ def process_inbound_message_task(self, inbound_message_id: str): ctx.parsed_email, inbound_msg, is_spam=bool(ctx.is_spam), + envelope=inbound_message.envelope, ) except Exception: # pylint: disable=broad-exception-caught logger.exception( @@ -494,7 +490,7 @@ def process_inbound_message_task(self, inbound_message_id: str): # the hard limit SIGKILLs us — so the ``finally`` below releases the # lock cleanly. Hold for retry: a message that *always* overruns # (e.g. far too many slow blocking webhooks) is bounded by the same - # 48h quarantine and ends up abandoned (kept + marked) rather than + # deferral window and ends up abandoned (kept + marked) rather than # looping forever. logger.warning( "Inbound message %s exceeded the %ss soft time limit — holding for retry", diff --git a/src/backend/core/mda/outbound.py b/src/backend/core/mda/outbound.py index 0ed65e958..7552df087 100644 --- a/src/backend/core/mda/outbound.py +++ b/src/backend/core/mda/outbound.py @@ -695,7 +695,11 @@ def _mark_delivered( recipient_email, parsed_email, blob_content, - is_internal=True, + envelope={ + "origin": "internal", + "mail_from": message.sender.email, + "rcpt_to": recipient_email, + }, blob=message.blob, ) _mark_delivered(recipient_email, delivered, True) diff --git a/src/backend/core/mda/webhook_payload.py b/src/backend/core/mda/webhook_payload.py index 7faaee156..0c1cf79cb 100644 --- a/src/backend/core/mda/webhook_payload.py +++ b/src/backend/core/mda/webhook_payload.py @@ -39,7 +39,11 @@ def _strip_body_part(part: Dict[str, Any]) -> Dict[str, Any]: def build_jmap_email( - parsed_email: JmapEmail, *, include_body: bool = True + parsed_email: JmapEmail, + *, + include_body: bool = True, + message_id: Optional[str] = None, + thread_id: Optional[str] = None, ) -> Dict[str, Any]: """Project the parsed JMAP ``Email`` object into the webhook payload. @@ -47,9 +51,16 @@ def build_jmap_email( (RFC 8621 §4.1), so this is mostly a copy. We stamp ``receivedAt`` (the moment the webhook fires) and strip the parser's project extensions (``_ext`` and per-part ``content`` / ``sha256``) so the - body is strict JMAP. Storage-time fields (``id``, ``blobId``, - ``threadId``, ``mailboxIds``, ``keywords``) don't exist at - webhook-fire time and are simply absent. + body is strict JMAP. + + Storage-time fields are populated only when the persisted ``Message`` + exists — i.e. the non-blocking ``message.delivered`` path, which fires + after creation and passes ``message_id`` / ``thread_id`` (also sent as + ``X-StMsg-Message-Id`` / ``X-StMsg-Thread-Id`` headers). The blocking + ``message.inbound`` / ``message.delivering`` paths fire *before* the row + exists, pass neither, and so omit them. ``blobId`` / ``mailboxIds`` / + ``keywords`` stay absent everywhere: they'd need a JMAP blob endpoint we + don't expose and a folder/flag mapping we haven't designed. With ``include_body=False`` the body parts, ``bodyValues`` and ``attachments`` are dropped — receivers get a notification-only @@ -61,6 +72,11 @@ def build_jmap_email( email: Dict[str, Any] = dict(parsed_email) email.pop("_ext", None) # project extension, not strict JMAP email["receivedAt"] = _utcdate(datetime.now(timezone.utc)) + # Present only on the post-creation (``message.delivered``) path. + if message_id is not None: + email["id"] = message_id + if thread_id is not None: + email["threadId"] = thread_id if include_body: email["textBody"] = [ diff --git a/src/backend/core/migrations/0032_inboundmessage_blob_envelope.py b/src/backend/core/migrations/0032_inboundmessage_blob_envelope.py new file mode 100644 index 000000000..3a2447c3f --- /dev/null +++ b/src/backend/core/migrations/0032_inboundmessage_blob_envelope.py @@ -0,0 +1,66 @@ +# Generated by Django 5.2.11 on 2026-06-27 17:18 + +import django.db.models.deletion +from django.db import migrations, models + + +def _raw_data_to_blob(apps, schema_editor): + """Preserve in-flight queue rows across the raw_data -> blob switch. + + Rows queued by the pre-blob code hold their only copy of the message in the + inline ``raw_data`` column, so committing each to a Blob here keeps + in-flight / deferred / abandoned mail from being lost when ``raw_data`` is + dropped below. No-op on a fresh database (empty queue), so the real + ``create_blob`` (imported lazily) only runs on an actual upgrade, where the + current code is authoritative. + """ + # pylint: disable=import-outside-toplevel + from core.models import Blob + + InboundMessage = apps.get_model("core", "inboundmessage") + rows = InboundMessage.objects.filter(blob__isnull=True).exclude(raw_data=None) + for inbound in rows.iterator(): + raw = bytes(inbound.raw_data) if inbound.raw_data else b"" + if not raw: + continue + inbound.blob = Blob.objects.create_blob( + content=raw, content_type="message/rfc822" + ) + inbound.save(update_fields=["blob"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0031_message_mime_id_index'), + ] + + operations = [ + migrations.AddField( + model_name='inboundmessage', + name='abandoned_at', + field=models.DateTimeField(blank=True, help_text='Set when processing was permanently abandoned after the retry window; the row is kept for inspection/replay and skipped by the retry sweep.', null=True, verbose_name='abandoned at'), + ), + migrations.AddField( + model_name='inboundmessage', + name='blob', + field=models.ForeignKey(blank=True, help_text='Content-addressed blob holding the raw message bytes.', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='inbound_messages', to='core.blob'), + ), + migrations.AddField( + model_name='inboundmessage', + name='envelope', + field=models.JSONField(blank=True, default=dict, help_text='Structured SMTP/provenance envelope for this delivery.', verbose_name='envelope'), + ), + # Move in-flight rows' bytes into a blob BEFORE dropping raw_data, so + # no queued/deferred/abandoned message is lost on upgrade. + migrations.RunPython(_raw_data_to_blob, migrations.RunPython.noop), + migrations.RemoveField( + model_name='inboundmessage', + name='raw_data', + ), + migrations.AddField( + model_name='message', + name='postmark', + field=models.JSONField(blank=True, help_text='Sparse per-delivery pipeline record.', null=True, verbose_name='postmark'), + ), + ] diff --git a/src/backend/core/migrations/0032_inboundmessage_blob_inboundmessage_is_internal_and_more.py b/src/backend/core/migrations/0032_inboundmessage_blob_inboundmessage_is_internal_and_more.py deleted file mode 100644 index 019daff4d..000000000 --- a/src/backend/core/migrations/0032_inboundmessage_blob_inboundmessage_is_internal_and_more.py +++ /dev/null @@ -1,38 +0,0 @@ -# Generated by Django 5.2.11 on 2026-06-27 17:18 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0031_message_mime_id_index'), - ] - - operations = [ - migrations.AddField( - model_name='inboundmessage', - name='abandoned_at', - field=models.DateTimeField(blank=True, help_text='Set when processing was permanently abandoned after the retry window; the row is kept for inspection/replay and skipped by the retry sweep.', null=True, verbose_name='abandoned at'), - ), - migrations.AddField( - model_name='inboundmessage', - name='blob', - field=models.ForeignKey(blank=True, help_text='Existing blob holding the message bytes (internal mail)', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='inbound_messages', to='core.blob'), - ), - migrations.AddField( - model_name='inboundmessage', - name='is_internal', - field=models.BooleanField(default=False, help_text='Internal mailbox-to-mailbox delivery (skips spam checking)', verbose_name='is internal'), - ), - migrations.AlterField( - model_name='inboundmessage', - name='raw_data', - field=models.BinaryField(blank=True, help_text='Raw email message bytes (external mail; null when backed by blob)', null=True, verbose_name='raw data'), - ), - migrations.AddConstraint( - model_name='inboundmessage', - constraint=models.CheckConstraint(condition=models.Q(models.Q(('blob__isnull', True), ('raw_data__isnull', False)), models.Q(('blob__isnull', False), ('raw_data__isnull', True)), _connector='OR'), name='inboundmessage_exactly_one_source'), - ), - ] diff --git a/src/backend/core/models.py b/src/backend/core/models.py index 9339046b0..4c7187667 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -2084,6 +2084,24 @@ class Message(BaseModel): blank=True, related_name="messages", ) + # Sparse structured record of what our inbound pipeline determined about + # this delivery — NULL in the happy path. Holds only data that is either + # computed AFTER the bytes are committed (so baking it would fork the blob) + # or per-recipient (so baking it would break blob dedup). Flat, short keys: + # "auth": "none" | "fail" sender-auth verdict (verified ⇒ absent) + # "processing": "fail" force-delivered past the deferral window + # "rcpt_to": "" envelope RCPT TO, only when it diverges + # from the MIME To/Cc (BCC / alias / catch-all) + # Immutable ingest-time facts (ip/helo/hostname, MAIL FROM, widget referer) + # live in headers in the blob instead — see the ingest paths. Keep values + # small and bounded; large/regenerated data (e.g. AI summaries) belongs in + # its own field, never here. Surfaced via ``get_stmsg_headers``. + postmark = models.JSONField( + "postmark", + null=True, + blank=True, + help_text="Sparse per-delivery pipeline record.", + ) objects = MessageManager() @@ -2150,17 +2168,44 @@ def get_mime_headers(self) -> list[EmailHeader]: return self.get_parsed_data().get("headers", []) def get_stmsg_headers(self) -> Dict[str, str]: - """Return the ``X-StMsg-*`` headers as ``{suffix_lower: value}``. - - ``X-StMsg-*`` headers are stamped by our MTA pipeline (one per - message) and any sender-supplied copies are stripped before - parsing; first occurrence wins on duplicates. + """Return the pipeline hint headers as ``{suffix_lower: value}``. + + Two sources, unioned, during the transition away from MIME-baked + verdicts: + + * **Legacy (bytes):** ``X-StMsg-*`` headers parsed from the stored + MIME. Serves every message written before ``postmark`` existed + (never backfilled in bulk), plus ``widget-referer``, which stays a + header permanently. Sender-supplied copies are stripped at ingest, + so this namespace is ours; first occurrence wins on duplicates. + * **Structured (``postmark``):** verdicts computed after the bytes are + committed now live here. Projected onto the same suffix shape and + overlaid LAST so the authoritative structured value wins over any + residual/legacy header. + + A future release drops the legacy branch once ``postmark`` is fully + backfilled (see the ``backfill_postmark`` management command). """ result: Dict[str, str] = {} for h in self.get_parsed_data().get("headers", []): name = h.get("name", "") if name.lower().startswith("x-stmsg-"): result.setdefault(name[len("x-stmsg-") :].lower(), h.get("value", "")) + + postmark = self.postmark or {} + if postmark.get("auth"): + # ``sender-auth`` carries the verdict value verbatim ("none"/"fail"), + # matching the legacy header. + result["sender-auth"] = postmark["auth"] + if postmark.get("processing"): + # ``processing-failed`` is a boolean flag; normalise to the legacy + # header value ("true") regardless of the structured reason. + result["processing-failed"] = "true" + if postmark.get("spam"): + # rspamd flagged probable spam but below the Junk threshold — the + # UI renders a "suspected spam" marker while the message stays in + # the inbox. + result["spam"] = postmark["spam"] return result def generate_mime_id(self) -> str: @@ -2241,40 +2286,38 @@ class InboundMessage(BaseModel): on_delete=models.CASCADE, related_name="inbound_messages", ) - # Two mutually-exclusive sources for the message bytes (enforced by - # the ``inboundmessage_exactly_one_source`` constraint below): - # - ``raw_data``: external mail arrives from the MTA as loose - # bytes with no Blob yet, so we store them inline (plaintext, - # transient — deleted once the message is created). - # - ``blob``: internal mail already has a committed, encrypted, - # deduped Blob (the sender's ``Message.blob``). We reference it - # instead of copying the bytes — no second plaintext copy, and - # ``get_content()`` reads it back transparently. - raw_data = models.BinaryField( - "raw data", - null=True, - blank=True, - help_text="Raw email message bytes (external mail; null when backed by blob)", - ) - # PROTECT mirrors ``Message.blob``: the GC sweep is the only - # authorised deleter and clears references first. ``is_referenced`` - # counts this FK, so a referenced blob is never collected. + # Sole byte source: a content-addressed, encrypted, deduped Blob created + # at ingest. (External mail used to be stored inline as a plaintext + # ``raw_data`` BinaryField; now every path is blob-backed, so a message to + # N recipients shares ONE blob from the moment it's queued, and nothing + # sits in plaintext.) PROTECT mirrors ``Message.blob``: only the GC sweep + # deletes, and it clears references first; ``is_referenced`` counts this + # FK. Nullable at the DB level for migration safety, but always set by the + # ingest paths (``deliver_inbound_message``). blob = models.ForeignKey( "Blob", on_delete=models.PROTECT, null=True, blank=True, related_name="inbound_messages", - help_text="Existing blob holding the message bytes (internal mail)", - ) - # Internal mail (mailbox-to-mailbox) is trusted: the task pre-sets - # ``is_spam=False`` so the spam steps no-op while the user-webhook - # steps still run. Kept explicit rather than inferred from ``blob`` - # so storage mechanism stays decoupled from trust semantics. - is_internal = models.BooleanField( - "is internal", - default=False, - help_text="Internal mailbox-to-mailbox delivery (skips spam checking)", + help_text="Content-addressed blob holding the raw message bytes.", + ) + # Structured SMTP / provenance envelope for this delivery — carried as + # data, NOT injected into the message bytes, so the blob stays pristine and + # shared across recipients. Shape: + # {"origin": "mta"|"internal"|"widget"|"import", + # "mail_from": , "rcpt_to": , + # "ip": , "helo": , "hostname": } + # SMTP fields are present only for ``origin="mta"``. Consumed by the spam + # pipeline (rspamd envelope) and the autoreply bounce check. ``origin`` is + # the trust discriminator (see ``is_internal``) and is set EXPLICITLY by + # each ingest path — never inferred from which fields happen to be present + # (inferring "internal" from missing SMTP data would fail open to trusted). + envelope = models.JSONField( + "envelope", + default=dict, + blank=True, + help_text="Structured SMTP/provenance envelope for this delivery.", ) channel = models.ForeignKey( "Channel", @@ -2289,7 +2332,7 @@ class InboundMessage(BaseModel): help_text="Error message if processing failed", ) # Terminal-failure marker. Set when processing is permanently given up - # after the quarantine window (a poison message that can never be + # after the deferral window (a poison message that can never be # parsed/created). The row and its bytes are KEPT — never deleted at # abandon time — so the mail stays recoverable; the retry sweep skips # rows with this set, so the pipeline (and user webhooks) stop re-firing. @@ -2315,30 +2358,29 @@ class Meta: indexes = [ models.Index(fields=["created_at"]), ] - constraints = [ - # Exactly one byte source: inline raw_data XOR a blob FK. - models.CheckConstraint( - check=( - models.Q(raw_data__isnull=False, blob__isnull=True) - | models.Q(raw_data__isnull=True, blob__isnull=False) - ), - name="inboundmessage_exactly_one_source", - ), - ] def __str__(self): return f"InboundMessage {self.id} - {self.mailbox}" + @property + def is_internal(self) -> bool: + """Whether this is a trusted same-instance mailbox-to-mailbox delivery + (the spam steps no-op for it while user webhooks still run). + + Derived from the EXPLICIT ``envelope["origin"]`` discriminator set by + the internal-delivery path — never inferred from the absence of SMTP + fields, which would fail open to "trusted" for a stripped external + message. + """ + return (self.envelope or {}).get("origin") == "internal" + def get_raw_bytes(self) -> bytes: - """Return the message bytes regardless of storage backing. + """Return the raw message bytes from the backing blob. - Internal mail references an existing (encrypted, deduped) blob; - external mail stores the bytes inline. ``Blob.get_content()`` - transparently decrypts and handles tiered storage. + ``Blob.get_content()`` transparently decrypts and handles tiered + storage. """ - if self.blob_id is not None: - return self.blob.get_content() - return bytes(self.raw_data) + return self.blob.get_content() class BlobManager(models.Manager): diff --git a/src/backend/core/tests/api/test_channel_scope_level.py b/src/backend/core/tests/api/test_channel_scope_level.py index 9ee357473..373a9d072 100644 --- a/src/backend/core/tests/api/test_channel_scope_level.py +++ b/src/backend/core/tests/api/test_channel_scope_level.py @@ -1008,7 +1008,13 @@ def test_user_in_post_body_is_ignored(self, api_client): created = models.Channel.objects.get(pk=response.data["id"]) assert created.user_id == alice.id - @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["api_key", "webhook"]) + # DEBUG=True skips the create-time SSRF DNS lookup (Test runs DEBUG=False), + # so the placeholder ``hook.example.com`` host doesn't 400 on resolution — + # this test covers the scope/serializer path, not SSRF (see + # TestWebhookChannelCreateSSRF in test_channels.py for that). + @override_settings( + FEATURE_MAILBOX_ADMIN_CHANNELS=["api_key", "webhook"], DEBUG=True + ) def test_personal_webhook_channel(self, api_client): """Webhooks ARE creatable as personal channels — once the type is enabled in FEATURE_MAILBOX_ADMIN_CHANNELS. The type is intentionally diff --git a/src/backend/core/tests/api/test_channels.py b/src/backend/core/tests/api/test_channels.py index c2c531cd8..4074ccbf9 100644 --- a/src/backend/core/tests/api/test_channels.py +++ b/src/backend/core/tests/api/test_channels.py @@ -1,8 +1,9 @@ """Tests for the channel API endpoints.""" -# pylint: disable=redefined-outer-name, unused-argument, too-many-public-methods, import-outside-toplevel +# pylint: disable=redefined-outer-name, unused-argument, too-many-public-methods, import-outside-toplevel, too-many-lines import uuid +from unittest.mock import patch from django.test import override_settings from django.urls import reverse @@ -20,6 +21,7 @@ MailDomainFactory, UserFactory, ) +from core.services.ssrf import SSRFValidationError @pytest.fixture @@ -570,7 +572,15 @@ def test_unrelated_settings_keys_pass_through(self, api_client, mailbox): @pytest.mark.django_db class TestWebhookChannelSettings: - """Validation of the outbound-webhook-specific settings fields.""" + """Validation of the outbound-webhook-specific settings fields. + + These target the non-SSRF settings validation (format / trigger / + auth_method / secret rotation) with placeholder hosts like + ``hook.example.com`` that don't resolve. The Test config runs DEBUG=False, + under which the serializer does a live ``validate_hostname`` DNS lookup and + 400s on an unresolvable host — so the create/update tests pin DEBUG=True to + skip that config-time SSRF layer (covered on its own in + ``TestWebhookChannelCreateSSRF``).""" URL_KEY = "url" @@ -586,7 +596,7 @@ def _post(self, api_client, mailbox, settings): format="json", ) - @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=True) def test_create_minimal_webhook(self, api_client, mailbox): """A JWT webhook surfaces its one-time ``secret`` on create.""" response = self._post( @@ -603,7 +613,7 @@ def test_create_minimal_webhook(self, api_client, mailbox): assert response.data.get("secret") assert "api_key" not in response.data - @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=True) def test_create_minimal_webhook_api_key(self, api_client, mailbox): """An api_key webhook surfaces its one-time ``api_key``.""" response = self._post( @@ -620,7 +630,7 @@ def test_create_minimal_webhook_api_key(self, api_client, mailbox): assert response.data.get("api_key") assert "secret" not in response.data - @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=True) def test_regenerate_secret_jwt_webhook(self, api_client, mailbox): """Rotating a JWT webhook returns a fresh ``secret``.""" create = self._post( @@ -648,7 +658,7 @@ def test_regenerate_secret_jwt_webhook(self, api_client, mailbox): assert new_secret != original_secret assert "api_key" not in response.data - @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=True) def test_regenerate_secret_api_key_webhook(self, api_client, mailbox): """Rotating an api_key webhook returns a fresh ``api_key``.""" create = self._post( @@ -676,7 +686,7 @@ def test_regenerate_secret_api_key_webhook(self, api_client, mailbox): assert new_key != original_key assert "secret" not in response.data - @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=True) def test_create_with_all_dispatcher_options(self, api_client, mailbox): """A webhook channel accepts the full set of dispatcher options.""" response = self._post( @@ -694,7 +704,7 @@ def test_create_with_all_dispatcher_options(self, api_client, mailbox): assert channel.settings["trigger"] == "message.inbound" assert channel.settings["format"] == "jmap" - @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=True) def test_rejects_invalid_format(self, api_client, mailbox): """An unknown webhook ``format`` is rejected with HTTP 400.""" response = self._post( @@ -709,7 +719,7 @@ def test_rejects_invalid_format(self, api_client, mailbox): ) assert response.status_code == status.HTTP_400_BAD_REQUEST - @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=True) def test_accepts_jmap_metadata_format(self, api_client, mailbox): """The ``jmap_metadata`` format is a valid webhook format.""" response = self._post( @@ -724,7 +734,7 @@ def test_accepts_jmap_metadata_format(self, api_client, mailbox): ) assert response.status_code == status.HTTP_201_CREATED, response.content - @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=True) def test_rejects_invalid_trigger(self, api_client, mailbox): """An unknown webhook ``trigger`` is rejected with HTTP 400.""" response = self._post( @@ -738,7 +748,7 @@ def test_rejects_invalid_trigger(self, api_client, mailbox): ) assert response.status_code == status.HTTP_400_BAD_REQUEST - @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=True) def test_rejects_missing_trigger(self, api_client, mailbox): """A webhook channel without a ``trigger`` is rejected with HTTP 400.""" response = self._post( @@ -751,7 +761,7 @@ def test_rejects_missing_trigger(self, api_client, mailbox): ) assert response.status_code == status.HTTP_400_BAD_REQUEST - @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=True) def test_rejects_patch_with_invalid_trigger(self, api_client, mailbox): """A PATCH that touches only ``settings`` must still re-run the webhook validator (same airtight rule as api_key scopes).""" @@ -782,7 +792,7 @@ def test_rejects_patch_with_invalid_trigger(self, api_client, mailbox): ) assert response.status_code == status.HTTP_400_BAD_REQUEST - @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=True) def test_partial_patch_preserves_auth_method(self, api_client, mailbox): """A settings PATCH that omits ``auth_method`` keeps the existing one instead of being rejected — auth_method pairs with the stored @@ -820,7 +830,7 @@ def test_partial_patch_preserves_auth_method(self, api_client, mailbox): assert channel.settings["url"] == "https://hook.example.com/changed" assert channel.settings["auth_method"] == "api_key" - @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=True) def test_regenerate_secret_rejects_unknown_auth_method(self, api_client, mailbox): """Rotating a webhook whose auth_method we can't surface is rejected up front, so the old secret is never invalidated for a new one the @@ -847,7 +857,7 @@ def test_regenerate_secret_rejects_unknown_auth_method(self, api_client, mailbox # Old secret untouched — rotation never ran. assert channel.encrypted_settings["secret"] == "whsec_original" - @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=True) def test_rejects_missing_url(self, api_client, mailbox): """A webhook channel without settings.url is rejected with 400.""" response = self._post( @@ -857,7 +867,7 @@ def test_rejects_missing_url(self, api_client, mailbox): ) assert response.status_code == status.HTTP_400_BAD_REQUEST - @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"]) + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=True) @pytest.mark.parametrize( "bad_url", [ @@ -895,3 +905,117 @@ def test_rejects_plain_http_outside_debug(self, api_client, mailbox): }, ) assert response.status_code == status.HTTP_400_BAD_REQUEST, response.content + + +@pytest.mark.django_db +class TestWebhookChannelCreateSSRF: + """Create/update-time SSRF validation of webhook ``settings.url``. + + A fail-fast UX layer (NOT the security boundary — ``SSRFSafeSession`` + re-validates with IP pinning on every POST): outside DEBUG, the + serializer resolves the host via ``validate_hostname`` and 400s on an + internal / IP-literal / unresolvable target. Under DEBUG (the Test + default) it is skipped so local-dev receivers work. + """ + + def _post(self, api_client, mailbox, settings): + url = reverse("mailbox-channels-list", kwargs={"mailbox_id": mailbox.id}) + return api_client.post( + url, + data={"name": "wh", "type": "webhook", "settings": settings}, + format="json", + ) + + _SETTINGS = { + "url": "https://hook.example.com/in", + "trigger": "message.delivered", + "auth_method": "jwt", + } + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=False) + @patch("core.api.serializers.validate_hostname") + def test_rejects_internal_host(self, mock_validate, api_client, mailbox): + """A host resolving to a private/internal address is a 400 with a + ``settings`` error.""" + mock_validate.side_effect = SSRFValidationError( + "hook.example.com resolves to private IP address" + ) + response = self._post(api_client, mailbox, self._SETTINGS) + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.content + assert "settings" in response.data + mock_validate.assert_called_once_with("hook.example.com") + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=False) + @patch("core.api.serializers.validate_hostname") + def test_rejects_unresolvable_host(self, mock_validate, api_client, mailbox): + """A host that doesn't resolve is rejected (would silently stall mail + at dispatch otherwise).""" + mock_validate.side_effect = SSRFValidationError("Unable to resolve hostname") + response = self._post(api_client, mailbox, self._SETTINGS) + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.content + assert "settings" in response.data + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=False) + @patch("core.api.serializers.validate_hostname") + def test_accepts_public_host(self, mock_validate, api_client, mailbox): + """A host that validates to a public IP is accepted (201).""" + mock_validate.return_value = ["93.184.216.34"] + response = self._post(api_client, mailbox, self._SETTINGS) + assert response.status_code == status.HTTP_201_CREATED, response.content + mock_validate.assert_called_once_with("hook.example.com") + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=True) + @patch("core.api.serializers.validate_hostname") + def test_skipped_under_debug(self, mock_validate, api_client, mailbox): + """Under DEBUG the SSRF lookup is skipped entirely — creation with an + unresolvable fake host still 201s and ``validate_hostname`` is never + called. (The Test config runs DEBUG=False, so this pins DEBUG=True to + exercise the local-dev escape hatch.)""" + response = self._post( + api_client, + mailbox, + { + "url": "https://totally-fake-host.invalid/in", + "trigger": "message.delivered", + "auth_method": "jwt", + }, + ) + assert response.status_code == status.HTTP_201_CREATED, response.content + mock_validate.assert_not_called() + + @override_settings(FEATURE_MAILBOX_ADMIN_CHANNELS=["webhook"], DEBUG=False) + @patch("core.api.serializers.validate_hostname") + def test_validation_runs_on_patch(self, mock_validate, api_client, mailbox): + """A settings PATCH that changes the url must re-run the SSRF check — + otherwise an admin could PATCH past the create-time guard.""" + # Build the channel directly so creation doesn't itself invoke the + # (patched) validator we want to scope to the PATCH. + channel = ChannelFactory( + type="webhook", + mailbox=mailbox, + settings={ + "url": "https://hook.example.com/in", + "trigger": "message.delivered", + "auth_method": "jwt", + }, + ) + mock_validate.side_effect = SSRFValidationError( + "evil.internal resolves to private IP address" + ) + url = reverse( + "mailbox-channels-detail", + kwargs={"mailbox_id": mailbox.id, "pk": channel.id}, + ) + response = api_client.patch( + url, + data={ + "settings": { + "url": "https://evil.internal/in", + "trigger": "message.delivered", + "auth_method": "jwt", + } + }, + format="json", + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.content + assert "settings" in response.data diff --git a/src/backend/core/tests/api/test_inbound_mta.py b/src/backend/core/tests/api/test_inbound_mta.py index ed3c780ed..ac49ee05c 100644 --- a/src/backend/core/tests/api/test_inbound_mta.py +++ b/src/backend/core/tests/api/test_inbound_mta.py @@ -152,17 +152,99 @@ def test_valid_email_submission( assert response.status_code == status.HTTP_200_OK assert response.json() == {"status": "ok", "delivered": len(recipients)} - mock_parse.assert_called_once_with(sample_email) + # The MTA path bakes an authoritative Return-Path (envelope MAIL FROM; + # "<>" for the null sender here) onto the bytes before parsing. + mock_parse.assert_called_once_with(b"Return-Path: <>\r\n" + sample_email) assert mock_deliver.call_count == len(recipients) - mock_deliver.assert_any_call(email, ANY, sample_email) - mock_deliver.assert_any_call("another@example.com", ANY, sample_email) + # Each recipient gets the SMTP envelope (origin=mta + per-recipient + # RCPT TO); the SMTP fields default to "" when the MTA omits them. + for rcpt in recipients: + mock_deliver.assert_any_call( + rcpt, + ANY, + b"Return-Path: <>\r\n" + sample_email, + envelope={ + "origin": "mta", + "mail_from": "", + "ip": "", + "helo": "", + "hostname": "", + "rcpt_to": rcpt, + }, + ) first_call_args = mock_deliver.call_args_list[0][0] second_call_args = mock_deliver.call_args_list[1][0] assert first_call_args[1]["subject"] == "Test Email" assert second_call_args[1]["subject"] == "Test Email" + @patch("core.mda.inbound.process_inbound_message_task.delay") + def test_multi_recipient_delivery_shares_one_blob( + self, _mock_delay, api_client, valid_jwt_token + ): + """One body delivered to N recipients dedups to a single blob — E2E + through the MTA endpoint. + + The endpoint fans one ``deliver`` (N ``original_recipients``, one body) + out to N ``InboundMessage`` rows; blob-at-ingest dedups the identical + bytes by content hash to ONE blob, while each row keeps its own + per-recipient RCPT envelope. Nothing duplicated, nothing in plaintext. + """ + mailboxes = [factories.MailboxFactory() for _ in range(3)] + recipients = [f"{m.local_part}@{m.domain.name}" for m in mailboxes] + raw = b"From: sender@example.com\r\nSubject: shared\r\n\r\nbody" + token = valid_jwt_token(raw, {"original_recipients": recipients}) + + response = api_client.post( + "/api/v1.0/inbound/mta/deliver/", + data=raw, + content_type="message/rfc822", + HTTP_AUTHORIZATION=f"Bearer {token}", + ) + + assert response.status_code == status.HTTP_200_OK + rows = list(models.InboundMessage.objects.all()) + assert len(rows) == 3 + assert len({r.blob_id for r in rows}) == 1 # one shared blob + assert {r.envelope["rcpt_to"] for r in rows} == set(recipients) + + @patch("core.api.viewsets.inbound.mta.deliver_inbound_message") + @patch("core.api.viewsets.inbound.mta.parse_email") + def test_forged_return_path_stripped_and_rewritten( + self, mock_parse, mock_deliver, api_client, valid_jwt_token + ): + """A sender-supplied Return-Path is stripped and replaced with our + authoritative one (the real envelope MAIL FROM) at ingest.""" + mock_parse.return_value = {"subject": "t"} + mock_deliver.return_value = True + forged = ( + b"Return-Path: \r\n" + b"From: s@example.com\r\nSubject: t\r\n\r\nbody" + ) + token = valid_jwt_token( + forged, + { + "original_recipients": ["rcpt@example.com"], + "sender": "real@example.com", + }, + ) + + response = api_client.post( + "/api/v1.0/inbound/mta/deliver/", + data=forged, + content_type="message/rfc822", + HTTP_AUTHORIZATION=f"Bearer {token}", + ) + + assert response.status_code == status.HTTP_200_OK + baked = mock_parse.call_args[0][0] + # Our authoritative Return-Path is first; the forged one is gone. + assert baked.startswith(b"Return-Path: \r\n") + assert b"evil@attacker.test" not in baked + # The MAIL FROM also reaches the envelope for the pipeline. + assert mock_deliver.call_args[1]["envelope"]["mail_from"] == "real@example.com" + @patch("core.api.viewsets.inbound.mta.deliver_inbound_message") @patch("core.api.viewsets.inbound.mta.parse_email") def test_email_parse_failure( @@ -188,7 +270,9 @@ def test_email_parse_failure( assert response.status_code == status.HTTP_400_BAD_REQUEST assert response.json() == {"status": "error", "detail": "Failed to parse email"} - mock_parse.assert_called_once_with(sample_email) + # The MTA path bakes an authoritative Return-Path (envelope MAIL FROM; + # "<>" for the null sender here) onto the bytes before parsing. + mock_parse.assert_called_once_with(b"Return-Path: <>\r\n" + sample_email) mock_deliver.assert_not_called() # Delivery should not be attempted @patch("core.api.viewsets.inbound.mta.deliver_inbound_message") @@ -230,7 +314,9 @@ def test_delivery_partial_failure( "failed": 1, "results": expected_results, } - mock_parse.assert_called_once_with(sample_email) + # The MTA path bakes an authoritative Return-Path (envelope MAIL FROM; + # "<>" for the null sender here) onto the bytes before parsing. + mock_parse.assert_called_once_with(b"Return-Path: <>\r\n" + sample_email) assert mock_deliver.call_count == 2 # Called for both recipients @patch("core.api.viewsets.inbound.mta.deliver_inbound_message") @@ -268,7 +354,9 @@ def test_delivery_total_failure( "detail": "Failed to deliver message to any recipient", "results": expected_results, } - mock_parse.assert_called_once_with(sample_email) + # The MTA path bakes an authoritative Return-Path (envelope MAIL FROM; + # "<>" for the null sender here) onto the bytes before parsing. + mock_parse.assert_called_once_with(b"Return-Path: <>\r\n" + sample_email) assert mock_deliver.call_count == 2 # Called for both def test_invalid_content_type( diff --git a/src/backend/core/tests/commands/test_backfill_postmark.py b/src/backend/core/tests/commands/test_backfill_postmark.py new file mode 100644 index 000000000..3da36746a --- /dev/null +++ b/src/backend/core/tests/commands/test_backfill_postmark.py @@ -0,0 +1,72 @@ +"""Tests for the ``backfill_postmark`` management command.""" + +from io import StringIO + +from django.core.management import call_command + +import pytest + +from core import factories, models + +_WITH_VERDICTS = ( + b"X-StMsg-Sender-Auth: fail\r\n" + b"X-StMsg-Processing-Failed: true\r\n" + b"From: s@example.com\r\nSubject: t\r\n\r\nbody" +) +_CLEAN = b"From: s@example.com\r\nSubject: t\r\n\r\nbody" + + +@pytest.mark.django_db +class TestBackfillPostmark: + """Progressive backfill of ``Message.postmark`` from legacy X-StMsg bytes.""" + + def test_verdicts_projected_from_legacy_headers(self): + """Baked X-StMsg verdicts are projected onto postmark keys.""" + message = factories.MessageFactory(raw_mime=_WITH_VERDICTS) + assert message.postmark is None + + call_command("backfill_postmark", stdout=StringIO()) + + message.refresh_from_db() + assert message.postmark == {"auth": "fail", "processing": "fail"} + + def test_clean_message_marked_scanned_as_empty(self): + """A legacy message with no X-StMsg headers is set to ``{}`` so it is + not re-read, and reads back the same as NULL.""" + message = factories.MessageFactory(raw_mime=_CLEAN) + + call_command("backfill_postmark", stdout=StringIO()) + + message.refresh_from_db() + assert message.postmark == {} + assert message.get_stmsg_headers() == {} + + def test_dry_run_writes_nothing(self): + """--dry-run reports but does not persist.""" + message = factories.MessageFactory(raw_mime=_WITH_VERDICTS) + + call_command("backfill_postmark", "--dry-run", stdout=StringIO()) + + message.refresh_from_db() + assert message.postmark is None + + def test_limit_bounds_the_batch(self): + """--limit caps how many rows are processed per run.""" + for _ in range(3): + factories.MessageFactory(raw_mime=_WITH_VERDICTS) + + call_command("backfill_postmark", "--limit", "2", stdout=StringIO()) + + remaining = models.Message.objects.filter(postmark__isnull=True).count() + assert remaining == 1 + + def test_already_backfilled_rows_are_skipped(self): + """A row with a non-NULL postmark is left untouched (idempotent).""" + message = factories.MessageFactory(raw_mime=_WITH_VERDICTS) + message.postmark = {"auth": "none"} + message.save(update_fields=["postmark"]) + + call_command("backfill_postmark", stdout=StringIO()) + + message.refresh_from_db() + assert message.postmark == {"auth": "none"} diff --git a/src/backend/core/tests/mda/test_autoreply.py b/src/backend/core/tests/mda/test_autoreply.py index 47d0aaa70..9a3f63a72 100644 --- a/src/backend/core/tests/mda/test_autoreply.py +++ b/src/backend/core/tests/mda/test_autoreply.py @@ -185,17 +185,32 @@ def test_auto_submitted_no_with_parameters(self): class TestIsAutoReplyMessageExtended: - """Tests for _is_auto_reply_message: Return-Path, List-*, and loop headers.""" - - def test_return_path_null(self): - """Return-Path: <> (null sender) is detected.""" + """Tests for _is_auto_reply_message: null envelope sender, List-*, loop + headers.""" + + def test_envelope_null_sender_is_bounce(self): + """Envelope MAIL FROM ``<>`` (null sender) is detected as a bounce.""" + assert _is_auto_reply_message({}, {"mail_from": "<>"}) is True + + def test_envelope_empty_sender_is_bounce(self): + """An explicitly empty envelope MAIL FROM is a bounce too.""" + assert _is_auto_reply_message({}, {"mail_from": ""}) is True + + def test_envelope_real_sender_is_not_bounce(self): + """A real envelope sender is not a bounce (authoritative, not a + forgeable Return-Path header).""" + # A forged Return-Path header must NOT flip the verdict — only the + # SMTP envelope is trusted. parsed_email = {"headers": [{"name": "Return-Path", "value": "<>"}]} - assert _is_auto_reply_message(parsed_email) is True + assert ( + _is_auto_reply_message(parsed_email, {"mail_from": "user@example.com"}) + is False + ) - def test_return_path_empty(self): - """Return-Path with empty value is detected.""" - parsed_email = {"headers": [{"name": "Return-Path", "value": ""}]} - assert _is_auto_reply_message(parsed_email) is True + def test_absent_envelope_mail_from_is_not_flagged(self): + """No envelope / no ``mail_from`` key → not treated as a bounce.""" + assert _is_auto_reply_message({}, {}) is False + assert _is_auto_reply_message({}, None) is False def test_list_post_header(self): """List-Post header is detected.""" @@ -545,6 +560,18 @@ def test_skip_bounce_sender(self, mailbox, autoreply_template): } assert should_send_autoreply(mailbox, parsed) is None + def test_skip_null_envelope_sender_bounce(self, mailbox, autoreply_template): + """A null envelope sender (MAIL FROM ``<>``) suppresses the autoreply, + even when the MIME From looks like a normal address (RFC 3834).""" + parsed = { + "from": [{"email": "real-looking@example.com"}], + "to": [{"email": str(mailbox)}], + "headers": [], + } + assert ( + should_send_autoreply(mailbox, parsed, envelope={"mail_from": "<>"}) is None + ) + def test_skip_owner_prefix_sender(self, mailbox, autoreply_template): """owner-list@ sender does not trigger autoreply.""" parsed = { diff --git a/src/backend/core/tests/mda/test_dispatch_webhooks.py b/src/backend/core/tests/mda/test_dispatch_webhooks.py index f6cd53398..80f34bac4 100644 --- a/src/backend/core/tests/mda/test_dispatch_webhooks.py +++ b/src/backend/core/tests/mda/test_dispatch_webhooks.py @@ -5,7 +5,6 @@ # pylint: disable=use-implicit-booleaness-not-comparison import hashlib -import hmac import json import uuid from dataclasses import dataclass, field @@ -32,6 +31,7 @@ WEBHOOK_TIMEOUT, UserWebhookStep, _classify_response_body, + _dispatch_webhook, _HttpResult, build_jmap_email, dispatch_webhook_task, @@ -41,7 +41,7 @@ webhook_steps_for_mailbox, ) from core.mda.inbound_pipeline import ( - QUARANTINE_AFTER, + DEFERRAL_MAX_AGE, Decision, InboundContext, ) @@ -69,6 +69,14 @@ class _PhaseResult: labels: Set[str] = field(default_factory=set) +def _queue_inbound(mailbox, content=b"raw", **extra): + """Create a queued InboundMessage backed by a blob (blob-at-ingest).""" + blob = models.Blob.objects.create_blob( + content=content, content_type="message/rfc822" + ) + return models.InboundMessage.objects.create(mailbox=mailbox, blob=blob, **extra) + + def dispatch_webhooks( *, phase, @@ -159,6 +167,23 @@ def _make_response(status_code: int, body: bytes = b"") -> Mock: return response +def _logged_config_error(mock_logger, channel_id) -> bool: + """True iff the patched module ``logger`` recorded a config-skip ERROR + for ``channel_id`` (the "deliver past it; fix or disable" message). + + Asserting on the patched logger rather than ``caplog`` because the + ``core`` logger does not propagate to the root handler caplog attaches + to, so caplog never sees these records. The log uses lazy ``%`` + formatting, so the channel id rides in ``call.args``, not the + pre-rendered string. + """ + for call in mock_logger.error.call_args_list: + fmt = call.args[0] if call.args else "" + if "fix or disable" in fmt and channel_id in call.args: + return True + return False + + # ChannelFactory auto-mints this for type=webhook so test channels are # never silently skipped by the dispatcher's fail-closed signing path. FACTORY_WEBHOOK_SECRET = "whsec_factory_test" @@ -303,10 +328,36 @@ def test_minimal_email_shape(self): assert email["textBody"][0]["partId"] == "1" assert email["textBody"][0]["type"] == "text/plain" assert email["hasAttachment"] is False - # Storage-time JMAP fields are absent at webhook-fire time. + # Storage-time JMAP fields are absent when no persisted Message is + # passed — i.e. the blocking, pre-creation path. for absent in ("id", "blobId", "threadId", "mailboxIds", "keywords"): assert absent not in email + def test_storage_ids_present_only_when_provided(self): + """``id`` / ``threadId`` are stamped only when the persisted Message + exists (the post-creation ``message.delivered`` path passes them); + ``blobId`` / ``mailboxIds`` / ``keywords`` stay absent regardless.""" + parsed = { + "subject": "x", + "from": [{"email": "a@x"}], + "to": [], + "cc": [], + "bcc": [], + "sentAt": None, + "messageId": ["m1"], + "textBody": [], + "htmlBody": [], + "attachments": [], + "hasAttachment": False, + "bodyValues": {}, + "headers": [], + } + email = build_jmap_email(parsed, message_id="msg-uuid", thread_id="thr-uuid") + assert email["id"] == "msg-uuid" + assert email["threadId"] == "thr-uuid" + for absent in ("blobId", "mailboxIds", "keywords"): + assert absent not in email + def test_msgid_lists_pass_through(self): """``parse_email`` already returns ``Id[]`` lists with the angle brackets stripped — the builder passes them straight through.""" @@ -556,13 +607,17 @@ def test_blocking_retries_on_429(self, mock_session, mailbox, parsed_email): ) assert outcome.decision == Decision.RETRY + @patch("core.mda.dispatch_webhooks.logger") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_blocking_retries_on_ssrf_rejection( - self, mock_session, mailbox, parsed_email + def test_blocking_ssrf_rejection_continues( + self, mock_session, mock_logger, mailbox, parsed_email ): - """SSRF rejection is a config error on our side — held for RETRY - (fix the URL and the next sweep delivers), never dropped.""" - factories.ChannelFactory( + """SSRF rejection at dispatch is a config error retry can't fix + (create-time validation already rejects internal/unresolvable URLs, + so this means a DNS rebind or a hand-edited row). Rather than stall + the whole scope's inbound for 48h, deliver the mail past the broken + webhook (CONTINUE) and log at ERROR so an admin fixes/disables it.""" + channel = factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, settings={ @@ -580,7 +635,8 @@ def test_blocking_retries_on_ssrf_rejection( raw_data=b"raw", is_spam=False, ) - assert outcome.decision == Decision.RETRY + assert outcome.decision == Decision.CONTINUE + assert _logged_config_error(mock_logger, channel.id) @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_blocking_retries_on_timeout(self, mock_session, mailbox, parsed_email): @@ -637,7 +693,7 @@ def test_blocking_retries_on_unknown_exception( self, mock_session, mailbox, parsed_email ): """Unknown transport-level errors land as RETRY — the 48-hour - quarantine window bounds how long we'll keep trying a busted + deferral window bounds how long we'll keep trying a busted receiver.""" factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, @@ -922,15 +978,14 @@ def test_constants_default(self): @pytest.mark.django_db class TestWebhookSigning: - """Every outgoing webhook is HMAC-signed; receivers verify by - recomputing HMAC-SHA256 over ``f"{ts}.{body}"`` with the channel - secret and constant-time comparing against - ``X-StMsg-Webhook-Signature``.""" + """``auth_method=jwt`` carries a single HS256 JWT in + ``Authorization: Bearer``; receivers verify it with the channel secret + and it binds the exact posted body via the ``body_sha256`` claim.""" SECRET = FACTORY_WEBHOOK_SECRET @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_eml_signature_covers_raw_body(self, mock_session, mailbox, parsed_email): + def test_eml_jwt_binds_raw_body(self, mock_session, mailbox, parsed_email): factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -952,25 +1007,18 @@ def test_eml_signature_covers_raw_body(self, mock_session, mailbox, parsed_email is_spam=False, ) headers = mock_session.return_value.post.call_args.kwargs["headers"] - ts = headers["X-StMsg-Webhook-Timestamp"] - sig_header = headers["X-StMsg-Webhook-Signature"] - scheme, _, sig = sig_header.partition("=") - assert scheme == "v1" - - expected = hmac.new( - self.SECRET.encode("utf-8"), - ts.encode("ascii") + b"." + raw, - hashlib.sha256, - ).hexdigest() - assert hmac.compare_digest(sig, expected) + token = headers["Authorization"].split(" ", 1)[1] + claims = jwt.decode(token, self.SECRET, algorithms=["HS256"]) + # For eml the posted body IS the raw bytes; the JWT binds them. + assert claims["body_sha256"] == hashlib.sha256(raw).hexdigest() @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_jmap_signature_covers_exact_serialised_bytes( + def test_jmap_jwt_binds_exact_serialised_bytes( self, mock_session, mailbox, parsed_email ): - """The body we sign MUST equal the body we POST byte-for-byte — + """The body the JWT binds MUST equal the body we POST byte-for-byte — otherwise ``requests`` could re-serialise JSON with different - separators/key order and break the signature.""" + separators/key order and break verification.""" factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -993,24 +1041,18 @@ def test_jmap_signature_covers_exact_serialised_bytes( kwargs = mock_session.return_value.post.call_args.kwargs body_bytes = kwargs["data"] assert isinstance(body_bytes, bytes) - ts = kwargs["headers"]["X-StMsg-Webhook-Timestamp"] - sig = kwargs["headers"]["X-StMsg-Webhook-Signature"].split("=", 1)[1] - expected = hmac.new( - self.SECRET.encode("utf-8"), - ts.encode("ascii") + b"." + body_bytes, - hashlib.sha256, - ).hexdigest() - assert hmac.compare_digest(sig, expected) + token = kwargs["headers"]["Authorization"].split(" ", 1)[1] + claims = jwt.decode(token, self.SECRET, algorithms=["HS256"]) + assert claims["body_sha256"] == hashlib.sha256(body_bytes).hexdigest() @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_api_key_mode_sends_only_api_key_header( + def test_api_key_mode_sends_bearer_derived_key( self, mock_session, mailbox, parsed_email ): - """auth_method=api_key: send X-StMsg-Api-Key, omit HMAC sig / JWT. - - Sending only the credential the receiver verifies keeps the - unused presentation off the wire (and out of any receiver-side - proxy/log).""" + """auth_method=api_key: present the derived key as an opaque + ``Authorization: Bearer`` token (uniform with jwt, auto-redacted by + proxies/logs). The Bearer value is the HMAC-derived key, NOT the root + secret — the root never travels on the wire.""" channel = factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -1030,22 +1072,15 @@ def test_api_key_mode_sends_only_api_key_header( is_spam=False, ) headers = mock_session.return_value.post.call_args.kwargs["headers"] - # X-StMsg-Api-Key carries the HMAC-DERIVED value, NOT the root - # secret — the root never travels on the wire. - assert headers["X-StMsg-Api-Key"] == channel.get_webhook_api_key() - assert headers["X-StMsg-Api-Key"] != self.SECRET - # The HMAC + JWT presentation is NOT sent — that's the whole - # point of the per-channel auth_method. - assert "X-StMsg-Webhook-Signature" not in headers - assert "X-StMsg-Webhook-Timestamp" not in headers - assert "Authorization" not in headers + assert headers["Authorization"] == f"Bearer {channel.get_webhook_api_key()}" + assert headers["Authorization"] != f"Bearer {self.SECRET}" + # The legacy custom header is gone. + assert "X-StMsg-Api-Key" not in headers @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_jwt_mode_sends_only_hmac_and_jwt_headers( - self, mock_session, mailbox, parsed_email - ): - """auth_method=jwt (default): HMAC sig + Authorization Bearer, - but never the raw secret as an API key header.""" + def test_jwt_mode_sends_only_bearer(self, mock_session, mailbox, parsed_email): + """auth_method=jwt (default): a single ``Authorization: Bearer`` JWT + and nothing else — no separate signature/timestamp headers.""" factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -1065,11 +1100,14 @@ def test_jwt_mode_sends_only_hmac_and_jwt_headers( is_spam=False, ) headers = mock_session.return_value.post.call_args.kwargs["headers"] - assert "X-StMsg-Webhook-Signature" in headers - assert headers["Authorization"].startswith("Bearer ") - # Receivers that verify HMAC never need the raw secret — keep it - # off the wire so it can't leak through receiver-side logs. - assert "X-StMsg-Api-Key" not in headers + # A JWT has two dots (header.payload.signature); the api_key Bearer + # is an opaque ``whk_`` token — this is how a receiver-agnostic check + # would tell them apart, though receivers know their configured method. + token = headers["Authorization"].split(" ", 1)[1] + assert token.count(".") == 2 + # The old dual-signing headers are gone — the JWT is the only proof. + assert "X-StMsg-Webhook-Signature" not in headers + assert "X-StMsg-Webhook-Timestamp" not in headers @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_missing_auth_method_fails_closed( @@ -1104,9 +1142,9 @@ def test_missing_auth_method_fails_closed( def test_api_key_value_is_derived_not_raw_secret( self, mock_session, mailbox, parsed_email ): - """The X-StMsg-Api-Key value MUST NOT be the raw root secret — + """The api_key Bearer token MUST NOT be the raw root secret — a receiver-side log leak of the API key would otherwise - compromise HMAC/JWT verification.""" + compromise JWT verification on other receivers.""" channel = factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -1125,9 +1163,10 @@ def test_api_key_value_is_derived_not_raw_secret( raw_data=b"raw", is_spam=False, ) - sent = mock_session.return_value.post.call_args.kwargs["headers"][ - "X-StMsg-Api-Key" + auth = mock_session.return_value.post.call_args.kwargs["headers"][ + "Authorization" ] + sent = auth.split(" ", 1)[1] # strip "Bearer " root = channel.encrypted_settings["secret"] assert sent != root, "raw root secret must never travel as the API key" assert sent.startswith("whk_"), ( @@ -1165,15 +1204,17 @@ def test_missing_secret_fails_closed(self, mock_session, mailbox, parsed_email): ) mock_session.return_value.post.assert_not_called() + @patch("core.mda.dispatch_webhooks.logger") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_missing_secret_retries_when_blocking( - self, mock_session, mailbox, parsed_email + def test_missing_secret_continues_when_blocking( + self, mock_session, mock_logger, mailbox, parsed_email ): """A misconfigured (secret-less) ``blocking`` channel can't sign - the POST, so the dispatcher holds the message for RETRY rather - than dropping it — re-minting the secret lets the next sweep - deliver. A config error must never discard the user's mail.""" - models.Channel.objects.create( + the POST. Re-minting the secret is the fix, but a 48h retry can't + do that — so the dispatcher delivers the mail past the broken + webhook (CONTINUE) and logs at ERROR, rather than stalling the + scope's inbound. It still never POSTs an unsigned request.""" + channel = models.Channel.objects.create( name="no-secret-blocking", type=enums.ChannelTypes.WEBHOOK, scope_level=enums.ChannelScopeLevel.MAILBOX, @@ -1193,7 +1234,88 @@ def test_missing_secret_retries_when_blocking( raw_data=b"raw", is_spam=False, ) - assert outcome.decision == Decision.RETRY + assert outcome.decision == Decision.CONTINUE + # Never POSTs an unsigned request. + mock_session.return_value.post.assert_not_called() + assert _logged_config_error(mock_logger, channel.id) + + @pytest.mark.parametrize("config_error", ["secret", "auth_method", "ssrf"]) + @patch("core.mda.dispatch_webhooks.logger") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_blocking_config_errors_continue_and_log_error( + self, mock_session, mock_logger, mailbox, parsed_email, config_error + ): + """Every config-error a 48h retry can never fix — missing secret, + unknown auth_method, or an SSRF-rejected URL — makes a BLOCKING + webhook deliver the mail past it (CONTINUE) and log at ERROR. + (Transient failures still RETRY — covered separately. Missing-url is + filtered before the step is built, so it can't reach the blocking + path — covered via ``_dispatch_webhook`` directly below.)""" + settings = { + "url": "https://hook.example.com", + "trigger": "message.delivering", + "auth_method": "jwt", + } + encrypted = {"secret": "whsec_test"} + if config_error == "secret": + encrypted = {} + elif config_error == "auth_method": + settings["auth_method"] = "bogus" # not a valid WebhookAuthMethod + + channel = models.Channel.objects.create( + name=f"cfg-err-{config_error}", + type=enums.ChannelTypes.WEBHOOK, + scope_level=enums.ChannelScopeLevel.MAILBOX, + mailbox=mailbox, + settings=settings, + encrypted_settings=encrypted, + ) + if config_error == "ssrf": + mock_session.return_value.post.side_effect = SSRFValidationError("blocked") + + outcome = dispatch_webhooks( + phase=PHASE_AFTER_SPAM, + mailbox=mailbox, + recipient_email=str(mailbox), + parsed_email=parsed_email, + raw_data=b"raw", + is_spam=False, + ) + + assert outcome.decision == Decision.CONTINUE + assert _logged_config_error(mock_logger, channel.id) + if config_error != "ssrf": + # The secret/auth_method errors are caught before any POST. + mock_session.return_value.post.assert_not_called() + + @patch("core.mda.dispatch_webhooks.logger") + @patch("core.mda.dispatch_webhooks.SSRFSafeSession") + def test_missing_url_continues_and_logs_error( + self, mock_session, mock_logger, mailbox + ): + """A url-less blocking channel reaches ``_dispatch_webhook`` only via + a direct call (``webhook_steps_for_mailbox`` filters url-less + channels before building a step). Exercise the branch directly: + CONTINUE past it + ERROR, never a POST.""" + channel = models.Channel.objects.create( + name="no-url", + type=enums.ChannelTypes.WEBHOOK, + scope_level=enums.ChannelScopeLevel.MAILBOX, + mailbox=mailbox, + settings={"trigger": "message.delivering", "auth_method": "jwt"}, + encrypted_settings={"secret": "whsec_test"}, + ) + result = _dispatch_webhook( + channel=channel, + mailbox=mailbox, + is_spam=False, + recipient_email=str(mailbox), + content_type="message/rfc822", + body_bytes=b"raw", + blocking=True, + ) + assert result.decision == Decision.CONTINUE + assert _logged_config_error(mock_logger, channel.id) mock_session.return_value.post.assert_not_called() @patch("core.mda.dispatch_webhooks.SSRFSafeSession") @@ -1245,8 +1367,8 @@ def test_jwt_bearer_token_verifies_and_binds_body( @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_rotation_changes_signing_secret(self, mock_session, mailbox, parsed_email): """``rotate_secret`` invalidates the old secret immediately: the - next dispatch signs with the NEW secret, and a signature recomputed - with the OLD secret no longer matches.""" + next dispatch's JWT verifies with the NEW secret and no longer + verifies with the OLD one.""" channel = factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -1286,21 +1408,13 @@ def test_rotation_changes_signing_secret(self, mock_session, mailbox, parsed_ema is_spam=False, ) headers = mock_session.return_value.post.call_args.kwargs["headers"] - ts = headers["X-StMsg-Webhook-Timestamp"] - sig = headers["X-StMsg-Webhook-Signature"].split("=", 1)[1] - - expected_new = hmac.new( - new_secret.encode("utf-8"), - ts.encode("ascii") + b"." + raw, - hashlib.sha256, - ).hexdigest() - expected_old = hmac.new( - old_secret.encode("utf-8"), - ts.encode("ascii") + b"." + raw, - hashlib.sha256, - ).hexdigest() - assert hmac.compare_digest(sig, expected_new) - assert not hmac.compare_digest(sig, expected_old) + token = headers["Authorization"].split(" ", 1)[1] + + # Verifies with the NEW secret... + jwt.decode(token, new_secret, algorithms=["HS256"]) + # ...and no longer with the OLD one. + with pytest.raises(jwt.InvalidSignatureError): + jwt.decode(token, old_secret, algorithms=["HS256"]) # --- integration with process_inbound_message_task --- # @@ -1320,9 +1434,7 @@ def test_before_spam_blocking_retries_message( b"To: " + str(mailbox).encode() + b"\r\n" b"Subject: test\r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -1357,9 +1469,7 @@ def test_after_spam_blocking_retries_message( b"To: " + str(mailbox).encode() + b"\r\n" b"Subject: test\r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -1394,9 +1504,7 @@ def test_after_spam_is_spam_header( b"Subject: test\r\n" b"Message-ID: \r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -1432,11 +1540,9 @@ def test_creation_failure_retries_then_abandons(self, mock_rspamd, mock_create): b"To: " + str(mailbox).encode() + b"\r\n" b"Subject: t\r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) - # Within the quarantine window → held for retry, row kept. + # Within the deferral window → held for retry, row kept. with patch.object(process_inbound_message_task, "update_state", Mock()): result = process_inbound_message_task.run(str(inbound_message.id)) assert result["error"] == "retry" @@ -1446,7 +1552,7 @@ def test_creation_failure_retries_then_abandons(self, mock_rspamd, mock_create): # are the only copy of the mail) but stamped terminally failed so # the sweep stops re-running the pipeline + webhooks on it. models.InboundMessage.objects.filter(id=inbound_message.id).update( - created_at=dj_timezone.now() - QUARANTINE_AFTER - QUARANTINE_AFTER + created_at=dj_timezone.now() - DEFERRAL_MAX_AGE - DEFERRAL_MAX_AGE ) with patch.object(process_inbound_message_task, "update_state", Mock()): result = process_inbound_message_task.run(str(inbound_message.id)) @@ -1556,8 +1662,8 @@ def test_dispatch_webhook_task_posts_signed(self, mock_session, mailbox): assert mock_session.return_value.post.call_args.kwargs["data"] == b"raw mime" headers = mock_session.return_value.post.call_args.kwargs["headers"] assert headers["X-StMsg-Trigger"] == "message.delivered" - # Signed at send time (jwt auth_method). - assert "X-StMsg-Webhook-Signature" in headers + # Signed at send time (jwt auth_method) — a single Bearer JWT. + assert headers["Authorization"].startswith("Bearer ") # Fired post-persist, so the platform's Message/Thread ids ride along # for receiver-side API callbacks. assert headers["X-StMsg-Message-Id"] == str(message.id) @@ -1765,7 +1871,7 @@ def test_recipient_webhook_failure_does_not_affect_sender( # The recipient's queue row is held (not lost, not delivered), # and no message has landed in their mailbox yet. assert models.InboundMessage.objects.filter( - mailbox=recipient_mailbox, is_internal=True + mailbox=recipient_mailbox, envelope__origin="internal" ).exists() assert not models.Message.objects.filter( thread__accesses__mailbox=recipient_mailbox @@ -1872,7 +1978,7 @@ def test_action_accept_is_continue(self): def test_action_unknown_is_continue(self): """An unknown action falls through to CONTINUE — receivers adding new verbs we don't know about shouldn't surprise-drop.""" - outcome = _classify_response_body(b'{"action": "quarantine"}') + outcome = _classify_response_body(b'{"action": "not-a-real-action"}') assert outcome.decision == Decision.CONTINUE def test_action_retry_is_continue(self): @@ -2356,9 +2462,7 @@ def test_5xx_retries_and_keeps_inbound_message( b"To: " + str(mailbox).encode() + b"\r\n" b"Subject: test\r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -2393,9 +2497,7 @@ def test_timeout_retries_and_keeps_inbound_message( b"To: " + str(mailbox).encode() + b"\r\n" b"Subject: test\r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -2418,7 +2520,7 @@ def test_timeout_retries_and_keeps_inbound_message( @patch("core.mda.inbound_tasks._create_message_from_inbound") @patch("core.mda.inbound_pipeline._call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_blocking_webhook_quarantine_delivers_flagged_after_window( + def test_blocking_webhook_deferral_delivers_flagged_after_window( self, mock_session, mock_check_spam, mock_create_message ): """Past the 48h window a still-failing blocking webhook stops @@ -2431,12 +2533,10 @@ def test_blocking_webhook_quarantine_delivers_flagged_after_window( b"To: " + str(mailbox).encode() + b"\r\n" b"Subject: test\r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) - # Backdate past the quarantine window (auto_now_add → update()). + inbound_message = _queue_inbound(mailbox, raw_data) + # Backdate past the deferral window (auto_now_add → update()). models.InboundMessage.objects.filter(id=inbound_message.id).update( - created_at=dj_timezone.now() - QUARANTINE_AFTER - timedelta(minutes=1) + created_at=dj_timezone.now() - DEFERRAL_MAX_AGE - timedelta(minutes=1) ) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, @@ -2454,14 +2554,15 @@ def test_blocking_webhook_quarantine_delivers_flagged_after_window( process_inbound_message_task.run(str(inbound_message.id)) # The pipeline aborts at the failing before_spam webhook (RETRY), - # so quarantine is decided at the task level — generic, not + # so deferral is decided at the task level — generic, not # webhook-specific: rspamd is never reached this run. mock_check_spam.assert_not_called() # Delivered, not dropped. mock_create_message.assert_called_once() kwargs = mock_create_message.call_args.kwargs - # Stamped for the UI banner... - assert b"X-StMsg-Processing-Failed: true" in kwargs["raw_data"] + # Stamped for the UI banner — structurally in postmark, not in the bytes. + assert kwargs["postmark"]["processing"] == "fail" + assert b"X-StMsg-Processing-Failed" not in kwargs["raw_data"] # ...and forced to the inbox (is_spam=False) so the banner is seen. assert kwargs["is_spam"] is False # Queue row consumed — not pinned, not dropped silently. @@ -2470,11 +2571,12 @@ def test_blocking_webhook_quarantine_delivers_flagged_after_window( @patch("core.mda.autoreply.try_send_autoreply") @patch("core.mda.inbound_pipeline._call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_blocking_webhook_quarantine_suppresses_autoreply( + def test_blocking_webhook_deferral_suppresses_autoreply( self, mock_session, _mock_rspamd, mock_autoreply ): - """A quarantine delivery forces ``is_spam=False`` to surface the - warning banner, but must NOT fire an autoreply: the spam verdict is + """Force-delivering past an expired deferral forces ``is_spam=False`` + to surface the warning banner, but must NOT fire an autoreply: the + spam verdict is unverified and the blocking step that might have suppressed the reply never completed.""" mailbox = factories.MailboxFactory() @@ -2482,15 +2584,13 @@ def test_blocking_webhook_quarantine_suppresses_autoreply( b"From: sender@example.com\r\n" b"To: " + str(mailbox).encode() + b"\r\n" b"Subject: test\r\n" - b"Message-ID: \r\n\r\nbody" + b"Message-ID: \r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) - # Backdate past the quarantine window so the failing webhook stops + inbound_message = _queue_inbound(mailbox, raw_data) + # Backdate past the deferral window so the failing webhook stops # holding and the message is force-delivered. models.InboundMessage.objects.filter(id=inbound_message.id).update( - created_at=dj_timezone.now() - QUARANTINE_AFTER - timedelta(minutes=1) + created_at=dj_timezone.now() - DEFERRAL_MAX_AGE - timedelta(minutes=1) ) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, @@ -2501,7 +2601,7 @@ def test_blocking_webhook_quarantine_suppresses_autoreply( "auth_method": "jwt", }, ) - # Webhook keeps failing → RETRY → quarantine-delivered. + # Webhook keeps failing → RETRY → force-delivered once deferral expires. mock_session.return_value.post.return_value = _make_response(503) with patch.object(process_inbound_message_task, "update_state", Mock()): @@ -2528,9 +2628,7 @@ def test_before_spam_is_spam_override_short_circuits_rspamd( b"To: " + str(mailbox).encode() + b"\r\n" b"Subject: test\r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -2567,9 +2665,7 @@ def test_after_spam_is_spam_override_replaces_verdict( b"To: " + str(mailbox).encode() + b"\r\n" b"Subject: test\r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -2613,9 +2709,7 @@ def test_webhook_label_attached_to_thread(self, mock_session, mock_check_spam): b"Subject: hello\r\n" b"Message-ID: \r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -2675,9 +2769,7 @@ def test_assign_to_resolves_email_and_attributes_channel( b"Subject: assign me\r\n" b"Message-ID: \r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) channel = factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -2750,9 +2842,7 @@ def test_assign_to_skips_unknown_ambiguous_and_viewer( b"Subject: assign mixed\r\n" b"Message-ID: \r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -2814,9 +2904,7 @@ def test_two_webhooks_each_produce_own_threadevent( b"Subject: multi\r\n" b"Message-ID: \r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) ch_a = factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -2880,9 +2968,7 @@ def _send(self, mailbox, mime_id, action_body: bytes): b"Subject: action\r\n" b"Message-ID: <" + mime_id.encode() + b">\r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -2941,9 +3027,7 @@ def test_skip_autoreply_suppresses_autoreply_call( b"Subject: noreply\r\n" b"Message-ID: \r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -2981,9 +3065,7 @@ def test_add_event_im_creates_threadevent(self, mock_session, mock_rspamd): b"Subject: comment\r\n" b"Message-ID: \r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) channel = factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -3054,9 +3136,7 @@ def test_reply_draft_creates_draft_with_template_body( b"Subject: I need help\r\n" b"Message-ID: \r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) channel = factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -3115,9 +3195,7 @@ def test_reply_draft_out_of_scope_template_skipped(self, mock_session, mock_rspa b"Subject: nope\r\n" b"Message-ID: \r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -3177,9 +3255,7 @@ def test_assign_failure_does_not_skip_labels( b"Subject: isolation\r\n" b"Message-ID: \r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -3212,8 +3288,8 @@ def test_assign_failure_does_not_skip_labels( @pytest.mark.django_db -class TestQuarantineDelivery: - """When a blocking webhook fails persistently past ``QUARANTINE_AFTER``, +class TestDeferralDelivery: + """When a blocking webhook fails persistently past ``DEFERRAL_MAX_AGE``, the message is delivered anyway — stamped with the ``X-StMsg-Processing- Failed`` marker, forced to the inbox (``is_spam=False``), and the autoreply is suppressed.""" @@ -3221,23 +3297,21 @@ class TestQuarantineDelivery: @patch("core.mda.autoreply.try_send_autoreply") @patch("core.mda.inbound_pipeline._call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_quarantine_delivers_message_past_window( + def test_deferral_delivers_message_past_window( self, mock_session, mock_rspamd, mock_autoreply ): mailbox = factories.MailboxFactory() raw_data = ( b"From: sender@example.com\r\n" b"To: " + str(mailbox).encode() + b"\r\n" - b"Subject: quarantine\r\n" - b"Message-ID: \r\n\r\nbody" + b"Subject: deferral\r\n" + b"Message-ID: \r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) - # Push ``created_at`` past the 48-hour quarantine window so the - # task takes the age > QUARANTINE_AFTER branch on first attempt. + inbound_message = _queue_inbound(mailbox, raw_data) + # Push ``created_at`` past the 48-hour deferral window so the + # task takes the age > DEFERRAL_MAX_AGE branch on first attempt. models.InboundMessage.objects.filter(id=inbound_message.id).update( - created_at=dj_timezone.now() - QUARANTINE_AFTER - timedelta(hours=1) + created_at=dj_timezone.now() - DEFERRAL_MAX_AGE - timedelta(hours=1) ) inbound_message.refresh_from_db() @@ -3260,36 +3334,37 @@ def test_quarantine_delivers_message_past_window( # Message delivered despite the blocking webhook failing. assert result["success"] is True assert result["is_spam"] is False - message = models.Message.objects.get(mime_id="quarantine-test@example.com") + message = models.Message.objects.get(mime_id="deferral-test@example.com") - # X-StMsg-Processing-Failed stamp is embedded in the stored MIME. - blob_content = message.blob.get_content() - assert b"X-StMsg-Processing-Failed: true" in blob_content + # Processing-failed stamp is recorded structurally in postmark (not + # baked into the bytes) and surfaced via get_stmsg_headers for the UI. + assert message.postmark["processing"] == "fail" + assert message.get_stmsg_headers()["processing-failed"] == "true" + assert b"X-StMsg-Processing-Failed" not in message.blob.get_content() - # Autoreply suppressed for quarantined messages. + # Autoreply suppressed when the deferral window expired. mock_autoreply.assert_not_called() @patch("core.mda.autoreply.try_send_autoreply") @patch("core.mda.inbound_pipeline._call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") - def test_quarantine_force_is_spam_false( + def test_deferral_force_is_spam_false( self, mock_session, mock_rspamd, mock_autoreply ): - """Even when the pipeline sets is_spam=True (e.g. rspamd), the - quarantine path forces is_spam=False so the message lands in the - inbox where the recipient sees the warning banner.""" + """Even when the pipeline sets is_spam=True (e.g. rspamd), + force-delivering past an expired deferral forces is_spam=False so the + message lands in the inbox where the recipient sees the warning + banner.""" mailbox = factories.MailboxFactory() raw_data = ( b"From: spammer@example.com\r\n" b"To: " + str(mailbox).encode() + b"\r\n" - b"Subject: quarantine spam\r\n" - b"Message-ID: \r\n\r\nbody" - ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data + b"Subject: deferral spam\r\n" + b"Message-ID: \r\n\r\nbody" ) + inbound_message = _queue_inbound(mailbox, raw_data) models.InboundMessage.objects.filter(id=inbound_message.id).update( - created_at=dj_timezone.now() - QUARANTINE_AFTER - timedelta(hours=1) + created_at=dj_timezone.now() - DEFERRAL_MAX_AGE - timedelta(hours=1) ) inbound_message.refresh_from_db() @@ -3302,8 +3377,8 @@ def test_quarantine_force_is_spam_false( "auth_method": "jwt", }, ) - # Rspamd votes spam, webhook errors → RETRY → quarantine should - # still force is_spam=False. + # Rspamd votes spam, webhook errors → RETRY → force-delivery (once the + # deferral window expires) should still force is_spam=False. mock_rspamd.return_value = (True, None, None) mock_session.return_value.post.return_value = _make_response(503) @@ -3312,15 +3387,16 @@ def test_quarantine_force_is_spam_false( assert result["success"] is True assert result["is_spam"] is False - message = models.Message.objects.get(mime_id="quarantine-spam@example.com") - blob_content = message.blob.get_content() - assert b"X-StMsg-Processing-Failed: true" in blob_content + message = models.Message.objects.get(mime_id="deferral-spam@example.com") + assert message.postmark["processing"] == "fail" + assert message.get_stmsg_headers()["processing-failed"] == "true" + assert b"X-StMsg-Processing-Failed" not in message.blob.get_content() mock_autoreply.assert_not_called() @pytest.mark.django_db class TestAbandonedRowHandling: - """A row that fails to be CREATED past the quarantine window is + """A row that fails to be CREATED past the deferral window is abandoned: kept (its raw bytes are the only copy of the mail) but stamped via ``abandoned_at`` so neither the 5-min sweep nor a direct re-dispatch ever re-runs the pipeline (and re-fires every webhook) @@ -3329,9 +3405,9 @@ class TestAbandonedRowHandling: def test_sweep_skips_abandoned_rows(self): """The retry sweep must not re-queue an abandoned row.""" mailbox = factories.MailboxFactory() - abandoned = models.InboundMessage.objects.create( - mailbox=mailbox, - raw_data=b"From: s@example.com\r\nSubject: t\r\n\r\nbody", + abandoned = _queue_inbound( + mailbox, + b"From: s@example.com\r\nSubject: t\r\n\r\nbody", error_message="persistent failure", abandoned_at=dj_timezone.now(), ) @@ -3356,9 +3432,9 @@ def test_redispatch_of_abandoned_row_early_returns(self, mock_rspamd, mock_creat ``{"error": "abandoned"}`` without ever running the pipeline — rspamd is never consulted and no creation is attempted.""" mailbox = factories.MailboxFactory() - abandoned = models.InboundMessage.objects.create( - mailbox=mailbox, - raw_data=b"From: s@example.com\r\nSubject: t\r\n\r\nbody", + abandoned = _queue_inbound( + mailbox, + b"From: s@example.com\r\nSubject: t\r\n\r\nbody", error_message="persistent failure", abandoned_at=dj_timezone.now(), ) @@ -3375,9 +3451,9 @@ def test_redispatch_of_abandoned_row_early_returns(self, mock_rspamd, mock_creat def test_purge_deletes_old_abandoned_rows(self): """The retention purge reclaims rows abandoned past the window.""" mailbox = factories.MailboxFactory() - old = models.InboundMessage.objects.create( - mailbox=mailbox, - raw_data=b"x", + old = _queue_inbound( + mailbox, + b"x", error_message="persistent failure", abandoned_at=dj_timezone.now() - timedelta(days=8), ) @@ -3390,9 +3466,9 @@ def test_purge_deletes_old_abandoned_rows(self): def test_purge_keeps_recent_abandoned_rows(self): """Rows abandoned within the retention window are kept.""" mailbox = factories.MailboxFactory() - recent = models.InboundMessage.objects.create( - mailbox=mailbox, - raw_data=b"x", + recent = _queue_inbound( + mailbox, + b"x", error_message="persistent failure", abandoned_at=dj_timezone.now() - timedelta(days=1), ) @@ -3405,10 +3481,7 @@ def test_purge_keeps_recent_abandoned_rows(self): def test_purge_ignores_live_rows(self): """A non-abandoned row is never touched by the purge, however old.""" mailbox = factories.MailboxFactory() - live = models.InboundMessage.objects.create( - mailbox=mailbox, - raw_data=b"x", - ) + live = _queue_inbound(mailbox, b"x") models.InboundMessage.objects.filter(id=live.id).update( created_at=dj_timezone.now() - timedelta(days=30) ) @@ -3474,7 +3547,7 @@ def test_dedup_hit_skips_finalize_side_effects( ) # --- First pass: creates the message + all side effects. --- - im1 = models.InboundMessage.objects.create(mailbox=mailbox, raw_data=raw_data) + im1 = _queue_inbound(mailbox, raw_data) with patch.object(process_inbound_message_task, "update_state", Mock()): r1 = process_inbound_message_task.run(str(im1.id)) assert r1["success"] is True @@ -3495,7 +3568,7 @@ def draft_count(): assert mock_autoreply.call_count == 1 # fired once, on create # --- Second pass: SAME Message-ID → dedup hit (_created_now=False). --- - im2 = models.InboundMessage.objects.create(mailbox=mailbox, raw_data=raw_data) + im2 = _queue_inbound(mailbox, raw_data) with patch.object(process_inbound_message_task, "update_state", Mock()): process_inbound_message_task.run(str(im2.id)) @@ -3541,14 +3614,14 @@ def test_dedup_hit_does_not_refire_nonblocking_webhook( b"Message-ID: <" + mime.encode() + b">\r\n\r\nbody" ) - im1 = models.InboundMessage.objects.create(mailbox=mailbox, raw_data=raw_data) + im1 = _queue_inbound(mailbox, raw_data) with patch.object(process_inbound_message_task, "update_state", Mock()): process_inbound_message_task.run(str(im1.id)) # Enqueued exactly once, on the original create. assert mock_delay.call_count == 1 # Duplicate delivery: same Message-ID, separate queue row. - im2 = models.InboundMessage.objects.create(mailbox=mailbox, raw_data=raw_data) + im2 = _queue_inbound(mailbox, raw_data) with patch.object(process_inbound_message_task, "update_state", Mock()): process_inbound_message_task.run(str(im2.id)) @@ -3710,9 +3783,7 @@ def test_blocking_webhook_not_refired_across_retries( b"Subject: storm\r\n" b"Message-ID: \r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, @@ -3759,9 +3830,7 @@ def test_happy_path_does_not_touch_cache( b"Subject: happy\r\n" b"Message-ID: \r\n\r\nbody" ) - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=raw_data - ) + inbound_message = _queue_inbound(mailbox, raw_data) factories.ChannelFactory( type=enums.ChannelTypes.WEBHOOK, mailbox=mailbox, diff --git a/src/backend/core/tests/mda/test_inbound.py b/src/backend/core/tests/mda/test_inbound.py index 9d78e67f1..f05a72354 100644 --- a/src/backend/core/tests/mda/test_inbound.py +++ b/src/backend/core/tests/mda/test_inbound.py @@ -14,10 +14,55 @@ from core.mda.inbound import deliver_inbound_message from core.mda.inbound_create import ( _create_message_from_inbound, + _record_divergent_rcpt, find_thread_for_inbound_message, ) +class TestRecordDivergentRcpt: + """``postmark["rcpt_to"]`` is written only when the envelope RCPT diverges + from the visible MIME To/Cc — the BCC / alias / catch-all signal.""" + + def _parsed(self, to=None, cc=None): + return {"to": to or [], "cc": cc or []} + + def test_rcpt_in_to_not_recorded(self): + """RCPT present in To → not recorded (happy path).""" + postmark = {} + _record_divergent_rcpt( + postmark, + "user@example.com", + self._parsed(to=[{"email": "user@example.com"}]), + ) + assert "rcpt_to" not in postmark + + def test_rcpt_in_cc_not_recorded(self): + """RCPT present in Cc (case-insensitively) → not recorded.""" + postmark = {} + _record_divergent_rcpt( + postmark, + "User@Example.com", + self._parsed(cc=[{"email": "user@example.com"}]), + ) + assert "rcpt_to" not in postmark + + def test_bcc_recipient_recorded(self): + """RCPT absent from To/Cc (BCC / alias) → recorded.""" + postmark = {} + _record_divergent_rcpt( + postmark, + "hidden@example.com", + self._parsed(to=[{"email": "list@example.com"}]), + ) + assert postmark["rcpt_to"] == "hidden@example.com" + + def test_empty_rcpt_is_noop(self): + """An empty RCPT is a no-op (nothing to record).""" + postmark = {} + _record_divergent_rcpt(postmark, "", self._parsed()) + assert not postmark + + @pytest.mark.django_db class TestFindThread: """Unit tests for the find_thread_for_inbound_message helper.""" @@ -919,7 +964,11 @@ def test_autoreply_called_on_internal_delivery( recipient_addr, parse_email(raw), raw, - is_internal=True, + envelope={ + "origin": "internal", + "mail_from": "sender@test.com", + "rcpt_to": recipient_addr, + }, blob=blob, ) diff --git a/src/backend/core/tests/mda/test_inbound_auth.py b/src/backend/core/tests/mda/test_inbound_auth.py index ad4ebb2e2..26ea0b280 100644 --- a/src/backend/core/tests/mda/test_inbound_auth.py +++ b/src/backend/core/tests/mda/test_inbound_auth.py @@ -27,6 +27,14 @@ ) +def _queue_inbound(mailbox, content=RAW_EMAIL): + """Create a queued InboundMessage backed by a blob (blob-at-ingest).""" + blob = models.Blob.objects.create_blob( + content=content, content_type="message/rfc822" + ) + return models.InboundMessage.objects.create(mailbox=mailbox, blob=blob) + + class TestCheckInboundAuthenticationDisabled: """When inbound_auth is absent or empty the check is a no-op.""" @@ -600,19 +608,16 @@ def test_dkim_pass_inside_comment_then_dmarc_fail_real(self): @pytest.mark.django_db class TestProcessInboundMessageAuthIntegration: - """End-to-end: a verdict prepends X-StMsg-Sender-Auth with its value.""" + """End-to-end: a verdict is recorded in ``postmark["auth"]``, off the bytes.""" @override_settings(SPAM_CONFIG={"inbound_auth": "native"}) @patch("core.mda.inbound_pipeline.check_inbound_authentication") @patch("core.mda.inbound_tasks._create_message_from_inbound") - def test_unverified_verdict_injects_none_header( + def test_unverified_verdict_recorded_in_postmark( self, mock_create_message, mock_auth_check ): mailbox = factories.MailboxFactory() - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, - raw_data=RAW_EMAIL, - ) + inbound_message = _queue_inbound(mailbox) mock_auth_check.return_value = VERDICT_UNVERIFIED mock_create_message.return_value = True @@ -621,25 +626,18 @@ def test_unverified_verdict_injects_none_header( assert mock_create_message.called call_kwargs = mock_create_message.call_args[1] - assert call_kwargs["raw_data"].startswith(b"X-StMsg-Sender-Auth: none\r\n") - parsed = call_kwargs["parsed_email"] - assert [ - h["value"] - for h in parsed["headers"] - if h["name"].lower() == "x-stmsg-sender-auth" - ] == ["none"] + # Verdict is structured, not baked into the bytes. + assert call_kwargs["postmark"]["auth"] == "none" + assert not call_kwargs["raw_data"].startswith(b"X-StMsg-Sender-Auth") @override_settings(SPAM_CONFIG={"inbound_auth": "rspamd"}) @patch("core.mda.inbound_pipeline.check_inbound_authentication") @patch("core.mda.inbound_tasks._create_message_from_inbound") - def test_forged_verdict_injects_fail_header( + def test_forged_verdict_recorded_in_postmark( self, mock_create_message, mock_auth_check ): mailbox = factories.MailboxFactory() - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, - raw_data=RAW_EMAIL, - ) + inbound_message = _queue_inbound(mailbox) mock_auth_check.return_value = VERDICT_FORGED mock_create_message.return_value = True @@ -647,25 +645,15 @@ def test_forged_verdict_injects_fail_header( process_inbound_message_task.run(str(inbound_message.id)) call_kwargs = mock_create_message.call_args[1] - assert call_kwargs["raw_data"].startswith(b"X-StMsg-Sender-Auth: fail\r\n") - parsed = call_kwargs["parsed_email"] - assert [ - h["value"] - for h in parsed["headers"] - if h["name"].lower() == "x-stmsg-sender-auth" - ] == ["fail"] + assert call_kwargs["postmark"]["auth"] == "fail" + assert not call_kwargs["raw_data"].startswith(b"X-StMsg-Sender-Auth") @override_settings(SPAM_CONFIG={"inbound_auth": "native"}) @patch("core.mda.inbound_pipeline.check_inbound_authentication") @patch("core.mda.inbound_tasks._create_message_from_inbound") - def test_verified_does_not_inject_header( - self, mock_create_message, mock_auth_check - ): + def test_verified_records_no_auth(self, mock_create_message, mock_auth_check): mailbox = factories.MailboxFactory() - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, - raw_data=RAW_EMAIL, - ) + inbound_message = _queue_inbound(mailbox) mock_auth_check.return_value = None mock_create_message.return_value = True @@ -673,6 +661,7 @@ def test_verified_does_not_inject_header( process_inbound_message_task.run(str(inbound_message.id)) call_kwargs = mock_create_message.call_args[1] + assert "auth" not in (call_kwargs.get("postmark") or {}) assert not call_kwargs["raw_data"].startswith(b"X-StMsg-Sender-Auth") @override_settings( @@ -686,10 +675,7 @@ def test_verified_does_not_inject_header( def test_rspamd_response_reused_by_auth_check(self, mock_create_message, mock_post): """Single rspamd call feeds both spam and auth.""" mailbox = factories.MailboxFactory() - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, - raw_data=RAW_EMAIL, - ) + inbound_message = _queue_inbound(mailbox) mock_response = Mock() mock_response.json.return_value = { "action": "no action", @@ -706,7 +692,7 @@ def test_rspamd_response_reused_by_auth_check(self, mock_create_message, mock_po assert mock_post.call_count == 1 call_kwargs = mock_create_message.call_args[1] - assert call_kwargs["raw_data"].startswith(b"X-StMsg-Sender-Auth: none\r\n") + assert call_kwargs["postmark"]["auth"] == "none" @override_settings( SPAM_CONFIG={ @@ -720,10 +706,7 @@ def test_dmarc_fail_injects_fail_header_end_to_end( self, mock_create_message, mock_post ): mailbox = factories.MailboxFactory() - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, - raw_data=RAW_EMAIL, - ) + inbound_message = _queue_inbound(mailbox) mock_response = Mock() mock_response.json.return_value = { "action": "no action", @@ -742,7 +725,7 @@ def test_dmarc_fail_injects_fail_header_end_to_end( process_inbound_message_task.run(str(inbound_message.id)) call_kwargs = mock_create_message.call_args[1] - assert call_kwargs["raw_data"].startswith(b"X-StMsg-Sender-Auth: fail\r\n") + assert call_kwargs["postmark"]["auth"] == "fail" @override_settings( SPAM_CONFIG={ @@ -759,10 +742,7 @@ def test_rspamd_fetched_on_demand_when_spam_skipped_rspamd( """Hardcoded spam rule short-circuits spam; rspamd still fetched for auth.""" mailbox = factories.MailboxFactory() raw = b"X-Spam: yes\r\n" + RAW_EMAIL - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, - raw_data=raw, - ) + inbound_message = _queue_inbound(mailbox, raw) mock_response = Mock() mock_response.json.return_value = { "action": "no action", @@ -779,7 +759,8 @@ def test_rspamd_fetched_on_demand_when_spam_skipped_rspamd( assert mock_post.call_count == 1 call_kwargs = mock_create_message.call_args[1] - assert not call_kwargs["raw_data"].startswith(b"X-StMsg-Sender-Auth") + # DKIM_ALLOW → verified → no auth verdict recorded. + assert "auth" not in (call_kwargs.get("postmark") or {}) def test_maildomain_override(self): """custom_settings.SPAM_CONFIG overrides the global default.""" @@ -791,8 +772,9 @@ def test_maildomain_override(self): config = maildomain.get_spam_config() assert config.get("inbound_auth") == "rspamd" - def test_header_injection_propagates_to_stmsg(self): - """After prepending, the parser exposes the header via x-stmsg-*.""" + def test_legacy_stmsg_header_still_parsed(self): + """The parser still exposes legacy baked ``X-StMsg-*`` headers, which + ``get_stmsg_headers`` reads for pre-``postmark`` messages.""" tagged = b"X-StMsg-Sender-Auth: fail\r\n" + RAW_EMAIL parsed = parse_email(tagged) values = [ @@ -801,50 +783,3 @@ def test_header_injection_propagates_to_stmsg(self): if h["name"].lower() == "x-stmsg-sender-auth" ] assert values == ["fail"] - - @override_settings(SPAM_CONFIG={"inbound_auth": "native"}) - @patch("core.mda.inbound_pipeline.parse_email") - @patch("core.mda.inbound_tasks.parse_email") - @patch("core.mda.inbound_pipeline.check_inbound_authentication") - @patch("core.mda.inbound_tasks._create_message_from_inbound") - def test_reparse_failure_after_prepend_keeps_views_in_sync( - self, - mock_create_message, - mock_auth_check, - mock_parse_tasks, - mock_parse_pipeline, - ): - """If re-parsing the prepended bytes blows up, the prepend is dropped. - - Otherwise raw_data (with the new header) and parsed_email (without it) - diverge, and `Message.get_parsed_data()` later returns {} for the - whole message because the same bytes fail to parse at display time. - """ - # Initial parse (inbound_tasks) succeeds; re-parse in the - # inbound_auth_step (inbound_pipeline) raises. - original_parsed = { - "headers": [{"name": "from", "value": "a@b"}], - "from": [{"email": "a@b"}], - } - # Initial parse (inbound_tasks) succeeds; the inbound_auth_step - # re-parse (inbound_pipeline) returns None — parse_email signals - # failure with None rather than raising. - mock_parse_tasks.return_value = original_parsed - mock_parse_pipeline.return_value = None - mock_auth_check.return_value = VERDICT_UNVERIFIED - mock_create_message.return_value = True - - mailbox = factories.MailboxFactory() - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, - raw_data=RAW_EMAIL, - ) - - with patch.object(process_inbound_message_task, "update_state", Mock()): - process_inbound_message_task.run(str(inbound_message.id)) - - call_kwargs = mock_create_message.call_args[1] - # Prepend was reverted: raw_data is the original bytes. - assert call_kwargs["raw_data"] == RAW_EMAIL - # parsed_email is the original parse, also without the header. - assert call_kwargs["parsed_email"] is original_parsed diff --git a/src/backend/core/tests/mda/test_inbound_e2e.py b/src/backend/core/tests/mda/test_inbound_e2e.py index 9cad67dc5..e6d12020a 100644 --- a/src/backend/core/tests/mda/test_inbound_e2e.py +++ b/src/backend/core/tests/mda/test_inbound_e2e.py @@ -188,12 +188,16 @@ def test_e2e_inbound_with_attachment( ) message_id = message_data["id"] - # Check message EML + # Check message EML. The stored blob is the ingest blob reused as + # Message.blob: the received bytes with our authoritative Return-Path + # (envelope MAIL FROM; "<>" for the null sender) and Received baked on + # at ingest — and, crucially, NO X-StMsg-* verdict headers (those are + # de-baked into postmark now), proving the single-blob reuse. response = client.get(reverse("messages-eml", kwargs={"id": message_id})) assert response.status_code == status.HTTP_200_OK - assert ( - response.content - == b"Received: from client.helo (client.hostname [127.1.2.3]);\r\n" + assert response.content == ( + b"Return-Path: <>\r\n" + b"Received: from client.helo (client.hostname [127.1.2.3]);\r\n" + multipart_email_with_attachment ) diff --git a/src/backend/core/tests/mda/test_inbound_spoofed_sender.py b/src/backend/core/tests/mda/test_inbound_spoofed_sender.py index e377710f3..96536659f 100644 --- a/src/backend/core/tests/mda/test_inbound_spoofed_sender.py +++ b/src/backend/core/tests/mda/test_inbound_spoofed_sender.py @@ -177,7 +177,16 @@ def test_inbound_spoofed_sender_legit_self_send_dedupes(self, victim_mailbox): } assert ( - deliver_inbound_message(recipient, parsed_email, b"raw", is_internal=True) + deliver_inbound_message( + recipient, + parsed_email, + b"raw", + envelope={ + "origin": "internal", + "mail_from": recipient, + "rcpt_to": recipient, + }, + ) is True ) diff --git a/src/backend/core/tests/mda/test_spam_processing.py b/src/backend/core/tests/mda/test_spam_processing.py index 17869a3ac..13341cef3 100644 --- a/src/backend/core/tests/mda/test_spam_processing.py +++ b/src/backend/core/tests/mda/test_spam_processing.py @@ -1,5 +1,7 @@ """Tests for spam processing with rspamd.""" +# pylint: disable=too-many-lines + from unittest.mock import Mock, patch from django.test import override_settings @@ -24,6 +26,16 @@ ) +def _queue_inbound(mailbox, content=b"raw", envelope=None): + """Create a queued InboundMessage backed by a blob (blob-at-ingest).""" + blob = models.Blob.objects.create_blob( + content=content, content_type="message/rfc822" + ) + return models.InboundMessage.objects.create( + mailbox=mailbox, blob=blob, envelope=envelope or {} + ) + + @pytest.mark.django_db class TestDeliverInboundMessageQueueing: """Test that deliver_inbound_message queues messages instead of creating them directly.""" @@ -50,9 +62,11 @@ def test_deliver_inbound_message_queues_message(self, mock_task_delay): assert result is True - # Check that an InboundMessage was created + # Check that an InboundMessage was created, with the bytes committed + # to a blob at ingest (blob-at-ingest; no inline plaintext). inbound_message = models.InboundMessage.objects.get(mailbox=mailbox) - assert inbound_message.raw_data == raw_data + assert inbound_message.blob is not None + assert inbound_message.get_raw_bytes() == raw_data assert inbound_message.mailbox == mailbox # Check that the task was queued @@ -107,9 +121,11 @@ def test_check_spam_with_rspamd_spam(self, mock_post): mock_post.return_value = mock_response raw_data = b"Spam email content" - is_spam, error, rspamd_result = _call_rspamd(raw_data, spam_config) + action, error, rspamd_result = _call_rspamd(raw_data, spam_config) - assert is_spam is True + # _call_rspamd returns the raw action; the action -> verdict mapping + # lives in the step (see TestRspamdStepFailureHandling). + assert action == "reject" assert error is None assert rspamd_result is not None mock_post.assert_called_once() @@ -132,11 +148,65 @@ def test_check_spam_with_rspamd_not_spam(self, mock_post): mock_post.return_value = mock_response raw_data = b"Legitimate email content" - is_spam, error, _rspamd_result = _call_rspamd(raw_data, spam_config) + action, error, _rspamd_result = _call_rspamd(raw_data, spam_config) - assert is_spam is False + assert action == "no action" assert error is None + @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) + @patch("core.mda.inbound_pipeline.requests.post") + def test_call_rspamd_forwards_smtp_envelope_headers(self, mock_post): + """The SMTP envelope is forwarded via rspamd's scan headers, and + CR/LF in attacker-influenced fields (HELO/hostname) is stripped.""" + spam_config = {"rspamd_url": "http://rspamd:8010/_api"} + mock_response = Mock() + mock_response.json.return_value = {"action": "no action", "score": 0.0} + mock_response.raise_for_status = Mock() + mock_post.return_value = mock_response + + envelope = { + "origin": "mta", + "mail_from": "sender@example.com", + "rcpt_to": "rcpt@example.com", + "ip": "203.0.113.7", + "helo": "evil\r\nX-Injected: 1", + "hostname": "mail.example.com", + } + _call_rspamd(b"content", spam_config, envelope=envelope) + + headers = mock_post.call_args[1]["headers"] + assert headers["From"] == "sender@example.com" + assert headers["Rcpt"] == "rcpt@example.com" + assert headers["IP"] == "203.0.113.7" + assert headers["Hostname"] == "mail.example.com" + # CR/LF stripped: no header injection into the rspamd request. + assert headers["Helo"] == "evilX-Injected: 1" + assert "\r" not in headers["Helo"] and "\n" not in headers["Helo"] + + @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) + @patch("core.mda.inbound_pipeline.requests.post") + def test_call_rspamd_omits_absent_envelope_headers(self, mock_post): + """Fields we don't have (widget/internal mail has no HELO/rDNS) are + omitted rather than sent empty, which would skew scoring.""" + spam_config = {"rspamd_url": "http://rspamd:8010/_api"} + mock_response = Mock() + mock_response.json.return_value = {"action": "no action", "score": 0.0} + mock_response.raise_for_status = Mock() + mock_post.return_value = mock_response + + _call_rspamd( + b"content", + spam_config, + envelope={"origin": "widget", "mail_from": "u@example.com"}, + ) + + headers = mock_post.call_args[1]["headers"] + assert headers["From"] == "u@example.com" + assert "Helo" not in headers + assert "Hostname" not in headers + assert "IP" not in headers + assert "Rcpt" not in headers + @override_settings( SPAM_CONFIG={ "rspamd_url": "http://rspamd:8010/_api", @@ -167,30 +237,29 @@ def test_check_spam_with_rspamd_auth_header(self, mock_post): @override_settings(SPAM_CONFIG={}) def test_check_spam_without_rspamd_config(self): - """When rspamd isn't configured the call returns ``is_spam=None`` - (= "no opinion"), so the pipeline falls through to whatever - verdict is already in the context instead of silently marking - the message as ham.""" + """When rspamd isn't configured the call returns ``action=None`` + (= "no opinion"), so the step falls through without deciding a + verdict instead of silently marking the message as ham.""" spam_config = {} raw_data = b"Email content" - is_spam, error, rspamd_result = _call_rspamd(raw_data, spam_config) + action, error, rspamd_result = _call_rspamd(raw_data, spam_config) - assert is_spam is None + assert action is None assert error is None assert rspamd_result is None @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) @patch("core.mda.inbound_pipeline.requests.post") def test_check_spam_with_rspamd_error(self, mock_post): - """Test that errors in rspamd check are handled gracefully.""" + """On error, ``action`` is None and the error is surfaced separately; + the step turns that into a RETRY (never fails open).""" spam_config = {"rspamd_url": "http://rspamd:8010/_api"} mock_post.side_effect = requests.exceptions.RequestException("Connection error") raw_data = b"Email content" - is_spam, error, rspamd_result = _call_rspamd(raw_data, spam_config) + action, error, rspamd_result = _call_rspamd(raw_data, spam_config) - # On error, treat as not spam to avoid blocking legitimate messages - assert is_spam is False + assert action is None assert error is not None assert rspamd_result is None @@ -749,10 +818,7 @@ def test_process_inbound_message_task_spam( mailbox = factories.MailboxFactory() raw_data = b"Spam content" - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, - raw_data=raw_data, - ) + inbound_message = _queue_inbound(mailbox, raw_data) mock_check_spam.return_value = (True, None, None) # is_spam=True mock_create_message.return_value = True @@ -782,10 +848,7 @@ def test_process_inbound_message_task_not_spam( mailbox = factories.MailboxFactory() raw_data = b"Legitimate content" - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, - raw_data=raw_data, - ) + inbound_message = _queue_inbound(mailbox, raw_data) mock_check_spam.return_value = (False, None, None) # is_spam=False mock_create_message.return_value = True @@ -811,10 +874,7 @@ def test_process_inbound_message_task_failure( mailbox = factories.MailboxFactory() raw_data = b"Test content" - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, - raw_data=raw_data, - ) + inbound_message = _queue_inbound(mailbox, raw_data) mock_check_spam.return_value = (False, None, None) mock_create_message.return_value = False # Creation failed @@ -842,11 +902,9 @@ def test_process_inbound_messages_queue_task(self, mock_task_delay): # Create multiple pending messages older than 5 minutes (for retry processing) old_time = timezone.now() - timezone.timedelta(minutes=6) - for _ in range(3): - inbound_message = models.InboundMessage.objects.create( - mailbox=mailbox, - raw_data=b"Content", - ) + for i in range(3): + # Distinct bytes per row so blob dedup doesn't collapse them. + inbound_message = _queue_inbound(mailbox, f"Content {i}".encode()) # Update created_at to make it old enough for retry models.InboundMessage.objects.filter(id=inbound_message.id).update( created_at=old_time @@ -865,11 +923,11 @@ def test_process_inbound_messages_queue_task(self, mock_task_delay): @pytest.mark.django_db class TestRspamdStepFailureHandling: """rspamd never fails open: an error holds the message for retry - (feeding the quarantine path) rather than delivering it unchecked.""" + (deferring it) rather than delivering it unchecked.""" def _ctx(self, spam_config): mailbox = factories.MailboxFactory() - inbound = models.InboundMessage.objects.create(mailbox=mailbox, raw_data=b"raw") + inbound = _queue_inbound(mailbox, b"raw") return InboundContext( mailbox=mailbox, inbound_message=inbound, @@ -883,8 +941,8 @@ def _ctx(self, spam_config): def test_error_holds_for_retry(self, mock_call): """On rspamd error, never fail open — hold the message for retry.""" spam_config = {"rspamd_url": "http://rspamd:11334"} - # _call_rspamd returns is_spam=False on error. - mock_call.return_value = (False, "connection refused", None) + # On error _call_rspamd returns action=None + an error message. + mock_call.return_value = (None, "connection refused", None) decision = _make_rspamd_step(spam_config)(self._ctx(spam_config)) @@ -894,7 +952,7 @@ def test_error_holds_for_retry(self, mock_call): @patch("core.mda.inbound_pipeline._call_rspamd") def test_not_configured_continues(self, mock_call): """When rspamd isn't configured, continue without a verdict.""" - # rspamd absent is "no opinion", not an error → keep moving. + # rspamd absent is "no opinion" (action=None, no error) → keep moving. mock_call.return_value = (None, None, None) ctx = self._ctx({}) @@ -903,14 +961,28 @@ def test_not_configured_continues(self, mock_call): assert decision == Decision.CONTINUE assert ctx.is_spam is None + # The single source of truth for the rspamd action -> outcome mapping. + @pytest.mark.parametrize( + "action,decision,is_spam,marker", + [ + ("no action", Decision.CONTINUE, False, None), + ("add header", Decision.CONTINUE, False, "possible"), + ("rewrite subject", Decision.CONTINUE, False, "likely"), + ("quarantine", Decision.CONTINUE, True, None), + ("reject", Decision.CONTINUE, True, None), + ("greylist", Decision.RETRY, None, None), + ("soft reject", Decision.RETRY, None, None), + ("discard", Decision.DROP, None, None), + ], + ) @patch("core.mda.inbound_pipeline._call_rspamd") - def test_success_sets_verdict(self, mock_call): - """A successful rspamd call sets the spam verdict and continues.""" + def test_action_mapping(self, mock_call, action, decision, is_spam, marker): + """Every rspamd action maps to (decision, Junk verdict, spam marker): + isolate → Junk, flag → inbox+marker, defer → RETRY, discard → DROP.""" spam_config = {"rspamd_url": "http://rspamd:11334"} - mock_call.return_value = (True, None, {"action": "reject"}) + mock_call.return_value = (action, None, {"action": action}) ctx = self._ctx(spam_config) - decision = _make_rspamd_step(spam_config)(ctx) - - assert decision == Decision.CONTINUE - assert ctx.is_spam is True + assert _make_rspamd_step(spam_config)(ctx) == decision + assert ctx.is_spam is is_spam + assert ctx.postmark.get("spam") == marker diff --git a/src/backend/core/tests/models/test_blob.py b/src/backend/core/tests/models/test_blob.py index cbead3ba2..09043bbec 100644 --- a/src/backend/core/tests/models/test_blob.py +++ b/src/backend/core/tests/models/test_blob.py @@ -94,7 +94,7 @@ def test_inbound_message_counts_as_a_blob_reference(self): content=b"internal mime bytes", content_type="message/rfc822" ) inbound = models.InboundMessage.objects.create( - mailbox=mailbox, blob=blob, is_internal=True + mailbox=mailbox, blob=blob, envelope={"origin": "internal"} ) # Referenced solely by the in-flight queue row → still alive. @@ -107,17 +107,3 @@ def test_inbound_message_counts_as_a_blob_reference(self): assert models.Blob.objects.is_referenced(blob.id) is False gc_orphan_blobs_task(mode="full") assert not models.Blob.objects.filter(id=blob.id).exists() - - def test_exactly_one_source_constraint(self): - """An InboundMessage must carry raw_data XOR a blob, never both.""" - mailbox = factories.MailboxFactory() - blob = models.Blob.objects.create_blob( - content=b"bytes", content_type="message/rfc822" - ) - - # Both set → rejected (full_clean surfaces the check constraint - # as a ValidationError before the INSERT reaches the DB). - with pytest.raises(ValidationError): - models.InboundMessage.objects.create( - mailbox=mailbox, raw_data=b"bytes", blob=blob - ) diff --git a/src/backend/core/tests/models/test_message_get_parsed_data.py b/src/backend/core/tests/models/test_message_get_parsed_data.py index f3de51aba..da00b2d58 100644 --- a/src/backend/core/tests/models/test_message_get_parsed_data.py +++ b/src/backend/core/tests/models/test_message_get_parsed_data.py @@ -57,3 +57,86 @@ def test_no_blob_returns_empty(self): models.Message, "blob", new_callable=PropertyMock, return_value=None ): assert not message.get_parsed_data() + + +class TestGetStmsgHeaders: + """``get_stmsg_headers`` unions legacy baked ``X-StMsg-*`` bytes with the + structured ``postmark``; the structured value wins on overlap.""" + + def _headers(self, parsed_headers): + return patch.object( + models.Message, "get_parsed_data", return_value={"headers": parsed_headers} + ) + + def test_legacy_bytes_only(self): + """Pre-postmark message: verdicts come from the baked X-StMsg headers.""" + message = models.Message() + message.postmark = None + with self._headers([{"name": "X-StMsg-Sender-Auth", "value": "fail"}]): + assert message.get_stmsg_headers() == {"sender-auth": "fail"} + + def test_postmark_only(self): + """New message: bytes carry no verdict header; postmark supplies it.""" + message = models.Message() + message.postmark = {"auth": "none", "processing": "fail"} + with self._headers([]): + assert message.get_stmsg_headers() == { + "sender-auth": "none", + "processing-failed": "true", + } + + def test_suspected_spam_marker_projected(self): + """``postmark["spam"]`` (graded: possible/likely) surfaces verbatim as + a ``spam`` marker for the inbox banner.""" + message = models.Message() + message.postmark = {"spam": "likely"} + with self._headers([]): + assert message.get_stmsg_headers() == {"spam": "likely"} + + def test_widget_referer_stays_a_header(self): + """widget-referer is a permanent header, surfaced from bytes alongside + the structured postmark auth.""" + message = models.Message() + message.postmark = {"auth": "none"} + with self._headers([{"name": "X-StMsg-Widget-Referer", "value": "https://x"}]): + assert message.get_stmsg_headers() == { + "widget-referer": "https://x", + "sender-auth": "none", + } + + def test_postmark_wins_over_legacy(self): + """On overlap the authoritative postmark value overrides a residual + (or forged, though stripped at ingest) legacy header.""" + message = models.Message() + message.postmark = {"auth": "fail"} + with self._headers([{"name": "X-StMsg-Sender-Auth", "value": "none"}]): + assert message.get_stmsg_headers()["sender-auth"] == "fail" + + def test_legacy_and_postmark_projections_agree(self): + """The transition invariant: a legacy-bytes message and a de-baked + message with the same verdicts are INDISTINGUISHABLE through + ``get_stmsg_headers`` — same keys AND same values. This is what pins + ``processing-failed`` to "true" from both sources (the postmark reason + "fail" must not leak into the projected value). + """ + legacy = models.Message() + legacy.postmark = None + legacy_bytes = [ + {"name": "X-StMsg-Sender-Auth", "value": "fail"}, + {"name": "X-StMsg-Processing-Failed", "value": "true"}, + ] + with patch.object( + models.Message, "get_parsed_data", return_value={"headers": legacy_bytes} + ): + legacy_out = legacy.get_stmsg_headers() + + new = models.Message() + new.postmark = {"auth": "fail", "processing": "fail"} + with self._headers([]): + new_out = new.get_stmsg_headers() + + assert ( + legacy_out + == new_out + == {"sender-auth": "fail", "processing-failed": "true"} + ) diff --git a/src/backend/messages/settings.py b/src/backend/messages/settings.py index ebe363dbc..1468bfda4 100644 --- a/src/backend/messages/settings.py +++ b/src/backend/messages/settings.py @@ -552,6 +552,22 @@ class Base(Configuration): environ_prefix=None, ) + # How long an inbound message is deferred (held and retried) when a + # processing step keeps failing, before the pipeline stops holding it. A + # message whose step keeps failing (rspamd unreachable, a blocking + # webhook erroring) is re-attempted every 5 minutes up to this age, then + # delivered anyway — flagged ``X-StMsg-Processing-Failed`` — so mail is + # never lost, only delayed. A parse/create failure that never clears is + # marked abandoned at this age instead. Operators trade delivery latency + # (longer = more time for a flaky dependency to recover before we deliver + # unchecked) against how long unchecked mail can sit held during an + # outage. See ``DEFERRAL_MAX_AGE`` in ``core/mda/inbound_pipeline.py``. + MESSAGES_INBOUND_DEFERRAL_MAX_AGE = values.PositiveIntegerValue( + 48 * 60 * 60, # 48 hours in seconds + environ_name="MESSAGES_INBOUND_DEFERRAL_MAX_AGE", + environ_prefix=None, + ) + # Default compression for new blobs. # Format: "" or ":". Examples: "none", "zstd", "zstd:7". MESSAGES_BLOBS_COMPRESS = values.Value( diff --git a/src/frontend/public/locales/common/en-US.json b/src/frontend/public/locales/common/en-US.json index e05e231ef..0442deb70 100755 --- a/src/frontend/public/locales/common/en-US.json +++ b/src/frontend/public/locales/common/en-US.json @@ -787,6 +787,8 @@ "This message has not been delivered. You cancelled the delivery.": "This message has not been delivered. You cancelled the delivery.", "This message has not yet been delivered to all recipients.": "This message has not yet been delivered to all recipients.", "This message is being delivered.": "This message is being delivered.", + "This message is likely spam. Review it with caution.": "This message is likely spam. Review it with caution.", + "This message may be spam. Review it with caution.": "This message may be spam. Review it with caution.", "This message was delivered without our usual safety checks. Please review it with caution.": "This message was delivered without our usual safety checks. Please review it with caution.", "This name is for internal use only and will not be visible to users.": "This name is for internal use only and will not be visible to users.", "This signature is forced": "This signature is forced", diff --git a/src/frontend/public/locales/common/fr-FR.json b/src/frontend/public/locales/common/fr-FR.json index 601acfe89..1d9b1e198 100755 --- a/src/frontend/public/locales/common/fr-FR.json +++ b/src/frontend/public/locales/common/fr-FR.json @@ -874,6 +874,8 @@ "This message has not been delivered. You cancelled the delivery.": "Ce message n'a pas pu être délivré. Vous avez annulé l'envoi.", "This message has not yet been delivered to all recipients.": "Ce message n'a pas encore été délivré à tous les destinataires.", "This message is being delivered.": "Ce message est en cours d'envoi.", + "This message is likely spam. Review it with caution.": "Ce message est probablement un spam. Examinez-le avec prudence.", + "This message may be spam. Review it with caution.": "Ce message est peut-être un spam. Examinez-le avec prudence.", "This message was delivered without our usual safety checks. Please review it with caution.": "Ce message a été délivré sans nos vérifications de sécurité habituelles. Veuillez l'examiner avec prudence.", "This name is for internal use only and will not be visible to users.": "Ce nom est réservé à un usage interne et ne sera pas visible par les utilisateurs.", "This signature is forced": "Cette signature est forcée", diff --git a/src/frontend/public/locales/common/nl-NL.json b/src/frontend/public/locales/common/nl-NL.json index aa90449b7..cfc1511d1 100644 --- a/src/frontend/public/locales/common/nl-NL.json +++ b/src/frontend/public/locales/common/nl-NL.json @@ -495,6 +495,8 @@ "This message has not been delivered. You cancelled the delivery.": "Dit bericht is niet verzonden. Je hebt de levering geannuleerd.", "This message has not yet been delivered to all recipients.": "Dit bericht is nog niet aan alle ontvangers afgeleverd.", "This message is being delivered.": "Dit bericht wordt verstuurd.", + "This message is likely spam. Review it with caution.": "Dit bericht is waarschijnlijk spam. Wees voorzichtig.", + "This message may be spam. Review it with caution.": "Dit bericht is mogelijk spam. Wees voorzichtig.", "This message was delivered without our usual safety checks. Please review it with caution.": "Dit bericht is bezorgd zonder onze gebruikelijke veiligheidscontroles. Wees voorzichtig.", "This name is for internal use only and will not be visible to users.": "Deze naam is alleen voor intern gebruik en is niet zichtbaar voor gebruikers.", "This signature is forced": "Deze handtekening is geforceerd", diff --git a/src/frontend/public/locales/common/ru-RU.json b/src/frontend/public/locales/common/ru-RU.json index 20c88dc3a..9f2b5bd92 100644 --- a/src/frontend/public/locales/common/ru-RU.json +++ b/src/frontend/public/locales/common/ru-RU.json @@ -736,6 +736,8 @@ "This message has not been delivered. You cancelled the delivery.": "Это сообщение не было доставлено. Вы отменили доставку.", "This message has not yet been delivered to all recipients.": "Это сообщение ещё не доставлено всем получателям.", "This message is being delivered.": "Это сообщение доставлено.", + "This message is likely spam. Review it with caution.": "Это письмо, вероятно, является спамом. Будьте осторожны.", + "This message may be spam. Review it with caution.": "Это письмо может быть спамом. Будьте осторожны.", "This message was delivered without our usual safety checks. Please review it with caution.": "Это сообщение было доставлено без наших обычных проверок безопасности. Будьте с ним осторожны.", "This name is for internal use only and will not be visible to users.": "Это имя только для внутреннего использования и не будет видно пользователям.", "This signature is forced": "Эта подпись обязательна", diff --git a/src/frontend/public/locales/common/uk-UA.json b/src/frontend/public/locales/common/uk-UA.json index c370cd33f..16c7c40fe 100644 --- a/src/frontend/public/locales/common/uk-UA.json +++ b/src/frontend/public/locales/common/uk-UA.json @@ -736,6 +736,8 @@ "This message has not been delivered. You cancelled the delivery.": "Це повідомлення не було доставлено. Ви скасували доставлення.", "This message has not yet been delivered to all recipients.": "Це повідомлення ще не було доставлено усім отримувачам.", "This message is being delivered.": "Це повідомлення доставлено.", + "This message is likely spam. Review it with caution.": "Цей лист, імовірно, є спамом. Будьте обережні.", + "This message may be spam. Review it with caution.": "Цей лист може бути спамом. Будьте обережні.", "This message was delivered without our usual safety checks. Please review it with caution.": "Це повідомлення було доставлено без наших звичайних перевірок безпеки. Будьте обачні.", "This name is for internal use only and will not be visible to users.": "Це ім'я тільки для внутрішнього використання та не буде видимим користувачам.", "This signature is forced": "Цей підпис є обов'язковим", diff --git a/src/frontend/src/features/api/gen/models/channel_create_response.ts b/src/frontend/src/features/api/gen/models/channel_create_response.ts index c39e3bb16..32bd86d90 100644 --- a/src/frontend/src/features/api/gen/models/channel_create_response.ts +++ b/src/frontend/src/features/api/gen/models/channel_create_response.ts @@ -56,8 +56,6 @@ export interface ChannelCreateResponse { readonly updated_at: string; /** Plaintext API key — api_key channels and webhook channels with auth_method=api_key. */ api_key?: string; - /** Plaintext password, when the channel type mints one. */ - password?: string; /** webhook channels with auth_method=jwt — the HMAC/JWT signing secret. */ secret?: string; } diff --git a/src/frontend/src/features/api/gen/models/regenerated_secret_response.ts b/src/frontend/src/features/api/gen/models/regenerated_secret_response.ts index ad427e93f..68b4d0cba 100644 --- a/src/frontend/src/features/api/gen/models/regenerated_secret_response.ts +++ b/src/frontend/src/features/api/gen/models/regenerated_secret_response.ts @@ -9,7 +9,7 @@ export interface RegeneratedSecretResponse { /** Channel id. */ id: string; - /** Present for ``api_key`` channels and webhook channels with ``auth_method='api_key'`` — the plaintext API key presented in a request header (X-API-Key / X-StMsg-Api-Key). Returned ONCE; for api_key webhooks it changes whenever the root rotates. */ + /** Present for ``api_key`` channels and webhook channels with ``auth_method='api_key'`` — the plaintext API key. api_key channels send it as ``X-API-Key`` on inbound API calls; api_key webhooks present it as ``Authorization: Bearer``. Returned ONCE; for api_key webhooks it changes whenever the root rotates. */ api_key?: string; /** Present for webhook channels with ``auth_method='jwt'`` — the freshly minted root receivers use to verify the HMAC sig and JWT. */ secret?: string; diff --git a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx index e26c888d1..1ca0e41b5 100644 --- a/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx +++ b/src/frontend/src/features/layouts/components/mailbox-settings/modal-compose-integration/webhook-integration-form.tsx @@ -30,10 +30,9 @@ import { handle } from "@/features/utils/errors"; // URLs; everywhere else https is required, so mirror that client-side. const DEV_ENVIRONMENTS = ["development", "developmentminimal", "e2e"]; -// A single flat trigger lifecycle event replaces the old (events, phase, -// blocking) triple — the event name itself says when it fires and whether -// it blocks, so only valid combinations are representable. Mirrors -// enums.WebhookTrigger. +// A webhook fires on a single lifecycle trigger. The trigger name itself +// says when it fires and whether it blocks delivery, so only valid +// combinations are representable. Mirrors enums.WebhookTrigger. type WebhookTrigger = | "message.inbound" | "message.delivering" diff --git a/src/frontend/src/features/layouts/components/thread-view/components/thread-message/thread-message-header.tsx b/src/frontend/src/features/layouts/components/thread-view/components/thread-message/thread-message-header.tsx index 5c86bb848..12df252ba 100644 --- a/src/frontend/src/features/layouts/components/thread-view/components/thread-message/thread-message-header.tsx +++ b/src/frontend/src/features/layouts/components/thread-view/components/thread-message/thread-message-header.tsx @@ -42,10 +42,17 @@ const ThreadMessageHeader = ({ const isForgedSender = senderAuth === 'fail'; // A processing step (a blocking integration/webhook, spam check, …) - // failed repeatedly, so the message was delivered without it (see the - // quarantine path in the inbound pipeline). Warn prominently. + // failed repeatedly, so the message was force-delivered without it once + // the inbound deferral window expired. Warn prominently. const processingFailed = Boolean(message.stmsg_headers?.['processing-failed']); + // Spam scanning flagged the message as probable spam but below the Junk + // threshold, so it was delivered to the inbox with a graded marker + // ('possible' < 'likely'). Show an inline caution banner. + const suspectedSpam = message.stmsg_headers?.['spam']; + const isPossibleSpam = suspectedSpam === 'possible'; + const isLikelySpam = suspectedSpam === 'likely'; + const isUserSender = useMemo(() => { if (!message.is_sender) return false; if (!message.sender_user) return true; @@ -178,6 +185,15 @@ const ThreadMessageHeader = ({
    )} + {(isPossibleSpam || isLikelySpam) && ( + +
    +

    {isLikelySpam + ? t("This message is likely spam. Review it with caution.") + : t("This message may be spam. Review it with caution.")}

    +
    +     + )}
    From d2b20163df0c250853fef38ef39b0bab96eabbbc Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Wed, 1 Jul 2026 13:08:09 +0200 Subject: [PATCH 19/21] refactor spam processing file --- src/backend/core/mda/inbound_pipeline.py | 161 +----------------- src/backend/core/mda/spam.py | 159 +++++++++++++++++ .../0032_inboundmessage_blob_envelope.py | 7 +- .../core/tests/mda/test_dispatch_webhooks.py | 100 +++++------ .../core/tests/mda/test_inbound_auth.py | 6 +- .../tests/mda/test_inbound_spoofed_sender.py | 8 +- .../core/tests/mda/test_spam_processing.py | 97 ++++++----- 7 files changed, 276 insertions(+), 262 deletions(-) create mode 100644 src/backend/core/mda/spam.py diff --git a/src/backend/core/mda/inbound_pipeline.py b/src/backend/core/mda/inbound_pipeline.py index 4b6f5e23f..d1193263a 100644 --- a/src/backend/core/mda/inbound_pipeline.py +++ b/src/backend/core/mda/inbound_pipeline.py @@ -23,7 +23,6 @@ from __future__ import annotations import logging -import re from dataclasses import dataclass, field from datetime import timedelta from enum import IntEnum @@ -33,15 +32,14 @@ from django.db.models import Q from django.utils import timezone -import requests -from jmap_email import JmapEmail, has_header +from jmap_email import JmapEmail from core import enums, models +from core.mda import spam from core.mda.inbound_auth import ( check_inbound_authentication, get_inbound_auth_mode, ) -from core.mda.utils import headers_blocks from core.services.thread_events import assign_users logger = logging.getLogger(__name__) @@ -161,157 +159,14 @@ class InboundContext: # pylint: disable=too-many-instance-attributes # A processing step that keeps failing (a blocking webhook or rspamd today, # any future RETRY-returning step tomorrow) must not hold a message # forever *or* silently lose it. After this window the task stops holding -# and delivers the message anyway, stamped with ``X-StMsg-Processing- -# Failed`` so the UI warns the recipient it bypassed a processing step. +# and delivers the message anyway, recording ``postmark["processing"]`` so +# the UI warns the recipient it bypassed a processing step. # Generic on purpose — see the RETRY branch in # ``process_inbound_message_task``. Operator-tunable via # ``MESSAGES_INBOUND_DEFERRAL_MAX_AGE`` (seconds; default 48h). DEFERRAL_MAX_AGE = timedelta(seconds=settings.MESSAGES_INBOUND_DEFERRAL_MAX_AGE) -# --------------------------------------------------------------------------- -# Spam-check helpers shared by the steps below. -# --------------------------------------------------------------------------- - - -def _check_hardcoded_rules( - parsed_email: JmapEmail, spam_config: Dict[str, Any] -) -> Optional[bool]: - """Apply the per-domain hardcoded ``rules`` list, header-matched - only against headers from trusted relay blocks. Returns ``True`` / - ``False`` on first matching rule, ``None`` if no rule matched.""" - rules = spam_config.get("rules", []) - for idx, rule in enumerate(rules): - header_match = rule.get("header_match") or rule.get("header_match_regex") - if not header_match: - continue - if ":" not in header_match: - # Log the rule position, not its raw value: ``spam_config`` - # also carries spam-service credentials, so we never echo - # values read from it into logs. - logger.warning( - "Invalid header_match format (missing colon) in spam rule #%d", idx - ) - continue - key, value = header_match.split(":", 1) - key = key.lower().strip() - value = value.lower().strip() - - # Existence check first; the trusted value is read from the - # Received-bounded blocks below. - if not has_header(parsed_email, key): - continue - - # Trust window is "block 0 (our MTA's Received + headers above it) - # + N upstream relay blocks". Default 0: a sender can prepend - # their own Received lines (landing in block 1+), so trusting - # those by default would let them forge an allowlist match. - # Slicing beyond list length is fine — yields all blocks. - trusted_relays = spam_config.get("trusted_relays", 0) - blocks_to_check = trusted_relays + 1 - found_value = None - for block in headers_blocks(parsed_email)[:blocks_to_check]: - if key in block and block[key]: - # Blocks are ordered most-recent → oldest; first match wins. - found_value = block[key][0] - break - if found_value is None: - continue - - header_value = ( - found_value.lower().strip() - if isinstance(found_value, str) - else str(found_value).lower().strip() - ) - if rule.get("header_match"): - is_match = header_value == value - else: # header_match_regex - is_match = re.fullmatch(value, header_value) is not None - if is_match: - action = rule.get("action") or "spam" - if action in ("spam", "reject"): - return True - if action in ("ham", "no action"): - return False - return None - - -def _call_rspamd( - raw_data: bytes, - spam_config: Dict[str, Any], - envelope: Optional[Dict[str, Any]] = None, -) -> Tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]: - """POST raw RFC-822 bytes to rspamd's ``/checkv2``. - - The SMTP ``envelope`` (MAIL FROM, RCPT TO, connecting IP / HELO / rDNS) - is forwarded via rspamd's documented scan headers - (``From``/``Rcpt``/``IP``/``Helo``/``Hostname``) so envelope-based checks - — SPF, DNS RBLs, the Return-Path-vs-From mismatch symbols — score against - the real peer. Without them rspamd sees an empty envelope and SPF/RBL - degrade to no-ops. - - Returns ``(action, error_message, result_dict)``. ``action`` is the raw - rspamd action string (e.g. "no action", "add header", "reject", …) — this - function does NOT interpret it; the whole action → verdict/decision mapping - lives in a single place, ``_make_rspamd_step``. ``action`` is ``None`` when - rspamd is not configured or on error (the ``error_message`` channel - distinguishes the two); errors are swallowed so a flaky rspamd never blocks - delivery. - """ - url = spam_config.get("rspamd_url") - if not url: - logger.debug("SPAM_CONFIG.rspamd_url not configured, skipping rspamd") - return None, None, None - - headers = {"Content-Type": "message/rfc822"} - auth = spam_config.get("rspamd_auth") - if auth: - headers["Authorization"] = auth - - # Forward the SMTP envelope as rspamd scan headers. Only send the fields - # we actually have (widget/internal mail carry no HELO/rDNS); an empty - # value would read as a genuinely empty envelope and skew scoring. HELO - # and hostname are attacker-influenced, so strip CR/LF to prevent header - # injection into the rspamd request. - for header_name, envelope_key in ( - ("From", "mail_from"), - ("Rcpt", "rcpt_to"), - ("IP", "ip"), - ("Helo", "helo"), - ("Hostname", "hostname"), - ): - value = (envelope or {}).get(envelope_key) - if value: - headers[header_name] = str(value).replace("\r", "").replace("\n", "") - - try: - response = requests.post( - f"{url}/checkv2", data=raw_data, headers=headers, timeout=10 - ) - response.raise_for_status() - result = response.json() - except (requests.exceptions.RequestException, ValueError) as exc: - # Network failures, non-2xx (raise_for_status), and a non-JSON - # body (ValueError covers JSONDecodeError) all funnel here. We - # don't let a flaky rspamd block delivery — fall through with - # is_spam=False so the pipeline keeps moving. The - # error_message channel lets the caller log loudly. - logger.exception("Error calling rspamd: %s", exc) - return None, str(exc), None - except Exception as exc: - logger.exception("Unexpected error calling rspamd: %s", exc) - return None, str(exc), None - - if not isinstance(result, dict): - logger.warning("rspamd returned non-object body: %r", result) - return None, "rspamd returned non-object body", None - - action = result.get("action", "") - score = result.get("score", 0.0) - logger.info("Rspamd: action=%s score=%.2f", action, score) - return action, None, result - - # --------------------------------------------------------------------------- # Steps. Each is callable as ``step(ctx) -> Decision`` and carries a # ``.name`` for log/return-value reporting. @@ -322,7 +177,7 @@ def _make_hardcoded_rules_step(spam_config: Dict[str, Any]) -> Step: def hardcoded_rules(ctx: InboundContext) -> Decision: if ctx.is_spam is not None: return Decision.CONTINUE - verdict = _check_hardcoded_rules(ctx.parsed_email, spam_config) + verdict = spam.check_hardcoded_rules(ctx.parsed_email, spam_config) if verdict is not None: ctx.is_spam = verdict return Decision.CONTINUE @@ -354,7 +209,7 @@ def _make_rspamd_step(spam_config: Dict[str, Any]) -> Step: spam-checked. The step RETRYs, so the message is held; if the outage lasts past ``DEFERRAL_MAX_AGE`` it is force-delivered flagged rather than silently unchecked. (rspamd not being configured is not an error — - ``_call_rspamd`` returns no opinion and the pipeline moves on.) + ``spam.call_rspamd`` returns no opinion and the pipeline moves on.) """ def rspamd(ctx: InboundContext) -> Decision: @@ -363,7 +218,7 @@ def rspamd(ctx: InboundContext) -> Decision: # rspamd's symbols for inbound_auth. The auth step has its # own fallback so we can cheaply skip rspamd entirely here. return Decision.CONTINUE - action, err, result = _call_rspamd( + action, err, result = spam.call_rspamd( ctx.raw_data, spam_config, envelope=ctx.inbound_message.envelope ) if err: @@ -428,7 +283,7 @@ def _make_inbound_auth_step(spam_config: Dict[str, Any]) -> Step: def inbound_auth(ctx: InboundContext) -> Decision: if ctx.rspamd_result is None and get_inbound_auth_mode(spam_config) == "rspamd": - _, _, ctx.rspamd_result = _call_rspamd( + _, _, ctx.rspamd_result = spam.call_rspamd( ctx.raw_data, spam_config, envelope=ctx.inbound_message.envelope ) verdict = check_inbound_authentication( diff --git a/src/backend/core/mda/spam.py b/src/backend/core/mda/spam.py new file mode 100644 index 000000000..c984d304b --- /dev/null +++ b/src/backend/core/mda/spam.py @@ -0,0 +1,159 @@ +"""Direction-agnostic spam-scanning primitives. + +The rspamd ``/checkv2`` call and the hardcoded header-match rules live here, +independent of any pipeline, so both the inbound pipeline and (in the future) +outbound submission can scan a message. This module deliberately does NOT +interpret the rspamd action into a verdict/decision — that mapping is +direction-specific (inbound routes to Junk/defer/drop; outbound will block the +send), so it stays with each caller (see ``inbound_pipeline._make_rspamd_step``). +""" + +# pylint: disable=broad-exception-caught + +import logging +import re +from typing import Any, Dict, Optional, Tuple + +import requests +from jmap_email import JmapEmail, has_header + +from core.mda.utils import headers_blocks + +logger = logging.getLogger(__name__) + + +def check_hardcoded_rules( + parsed_email: JmapEmail, spam_config: Dict[str, Any] +) -> Optional[bool]: + """Apply the per-domain hardcoded ``rules`` list, header-matched + only against headers from trusted relay blocks. Returns ``True`` / + ``False`` on first matching rule, ``None`` if no rule matched.""" + rules = spam_config.get("rules", []) + for idx, rule in enumerate(rules): + header_match = rule.get("header_match") or rule.get("header_match_regex") + if not header_match: + continue + if ":" not in header_match: + # Log the rule position, not its raw value: ``spam_config`` + # also carries spam-service credentials, so we never echo + # values read from it into logs. + logger.warning( + "Invalid header_match format (missing colon) in spam rule #%d", idx + ) + continue + key, value = header_match.split(":", 1) + key = key.lower().strip() + value = value.lower().strip() + + # Existence check first; the trusted value is read from the + # Received-bounded blocks below. + if not has_header(parsed_email, key): + continue + + # Trust window is "block 0 (our MTA's Received + headers above it) + # + N upstream relay blocks". Default 0: a sender can prepend + # their own Received lines (landing in block 1+), so trusting + # those by default would let them forge an allowlist match. + # Slicing beyond list length is fine — yields all blocks. + trusted_relays = spam_config.get("trusted_relays", 0) + blocks_to_check = trusted_relays + 1 + found_value = None + for block in headers_blocks(parsed_email)[:blocks_to_check]: + if key in block and block[key]: + # Blocks are ordered most-recent → oldest; first match wins. + found_value = block[key][0] + break + if found_value is None: + continue + + header_value = ( + found_value.lower().strip() + if isinstance(found_value, str) + else str(found_value).lower().strip() + ) + if rule.get("header_match"): + is_match = header_value == value + else: # header_match_regex + is_match = re.fullmatch(value, header_value) is not None + if is_match: + action = rule.get("action") or "spam" + if action in ("spam", "reject"): + return True + if action in ("ham", "no action"): + return False + return None + + +def call_rspamd( + raw_data: bytes, + spam_config: Dict[str, Any], + envelope: Optional[Dict[str, Any]] = None, +) -> Tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]: + """POST raw RFC-822 bytes to rspamd's ``/checkv2``. + + The SMTP ``envelope`` (MAIL FROM, RCPT TO, connecting IP / HELO / rDNS) + is forwarded via rspamd's documented scan headers + (``From``/``Rcpt``/``IP``/``Helo``/``Hostname``) so envelope-based checks + — SPF, DNS RBLs, the Return-Path-vs-From mismatch symbols — score against + the real peer. Without them rspamd sees an empty envelope and SPF/RBL + degrade to no-ops. + + Returns ``(action, error_message, result_dict)``. ``action`` is the raw + rspamd action string (e.g. "no action", "add header", "reject", …) — this + function does NOT interpret it; the whole action → verdict/decision mapping + lives with each caller. ``action`` is ``None`` when rspamd is not + configured or on error (the ``error_message`` channel distinguishes the + two); errors are swallowed so a flaky rspamd never blocks delivery. + """ + url = spam_config.get("rspamd_url") + if not url: + logger.debug("SPAM_CONFIG.rspamd_url not configured, skipping rspamd") + return None, None, None + + headers = {"Content-Type": "message/rfc822"} + auth = spam_config.get("rspamd_auth") + if auth: + headers["Authorization"] = auth + + # Forward the SMTP envelope as rspamd scan headers. Only send the fields + # we actually have (widget/internal mail carry no HELO/rDNS); an empty + # value would read as a genuinely empty envelope and skew scoring. HELO + # and hostname are attacker-influenced, so strip CR/LF to prevent header + # injection into the rspamd request. + for header_name, envelope_key in ( + ("From", "mail_from"), + ("Rcpt", "rcpt_to"), + ("IP", "ip"), + ("Helo", "helo"), + ("Hostname", "hostname"), + ): + value = (envelope or {}).get(envelope_key) + if value: + headers[header_name] = str(value).replace("\r", "").replace("\n", "") + + try: + response = requests.post( + f"{url}/checkv2", data=raw_data, headers=headers, timeout=10 + ) + response.raise_for_status() + result = response.json() + except (requests.exceptions.RequestException, ValueError) as exc: + # Network failures, non-2xx (raise_for_status), and a non-JSON + # body (ValueError covers JSONDecodeError) all funnel here. We + # don't let a flaky rspamd block delivery — surface the error via the + # error_message channel and let the caller decide (the inbound step + # RETRYs rather than failing open). + logger.exception("Error calling rspamd: %s", exc) + return None, str(exc), None + except Exception as exc: + logger.exception("Unexpected error calling rspamd: %s", exc) + return None, str(exc), None + + if not isinstance(result, dict): + logger.warning("rspamd returned non-object body: %r", result) + return None, "rspamd returned non-object body", None + + action = result.get("action", "") + score = result.get("score", 0.0) + logger.info("Rspamd: action=%s score=%.2f", action, score) + return action, None, result diff --git a/src/backend/core/migrations/0032_inboundmessage_blob_envelope.py b/src/backend/core/migrations/0032_inboundmessage_blob_envelope.py index 3a2447c3f..09792cee2 100644 --- a/src/backend/core/migrations/0032_inboundmessage_blob_envelope.py +++ b/src/backend/core/migrations/0032_inboundmessage_blob_envelope.py @@ -23,9 +23,10 @@ def _raw_data_to_blob(apps, schema_editor): raw = bytes(inbound.raw_data) if inbound.raw_data else b"" if not raw: continue - inbound.blob = Blob.objects.create_blob( - content=raw, content_type="message/rfc822" - ) + blob = Blob.objects.create_blob(content=raw, content_type="message/rfc822") + # Assign by id: the real Blob instance isn't an instance of the + # historical Blob model ``apps.get_model`` gave InboundMessage.blob. + inbound.blob_id = blob.id inbound.save(update_fields=["blob"]) diff --git a/src/backend/core/tests/mda/test_dispatch_webhooks.py b/src/backend/core/tests/mda/test_dispatch_webhooks.py index 80f34bac4..343ece0a3 100644 --- a/src/backend/core/tests/mda/test_dispatch_webhooks.py +++ b/src/backend/core/tests/mda/test_dispatch_webhooks.py @@ -1423,7 +1423,7 @@ def test_rotation_changes_signing_secret(self, mock_session, mailbox, parsed_ema @pytest.mark.django_db class TestPipelineIntegration: @patch("core.mda.inbound_tasks._create_message_from_inbound") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_before_spam_blocking_retries_message( self, mock_session, mock_check_spam, mock_create_message @@ -1458,7 +1458,7 @@ def test_before_spam_blocking_retries_message( assert models.InboundMessage.objects.filter(id=inbound_message.id).exists() @patch("core.mda.inbound_tasks._create_message_from_inbound") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_after_spam_blocking_retries_message( self, mock_session, mock_check_spam, mock_create_message @@ -1479,7 +1479,7 @@ def test_after_spam_blocking_retries_message( "auth_method": "jwt", }, ) - mock_check_spam.return_value = (False, None, None) + mock_check_spam.return_value = ("no action", None, None) # 4xx is a webhook error, not an explicit drop → hold for RETRY. mock_session.return_value.post.return_value = _make_response(403) @@ -1492,7 +1492,7 @@ def test_after_spam_blocking_retries_message( mock_create_message.assert_not_called() @patch("core.mda.inbound_tasks._create_message_from_inbound") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_after_spam_is_spam_header( self, mock_session, mock_check_spam, mock_create_message @@ -1514,7 +1514,7 @@ def test_after_spam_is_spam_header( "auth_method": "jwt", }, ) - mock_check_spam.return_value = (True, None, None) + mock_check_spam.return_value = ("reject", None, None) mock_session.return_value.post.return_value = _make_response(200) mock_create_message.return_value = True @@ -1527,12 +1527,12 @@ def test_after_spam_is_spam_header( assert headers["X-StMsg-Trigger"] == "message.delivering" @patch("core.mda.inbound_tasks._create_message_from_inbound") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") def test_creation_failure_retries_then_abandons(self, mock_rspamd, mock_create): """A message that parses but can never be created is held for a bounded retry, then abandoned — it must not loop (re-firing the pipeline + webhooks) forever.""" - mock_rspamd.return_value = (False, None, None) + mock_rspamd.return_value = ("no action", None, None) mock_create.return_value = None # creation always fails mailbox = factories.MailboxFactory() raw_data = ( @@ -1791,7 +1791,7 @@ def _build_internal_message(self, sender_mailbox, recipient_email): ) return message - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_internal_delivery_fires_recipient_webhook( self, mock_session, _mock_rspamd @@ -1833,7 +1833,7 @@ def test_internal_delivery_fires_recipient_webhook( == enums.MessageDeliveryStatusChoices.SENT_INTERNAL ) - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_recipient_webhook_failure_does_not_affect_sender( self, mock_session, _mock_rspamd @@ -1877,7 +1877,7 @@ def test_recipient_webhook_failure_does_not_affect_sender( thread__accesses__mailbox=recipient_mailbox ).exists() - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_internal_delivery_skips_spam_scan(self, mock_session, mock_rspamd): """Internal mail is trusted: the spam steps are skipped (rspamd is @@ -2449,7 +2449,7 @@ def test_post_uses_bounded_timeout_and_streaming( @pytest.mark.django_db class TestPipelineRetry: @patch("core.mda.inbound_tasks._create_message_from_inbound") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_5xx_retries_and_keeps_inbound_message( self, mock_session, mock_check_spam, mock_create_message @@ -2485,7 +2485,7 @@ def test_5xx_retries_and_keeps_inbound_message( mock_create_message.assert_not_called() @patch("core.mda.inbound_tasks._create_message_from_inbound") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_timeout_retries_and_keeps_inbound_message( self, mock_session, mock_check_spam, mock_create_message @@ -2518,7 +2518,7 @@ def test_timeout_retries_and_keeps_inbound_message( mock_create_message.assert_not_called() @patch("core.mda.inbound_tasks._create_message_from_inbound") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_blocking_webhook_deferral_delivers_flagged_after_window( self, mock_session, mock_check_spam, mock_create_message @@ -2569,7 +2569,7 @@ def test_blocking_webhook_deferral_delivers_flagged_after_window( assert not models.InboundMessage.objects.filter(id=inbound_message.id).exists() @patch("core.mda.autoreply.try_send_autoreply") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_blocking_webhook_deferral_suppresses_autoreply( self, mock_session, _mock_rspamd, mock_autoreply @@ -2615,7 +2615,7 @@ def test_blocking_webhook_deferral_suppresses_autoreply( @pytest.mark.django_db class TestPipelineWebhookAntispam: @patch("core.mda.inbound_tasks._create_message_from_inbound") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_before_spam_is_spam_override_short_circuits_rspamd( self, mock_session, mock_check_spam, mock_create_message @@ -2652,7 +2652,7 @@ def test_before_spam_is_spam_override_short_circuits_rspamd( assert mock_create_message.call_args.kwargs["is_spam"] is True @patch("core.mda.inbound_tasks._create_message_from_inbound") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_after_spam_is_spam_override_replaces_verdict( self, mock_session, mock_check_spam, mock_create_message @@ -2676,7 +2676,7 @@ def test_after_spam_is_spam_override_replaces_verdict( }, ) # rspamd says ham; webhook says spam. - mock_check_spam.return_value = (False, None, None) + mock_check_spam.return_value = ("no action", None, None) mock_session.return_value.post.return_value = _make_response( 200, body=b'{"is_spam": true}' ) @@ -2691,7 +2691,7 @@ def test_after_spam_is_spam_override_replaces_verdict( @pytest.mark.django_db class TestPipelineWebhookLabels: - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_webhook_label_attached_to_thread(self, mock_session, mock_check_spam): """Labels from a blocking webhook are attached to the new thread, @@ -2719,7 +2719,7 @@ def test_webhook_label_attached_to_thread(self, mock_session, mock_check_spam): "auth_method": "jwt", }, ) - mock_check_spam.return_value = (False, None, None) + mock_check_spam.return_value = ("no action", None, None) mock_session.return_value.post.return_value = _make_response( 200, body=json.dumps( @@ -2750,7 +2750,7 @@ class TestPipelineWebhookAssign: ambiguous, and non-assignable users are silently skipped — delivery is never blocked because of an assign hiccup.""" - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_assign_to_resolves_email_and_attributes_channel( self, mock_session, mock_check_spam @@ -2779,7 +2779,7 @@ def test_assign_to_resolves_email_and_attributes_channel( "auth_method": "jwt", }, ) - mock_check_spam.return_value = (False, None, None) + mock_check_spam.return_value = ("no action", None, None) # Email case differs from User.email to exercise iexact. mock_session.return_value.post.return_value = _make_response( 200, @@ -2811,7 +2811,7 @@ def test_assign_to_resolves_email_and_attributes_channel( type=enums.UserEventTypeChoices.ASSIGN, ).exists() - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_assign_to_skips_unknown_ambiguous_and_viewer( self, mock_session, mock_check_spam @@ -2852,7 +2852,7 @@ def test_assign_to_skips_unknown_ambiguous_and_viewer( "auth_method": "jwt", }, ) - mock_check_spam.return_value = (False, None, None) + mock_check_spam.return_value = ("no action", None, None) mock_session.return_value.post.return_value = _make_response( 200, body=json.dumps( @@ -2879,7 +2879,7 @@ def test_assign_to_skips_unknown_ambiguous_and_viewer( ) assert assignees == [editor_user.id] - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_two_webhooks_each_produce_own_threadevent( self, mock_session, mock_check_spam @@ -2923,7 +2923,7 @@ def test_two_webhooks_each_produce_own_threadevent( "auth_method": "jwt", }, ) - mock_check_spam.return_value = (False, None, None) + mock_check_spam.return_value = ("no action", None, None) # Each webhook returns a distinct assignee. The mock fires both # in order (dispatcher iterates channels in DB order). mock_session.return_value.post.side_effect = [ @@ -2979,10 +2979,10 @@ def _send(self, mailbox, mime_id, action_body: bytes): }, ) with ( - patch("core.mda.inbound_pipeline._call_rspamd") as mock_rspamd, + patch("core.mda.spam.call_rspamd") as mock_rspamd, patch("core.mda.dispatch_webhooks.SSRFSafeSession") as mock_session, ): - mock_rspamd.return_value = (False, None, None) + mock_rspamd.return_value = ("no action", None, None) mock_session.return_value.post.return_value = _make_response( 200, body=action_body ) @@ -3012,7 +3012,7 @@ def test_mark_trashed_and_archived_set_message_fields(self): assert message.is_archived is True @patch("core.mda.autoreply.try_send_autoreply") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_skip_autoreply_suppresses_autoreply_call( self, mock_session, mock_rspamd, mock_autoreply @@ -3037,7 +3037,7 @@ def test_skip_autoreply_suppresses_autoreply_call( "auth_method": "jwt", }, ) - mock_rspamd.return_value = (False, None, None) + mock_rspamd.return_value = ("no action", None, None) mock_session.return_value.post.return_value = _make_response( 200, body=b'{"skip_autoreply": true}' ) @@ -3055,7 +3055,7 @@ class TestPipelineWebhookAddEvent: are silently skipped at the classifier (the contract stays forward- compatible for future ``type=iframe``).""" - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_add_event_im_creates_threadevent(self, mock_session, mock_rspamd): mailbox = factories.MailboxFactory() @@ -3075,7 +3075,7 @@ def test_add_event_im_creates_threadevent(self, mock_session, mock_rspamd): "auth_method": "jwt", }, ) - mock_rspamd.return_value = (False, None, None) + mock_rspamd.return_value = ("no action", None, None) mock_session.return_value.post.return_value = _make_response( 200, body=json.dumps( @@ -3116,7 +3116,7 @@ class TestPipelineWebhookReplyDraft: user can refine the draft inline before sending — same UI affordance as a hand-composed draft.""" - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_reply_draft_creates_draft_with_template_body( self, mock_session, mock_rspamd @@ -3146,7 +3146,7 @@ def test_reply_draft_creates_draft_with_template_body( "auth_method": "jwt", }, ) - mock_rspamd.return_value = (False, None, None) + mock_rspamd.return_value = ("no action", None, None) mock_session.return_value.post.return_value = _make_response( 200, body=json.dumps({"reply_draft": {"template": str(template.id)}}).encode( @@ -3176,7 +3176,7 @@ def test_reply_draft_creates_draft_with_template_body( # And the bytes are exactly the template's raw_body json. assert draft.draft_blob.get_content() == template.raw_body.encode("utf-8") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_reply_draft_out_of_scope_template_skipped(self, mock_session, mock_rspamd): """A template belonging to a different mailbox / maildomain @@ -3205,7 +3205,7 @@ def test_reply_draft_out_of_scope_template_skipped(self, mock_session, mock_rspa "auth_method": "jwt", }, ) - mock_rspamd.return_value = (False, None, None) + mock_rspamd.return_value = ("no action", None, None) mock_session.return_value.post.return_value = _make_response( 200, body=json.dumps({"reply_draft": {"template": str(template.id)}}).encode( @@ -3232,7 +3232,7 @@ class TestFinalizeStepIsolation: log loudly rather than swallow other receiver-requested changes.""" @patch("core.mda.inbound_tasks.apply_pending_assigns") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_assign_failure_does_not_skip_labels( self, mock_session, mock_rspamd, mock_apply_assigns @@ -3265,7 +3265,7 @@ def test_assign_failure_does_not_skip_labels( "auth_method": "jwt", }, ) - mock_rspamd.return_value = (False, None, None) + mock_rspamd.return_value = ("no action", None, None) mock_session.return_value.post.return_value = _make_response( 200, body=json.dumps( @@ -3295,7 +3295,7 @@ class TestDeferralDelivery: autoreply is suppressed.""" @patch("core.mda.autoreply.try_send_autoreply") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_deferral_delivers_message_past_window( self, mock_session, mock_rspamd, mock_autoreply @@ -3324,7 +3324,7 @@ def test_deferral_delivers_message_past_window( "auth_method": "jwt", }, ) - mock_rspamd.return_value = (False, None, None) + mock_rspamd.return_value = ("no action", None, None) # Non-2xx triggers RETRY from the blocking webhook. mock_session.return_value.post.return_value = _make_response(503) @@ -3346,7 +3346,7 @@ def test_deferral_delivers_message_past_window( mock_autoreply.assert_not_called() @patch("core.mda.autoreply.try_send_autoreply") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_deferral_force_is_spam_false( self, mock_session, mock_rspamd, mock_autoreply @@ -3379,7 +3379,7 @@ def test_deferral_force_is_spam_false( ) # Rspamd votes spam, webhook errors → RETRY → force-delivery (once the # deferral window expires) should still force is_spam=False. - mock_rspamd.return_value = (True, None, None) + mock_rspamd.return_value = ("reject", None, None) mock_session.return_value.post.return_value = _make_response(503) with patch.object(process_inbound_message_task, "update_state", Mock()): @@ -3426,7 +3426,7 @@ def test_sweep_skips_abandoned_rows(self): assert str(abandoned.id) not in dispatched @patch("core.mda.inbound_tasks._create_message_from_inbound") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") def test_redispatch_of_abandoned_row_early_returns(self, mock_rspamd, mock_create): """A direct re-dispatch of an abandoned row early-returns ``{"error": "abandoned"}`` without ever running the pipeline — @@ -3504,7 +3504,7 @@ class TestPipelineIdempotency: duplicate them.""" @patch("core.mda.autoreply.try_send_autoreply") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_dedup_hit_skips_finalize_side_effects( self, mock_session, mock_rspamd, mock_autoreply @@ -3527,7 +3527,7 @@ def test_dedup_hit_skips_finalize_side_effects( "auth_method": "jwt", }, ) - mock_rspamd.return_value = (False, None, None) + mock_rspamd.return_value = ("no action", None, None) action_body = json.dumps( { "add_event": [{"type": "im", "content": "AI: urgent"}], @@ -3585,7 +3585,7 @@ def draft_count(): assert not models.InboundMessage.objects.filter(id=im2.id).exists() @patch("core.mda.dispatch_webhooks.dispatch_webhook_task.delay") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") def test_dedup_hit_does_not_refire_nonblocking_webhook( self, mock_rspamd, mock_delay ): @@ -3604,7 +3604,7 @@ def test_dedup_hit_does_not_refire_nonblocking_webhook( "auth_method": "jwt", }, ) - mock_rspamd.return_value = (False, None, None) + mock_rspamd.return_value = ("no action", None, None) mime = "idem-nb@example.com" raw_data = ( @@ -3768,7 +3768,7 @@ def test_cache_hit_skips_post_but_applies_side_effects( assert ctx.is_spam is True assert label_id in ctx.labels - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_blocking_webhook_not_refired_across_retries( self, mock_session, mock_rspamd @@ -3796,7 +3796,7 @@ def test_blocking_webhook_not_refired_across_retries( # Webhook succeeds (empty 200 → no is_spam opinion, so the later # rspamd step still runs); rspamd is erroring → pipeline RETRYs. mock_session.return_value.post.return_value = _make_response(200) - mock_rspamd.return_value = (False, "rspamd unreachable", None) + mock_rspamd.return_value = (None, "rspamd unreachable", None) # Attempt 1: webhook POSTed once, message held for retry. with patch.object(process_inbound_message_task, "update_state", Mock()): @@ -3816,7 +3816,7 @@ def test_blocking_webhook_not_refired_across_retries( @patch("core.mda.inbound_tasks.persist_cached_webhook_results") @patch("core.mda.inbound_tasks.load_cached_webhook_results") - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.dispatch_webhooks.SSRFSafeSession") def test_happy_path_does_not_touch_cache( self, mock_session, mock_rspamd, mock_load, mock_persist @@ -3840,7 +3840,7 @@ def test_happy_path_does_not_touch_cache( "auth_method": "jwt", }, ) - mock_rspamd.return_value = (False, None, None) + mock_rspamd.return_value = ("no action", None, None) mock_session.return_value.post.return_value = _make_response(200) with patch.object(process_inbound_message_task, "update_state", Mock()): diff --git a/src/backend/core/tests/mda/test_inbound_auth.py b/src/backend/core/tests/mda/test_inbound_auth.py index 26ea0b280..b4de4312f 100644 --- a/src/backend/core/tests/mda/test_inbound_auth.py +++ b/src/backend/core/tests/mda/test_inbound_auth.py @@ -670,7 +670,7 @@ def test_verified_records_no_auth(self, mock_create_message, mock_auth_check): "inbound_auth": "rspamd", } ) - @patch("core.mda.inbound_pipeline.requests.post") + @patch("core.mda.spam.requests.post") @patch("core.mda.inbound_tasks._create_message_from_inbound") def test_rspamd_response_reused_by_auth_check(self, mock_create_message, mock_post): """Single rspamd call feeds both spam and auth.""" @@ -700,7 +700,7 @@ def test_rspamd_response_reused_by_auth_check(self, mock_create_message, mock_po "inbound_auth": "rspamd", } ) - @patch("core.mda.inbound_pipeline.requests.post") + @patch("core.mda.spam.requests.post") @patch("core.mda.inbound_tasks._create_message_from_inbound") def test_dmarc_fail_injects_fail_header_end_to_end( self, mock_create_message, mock_post @@ -734,7 +734,7 @@ def test_dmarc_fail_injects_fail_header_end_to_end( "rules": [{"header_match": "X-Spam:yes", "action": "ham"}], } ) - @patch("core.mda.inbound_pipeline.requests.post") + @patch("core.mda.spam.requests.post") @patch("core.mda.inbound_tasks._create_message_from_inbound") def test_rspamd_fetched_on_demand_when_spam_skipped_rspamd( self, mock_create_message, mock_post diff --git a/src/backend/core/tests/mda/test_inbound_spoofed_sender.py b/src/backend/core/tests/mda/test_inbound_spoofed_sender.py index 96536659f..48ae93112 100644 --- a/src/backend/core/tests/mda/test_inbound_spoofed_sender.py +++ b/src/backend/core/tests/mda/test_inbound_spoofed_sender.py @@ -52,7 +52,7 @@ def victim_mailbox(): class TestInboundSpoofedSender: """Inbound delivery with ``From == To`` must not flag ``is_sender=True``.""" - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") def test_inbound_spoofed_sender_not_marked_as_sender( self, mock_rspamd, victim_mailbox ): @@ -63,7 +63,7 @@ def test_inbound_spoofed_sender_not_marked_as_sender( message because of the ``sender_email == recipient_email`` shortcut. """ # Rspamd doesn't catch this kind of spoof in the current config. - mock_rspamd.return_value = (False, None, None) + mock_rspamd.return_value = ("no action", None, None) recipient = str(victim_mailbox) raw = _build_spoofed_raw(recipient) @@ -92,7 +92,7 @@ def test_inbound_spoofed_sender_not_marked_as_sender( "as is_sender=True; otherwise retry_messages_task re-emits them." ) - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.outbound_tasks.send_message") def test_inbound_spoofed_sender_not_picked_up_by_retry( self, mock_send_message, mock_rspamd, victim_mailbox @@ -103,7 +103,7 @@ def test_inbound_spoofed_sender_not_picked_up_by_retry( the retry pipeline must not invoke ``send_message`` on it, because that would DKIM-sign and re-emit the spam. """ - mock_rspamd.return_value = (False, None, None) + mock_rspamd.return_value = ("no action", None, None) recipient = str(victim_mailbox) raw = _build_spoofed_raw(recipient) diff --git a/src/backend/core/tests/mda/test_spam_processing.py b/src/backend/core/tests/mda/test_spam_processing.py index 13341cef3..91e82165d 100644 --- a/src/backend/core/tests/mda/test_spam_processing.py +++ b/src/backend/core/tests/mda/test_spam_processing.py @@ -16,14 +16,13 @@ from core.mda.inbound_pipeline import ( Decision, InboundContext, - _call_rspamd, - _check_hardcoded_rules, _make_rspamd_step, ) from core.mda.inbound_tasks import ( process_inbound_message_task, process_inbound_messages_queue_task, ) +from core.mda.spam import call_rspamd, check_hardcoded_rules def _queue_inbound(mailbox, content=b"raw", envelope=None): @@ -107,7 +106,7 @@ class TestRspamdSpamCheck: """Test rspamd spam checking functionality.""" @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) - @patch("core.mda.inbound_pipeline.requests.post") + @patch("core.mda.spam.requests.post") def test_check_spam_with_rspamd_spam(self, mock_post): """Test that spam messages are correctly identified.""" spam_config = {"rspamd_url": "http://rspamd:8010/_api"} @@ -121,9 +120,9 @@ def test_check_spam_with_rspamd_spam(self, mock_post): mock_post.return_value = mock_response raw_data = b"Spam email content" - action, error, rspamd_result = _call_rspamd(raw_data, spam_config) + action, error, rspamd_result = call_rspamd(raw_data, spam_config) - # _call_rspamd returns the raw action; the action -> verdict mapping + # call_rspamd returns the raw action; the action -> verdict mapping # lives in the step (see TestRspamdStepFailureHandling). assert action == "reject" assert error is None @@ -134,7 +133,7 @@ def test_check_spam_with_rspamd_spam(self, mock_post): assert call_args[1]["data"] == raw_data @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) - @patch("core.mda.inbound_pipeline.requests.post") + @patch("core.mda.spam.requests.post") def test_check_spam_with_rspamd_not_spam(self, mock_post): """Test that non-spam messages are correctly identified.""" spam_config = {"rspamd_url": "http://rspamd:8010/_api"} @@ -148,13 +147,13 @@ def test_check_spam_with_rspamd_not_spam(self, mock_post): mock_post.return_value = mock_response raw_data = b"Legitimate email content" - action, error, _rspamd_result = _call_rspamd(raw_data, spam_config) + action, error, _rspamd_result = call_rspamd(raw_data, spam_config) assert action == "no action" assert error is None @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) - @patch("core.mda.inbound_pipeline.requests.post") + @patch("core.mda.spam.requests.post") def test_call_rspamd_forwards_smtp_envelope_headers(self, mock_post): """The SMTP envelope is forwarded via rspamd's scan headers, and CR/LF in attacker-influenced fields (HELO/hostname) is stripped.""" @@ -172,7 +171,7 @@ def test_call_rspamd_forwards_smtp_envelope_headers(self, mock_post): "helo": "evil\r\nX-Injected: 1", "hostname": "mail.example.com", } - _call_rspamd(b"content", spam_config, envelope=envelope) + call_rspamd(b"content", spam_config, envelope=envelope) headers = mock_post.call_args[1]["headers"] assert headers["From"] == "sender@example.com" @@ -184,7 +183,7 @@ def test_call_rspamd_forwards_smtp_envelope_headers(self, mock_post): assert "\r" not in headers["Helo"] and "\n" not in headers["Helo"] @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) - @patch("core.mda.inbound_pipeline.requests.post") + @patch("core.mda.spam.requests.post") def test_call_rspamd_omits_absent_envelope_headers(self, mock_post): """Fields we don't have (widget/internal mail has no HELO/rDNS) are omitted rather than sent empty, which would skew scoring.""" @@ -194,7 +193,7 @@ def test_call_rspamd_omits_absent_envelope_headers(self, mock_post): mock_response.raise_for_status = Mock() mock_post.return_value = mock_response - _call_rspamd( + call_rspamd( b"content", spam_config, envelope={"origin": "widget", "mail_from": "u@example.com"}, @@ -213,7 +212,7 @@ def test_call_rspamd_omits_absent_envelope_headers(self, mock_post): "rspamd_auth": "Bearer token123", } ) - @patch("core.mda.inbound_pipeline.requests.post") + @patch("core.mda.spam.requests.post") def test_check_spam_with_rspamd_auth_header(self, mock_post): """Test that Authorization header is included when configured.""" spam_config = { @@ -230,7 +229,7 @@ def test_check_spam_with_rspamd_auth_header(self, mock_post): mock_post.return_value = mock_response raw_data = b"Email content" - _call_rspamd(raw_data, spam_config) + call_rspamd(raw_data, spam_config) call_args = mock_post.call_args assert call_args[1]["headers"]["Authorization"] == "Bearer token123" @@ -242,14 +241,14 @@ def test_check_spam_without_rspamd_config(self): verdict instead of silently marking the message as ham.""" spam_config = {} raw_data = b"Email content" - action, error, rspamd_result = _call_rspamd(raw_data, spam_config) + action, error, rspamd_result = call_rspamd(raw_data, spam_config) assert action is None assert error is None assert rspamd_result is None @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) - @patch("core.mda.inbound_pipeline.requests.post") + @patch("core.mda.spam.requests.post") def test_check_spam_with_rspamd_error(self, mock_post): """On error, ``action`` is None and the error is surfaced separately; the step turns that into a RETRY (never fails open).""" @@ -257,7 +256,7 @@ def test_check_spam_with_rspamd_error(self, mock_post): mock_post.side_effect = requests.exceptions.RequestException("Connection error") raw_data = b"Email content" - action, error, rspamd_result = _call_rspamd(raw_data, spam_config) + action, error, rspamd_result = call_rspamd(raw_data, spam_config) assert action is None assert error is not None @@ -269,7 +268,7 @@ def test_check_spam_with_rspamd_error(self, mock_post): "rspamd_auth": "Bearer global", } ) - @patch("core.mda.inbound_pipeline.requests.post") + @patch("core.mda.spam.requests.post") def test_check_spam_with_maildomain_override(self, mock_post): """Test that maildomain custom_settings can override SPAM_CONFIG.""" # Create a maildomain with custom spam config @@ -294,7 +293,7 @@ def test_check_spam_with_maildomain_override(self, mock_post): spam_config = mailbox.domain.get_spam_config() raw_data = b"Email content" - _call_rspamd(raw_data, spam_config) + call_rspamd(raw_data, spam_config) # Verify that the domain-specific URL was used call_args = mock_post.call_args @@ -318,7 +317,7 @@ def test_check_spam_with_hardcoded_rules_spam(self): parsed_email = parse_email(raw_email) spam_config = {"rules": [{"header_match": "X-Spam:yes", "action": "spam"}]} - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -334,7 +333,7 @@ def test_check_spam_with_hardcoded_rules_ham(self): parsed_email = parse_email(raw_email) spam_config = {"rules": [{"header_match": "X-Spam:no", "action": "ham"}]} - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is False @@ -350,7 +349,7 @@ def test_check_spam_with_hardcoded_rules_no_match(self): parsed_email = parse_email(raw_email) spam_config = {"rules": [{"header_match": "X-Spam:yes", "action": "spam"}]} - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is None @@ -366,7 +365,7 @@ def test_check_spam_with_hardcoded_rules_no_rules(self): parsed_email = parse_email(raw_email) spam_config = {} - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is None @@ -387,7 +386,7 @@ def test_check_spam_with_hardcoded_rules_multiple_headers(self): parsed_email = parse_email(raw_email) spam_config = {"rules": [{"header_match": "X-Spam:no", "action": "ham"}]} - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is False @@ -403,7 +402,7 @@ def test_check_spam_with_hardcoded_rules_case_insensitive(self): parsed_email = parse_email(raw_email) spam_config = {"rules": [{"header_match": "X-Spam:yes", "action": "spam"}]} - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -421,7 +420,7 @@ def test_check_spam_with_hardcoded_rules_value_with_colon(self): "rules": [{"header_match": "X-Custom:value:with:colons", "action": "spam"}] } - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -439,7 +438,7 @@ def test_check_spam_with_hardcoded_rules_header_match_regex_spam(self): "rules": [{"header_match_regex": "X-Spam:.*spam.*", "action": "spam"}] } - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -457,7 +456,7 @@ def test_check_spam_with_hardcoded_rules_header_match_regex_spam_no_fullmatch(se "rules": [{"header_match_regex": "X-Spam:spam", "action": "spam"}] } - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is None @@ -475,7 +474,7 @@ def test_check_spam_with_hardcoded_rules_header_match_regex_case_insensitive(sel "rules": [{"header_match_regex": "X-Spam:.*spam.*", "action": "spam"}] } - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -493,7 +492,7 @@ def test_check_spam_with_hardcoded_rules_header_match_regex_pattern(self): "rules": [{"header_match_regex": "X-Spam-Level:[4-9]", "action": "spam"}] } - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -513,7 +512,7 @@ def test_check_spam_with_hardcoded_rules_default_action(self): ] } - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -529,7 +528,7 @@ def test_check_spam_with_hardcoded_rules_reject_action(self): parsed_email = parse_email(raw_email) spam_config = {"rules": [{"header_match": "X-Spam:yes", "action": "reject"}]} - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -545,7 +544,7 @@ def test_check_spam_with_hardcoded_rules_no_action(self): parsed_email = parse_email(raw_email) spam_config = {"rules": [{"header_match": "X-Spam:no", "action": "no action"}]} - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is False @@ -571,7 +570,7 @@ def test_check_spam_with_hardcoded_rules_multiple_rules_order(self): ] } - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) # Should return False (ham) because second rule matched first # Third rule should not be evaluated @@ -596,7 +595,7 @@ def test_check_spam_with_hardcoded_rules_multiple_rules_first_match_wins(self): ] } - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) # Should return True (spam) because first rule matched # Second rule should not be evaluated @@ -624,7 +623,7 @@ def test_check_spam_with_hardcoded_rules_x_spam_single_relay(self): "trusted_relays": 1, # Trust block 0 and block 1 } - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is True @@ -663,7 +662,7 @@ def test_check_spam_with_hardcoded_rules_x_spam_raw_email_relay_no_header(self): "trusted_relays": 1, # Trust block 0 and block 1 } - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) # Should return None (no match) because sender's X-Spam is in block 2, not in trusted blocks assert result is None @@ -707,7 +706,7 @@ def test_check_spam_with_hardcoded_rules_x_spam_raw_email_with_relay( "trusted_relays": 1, # Trust block 0 and block 1 } - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) # Should match the first X-Spam header (No from last relay), not the sender's (Yes) assert result is False # ham = False (not spam) @@ -766,7 +765,7 @@ def test_check_spam_with_hardcoded_rules_trusted_relays( "trusted_relays": trusted_relays_setting, } - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is expected_result def test_default_ignores_sender_injected_ham_header(self): @@ -800,7 +799,7 @@ def test_default_ignores_sender_injected_ham_header(self): ], } - result = _check_hardcoded_rules(parsed_email, spam_config) + result = check_hardcoded_rules(parsed_email, spam_config) assert result is None # forged ham not honoured @@ -809,7 +808,7 @@ class TestProcessInboundMessageTask: """Test the process_inbound_message_task.""" @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.inbound_tasks._create_message_from_inbound") def test_process_inbound_message_task_spam( self, mock_create_message, mock_check_spam @@ -820,7 +819,7 @@ def test_process_inbound_message_task_spam( inbound_message = _queue_inbound(mailbox, raw_data) - mock_check_spam.return_value = (True, None, None) # is_spam=True + mock_check_spam.return_value = ("reject", None, None) # spam mock_create_message.return_value = True # Call the bound task directly using .run() method @@ -839,7 +838,7 @@ def test_process_inbound_message_task_spam( assert not models.InboundMessage.objects.filter(id=inbound_message.id).exists() @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.inbound_tasks._create_message_from_inbound") def test_process_inbound_message_task_not_spam( self, mock_create_message, mock_check_spam @@ -850,7 +849,7 @@ def test_process_inbound_message_task_not_spam( inbound_message = _queue_inbound(mailbox, raw_data) - mock_check_spam.return_value = (False, None, None) # is_spam=False + mock_check_spam.return_value = ("no action", None, None) # ham mock_create_message.return_value = True # Call the bound task directly using .run() method @@ -865,7 +864,7 @@ def test_process_inbound_message_task_not_spam( assert call_kwargs["is_spam"] is False @override_settings(SPAM_CONFIG={"rspamd_url": "http://rspamd:8010/_api"}) - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") @patch("core.mda.inbound_tasks._create_message_from_inbound") def test_process_inbound_message_task_failure( self, mock_create_message, mock_check_spam @@ -876,7 +875,7 @@ def test_process_inbound_message_task_failure( inbound_message = _queue_inbound(mailbox, raw_data) - mock_check_spam.return_value = (False, None, None) + mock_check_spam.return_value = ("no action", None, None) mock_create_message.return_value = False # Creation failed # Call the bound task directly using .run() method @@ -937,11 +936,11 @@ def _ctx(self, spam_config): spam_config=spam_config, ) - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") def test_error_holds_for_retry(self, mock_call): """On rspamd error, never fail open — hold the message for retry.""" spam_config = {"rspamd_url": "http://rspamd:11334"} - # On error _call_rspamd returns action=None + an error message. + # On error call_rspamd returns action=None + an error message. mock_call.return_value = (None, "connection refused", None) decision = _make_rspamd_step(spam_config)(self._ctx(spam_config)) @@ -949,7 +948,7 @@ def test_error_holds_for_retry(self, mock_call): # Never fail open — hold, don't deliver unchecked. assert decision == Decision.RETRY - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") def test_not_configured_continues(self, mock_call): """When rspamd isn't configured, continue without a verdict.""" # rspamd absent is "no opinion" (action=None, no error) → keep moving. @@ -975,7 +974,7 @@ def test_not_configured_continues(self, mock_call): ("discard", Decision.DROP, None, None), ], ) - @patch("core.mda.inbound_pipeline._call_rspamd") + @patch("core.mda.spam.call_rspamd") def test_action_mapping(self, mock_call, action, decision, is_spam, marker): """Every rspamd action maps to (decision, Junk verdict, spam marker): isolate → Junk, flag → inbox+marker, defer → RETRY, discard → DROP.""" From 897b35aaf2e53a6f772089ded21e4ee9a2aa8f73 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Wed, 1 Jul 2026 15:10:08 +0200 Subject: [PATCH 20/21] review fixes --- docs/env.md | 2 +- docs/tiered-storage.md | 4 +- docs/webhooks.md | 12 +++-- src/backend/core/admin.py | 13 +++++- src/backend/core/api/serializers.py | 14 +++--- src/backend/core/api/viewsets/inbound/mta.py | 4 +- .../core/api/viewsets/inbound/widget.py | 4 +- src/backend/core/enums.py | 41 +++++++++++++++++ src/backend/core/mda/dispatch_webhooks.py | 34 ++++++++++++-- src/backend/core/mda/inbound.py | 10 ++-- src/backend/core/mda/inbound_auth.py | 6 +-- src/backend/core/mda/inbound_create.py | 2 +- src/backend/core/mda/inbound_pipeline.py | 46 +++++++++++-------- src/backend/core/mda/inbound_tasks.py | 43 +++++++++++++---- src/backend/core/mda/outbound.py | 4 +- src/backend/core/mda/selfcheck.py | 14 +++--- src/backend/core/mda/spam.py | 21 ++++++++- .../0032_inboundmessage_blob_envelope.py | 6 ++- src/backend/core/models.py | 30 ++++++------ src/backend/core/services/ssrf.py | 20 ++++++++ src/backend/core/signals.py | 15 +++--- .../admin/core/channel/change_form.html | 3 +- .../core/tests/api/test_threads_list.py | 7 ++- src/backend/core/tests/mda/test_autoreply.py | 7 +++ src/backend/core/tests/mda/test_retry.py | 14 +++++- .../core/tests/mda/test_spam_processing.py | 21 +++++++++ src/backend/core/tests/services/test_ssrf.py | 20 +++++++- src/frontend/public/locales/common/en-US.json | 1 + src/frontend/public/locales/common/fr-FR.json | 1 + src/frontend/public/locales/common/nl-NL.json | 1 + src/frontend/public/locales/common/ru-RU.json | 1 + src/frontend/public/locales/common/uk-UA.json | 1 + .../webhook-integration-form.tsx | 14 ++++-- .../thread-event/assignment-message.ts | 8 ++-- 34 files changed, 336 insertions(+), 108 deletions(-) diff --git a/docs/env.md b/docs/env.md index c0a255af9..03f65af0b 100644 --- a/docs/env.md +++ b/docs/env.md @@ -244,7 +244,7 @@ _Those settings are deprecated and will be removed in the future._ | `CORS_ALLOWED_ORIGINS` | `[]` | Specific allowed CORS origins | Optional | | `CORS_ALLOWED_ORIGIN_REGEXES` | `[]` | Regex patterns for allowed origins | Optional | | `CSRF_TRUSTED_ORIGINS` | `["http://localhost:8900", "http://localhost:8901"]` | Trusted origins for CSRF | Optional | -| `ALLOWED_HOSTS` | `[]` | Host/domain allow-list (Production configuration; the Development class sets this from `DJANGO_ALLOWED_HOSTS`). | Optional | +| `ALLOWED_HOSTS` | `[]` | Django host/domain allow-list setting (`settings.ALLOWED_HOSTS`). In the Base/Production configurations it is populated from the `DJANGO_ALLOWED_HOSTS` env var (see above); the Development configuration hardcodes `["*"]`. The `bucket_cors` management command also reads it to build S3 CORS origins. | Optional | | `SERVER_TO_SERVER_API_TOKENS` | `[]` | API tokens for server-to-server auth | Optional | | `SALT_KEY` | `[]` | Key(s) for Django Fernet-encrypted model fields. Accepts a list for rotation (`["new_key", "old_key"]`); the first is used to encrypt, all are tried to decrypt. | Optional | diff --git a/docs/tiered-storage.md b/docs/tiered-storage.md index 60160c7e8..0eec09cb6 100644 --- a/docs/tiered-storage.md +++ b/docs/tiered-storage.md @@ -61,8 +61,8 @@ blob_id survives until the follow-up attach call lands; the attach flow drops it once the ``Attachment`` row exists. When a reference source is deleted (Message, Attachment, -MessageTemplate ``post_delete``), the affected blob_id is pushed -into a Redis candidate set. A periodic Celery task — +MessageTemplate, InboundMessage ``post_delete``), the affected blob_id +is pushed into a Redis candidate set. A periodic Celery task — ``gc_orphan_blobs_task`` in ``core/services/blob_gc.py`` — drains the set, re-checks the reference graph under the per-sha advisory lock, deletes the row if no references remain, and cleans up the S3 diff --git a/docs/webhooks.md b/docs/webhooks.md index 700713949..06d981675 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -105,8 +105,13 @@ Every call is: ### Authentication Every webhook channel has **one root secret**, minted server-side, -returned exactly once at create time and rotatable via -`POST /channels/{id}/regenerate-secret/`. The `auth_method` +returned exactly once at create time and rotatable by POSTing to the +channel's `regenerate-secret/` action. That action's path prefix +depends on the channel's scope: a mailbox-scoped channel is reached via +the mailbox route as `POST +/mailboxes/{mailbox_id}/channels/{id}/regenerate-secret/`, while a +caller's own channels are at `POST +/users/me/channels/{id}/regenerate-secret/`. The `auth_method` setting picks how that root is presented on each POST. The root itself never travels on the wire. @@ -136,7 +141,8 @@ derivation) on other receivers stays unforgeable. PATCH the channel's `settings.auth_method`. The root secret is **not** rotated — only the wire presentation changes — but the receiver was given the old method's credential at creation. To get the new method's -credential, call `POST /channels/{id}/regenerate-secret/`: the +credential, call the channel's `regenerate-secret/` action (scoped path +as above): the response returns either `secret` (jwt) or `api_key` (api_key), matching the channel's current method. Rotation invalidates the previous credential, so update the receiver before the next inbound diff --git a/src/backend/core/admin.py b/src/backend/core/admin.py index b802dc3df..7b24963be 100644 --- a/src/backend/core/admin.py +++ b/src/backend/core/admin.py @@ -1601,10 +1601,20 @@ def replay_abandoned(self, request, queryset): replayed = 0 for inbound_message in queryset.filter(abandoned_at__isnull=False): + # Publish to the broker *before* clearing the terminal markers: if + # the publish fails, leave the row abandoned (with its error) so the + # operator can retry, rather than erasing the state without a re-queue. + try: + process_inbound_message_task.delay(str(inbound_message.id)) + except Exception: # pylint: disable=broad-except + logging.exception( + "Failed to re-queue abandoned inbound message %s", + inbound_message.id, + ) + continue inbound_message.abandoned_at = None inbound_message.error_message = "" inbound_message.save(update_fields=["abandoned_at", "error_message"]) - process_inbound_message_task.delay(str(inbound_message.id)) replayed += 1 self.message_user(request, f"Re-queued {replayed} abandoned message(s).") @@ -1614,7 +1624,6 @@ def get_queryset(self, request): super() .get_queryset(request) .select_related("mailbox", "mailbox__domain", "channel") - .defer("raw_data") # Exclude large binary content from list view ) diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index 82ccd995b..7ea1873d7 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -2380,13 +2380,13 @@ def validate(self, attrs): explicit mailbox or maildomain. """ if self.context.get("mailbox") or self.context.get("user_channel"): - # On CREATE, ``type`` MUST be supplied explicitly. The model - # field used to default to "mta", which let a caller omit - # ``type`` from the body and bypass FEATURE_MAILBOX_ADMIN_CHANNELS - # even when "mta" was not in the allowlist. On UPDATE, ``type`` - # is made read-only by ``CreateOnlyFieldsMixin`` so it's never - # in ``attrs`` — fall through to the other validators which - # read the instance's existing type. + # On CREATE, ``type`` MUST be supplied explicitly: a default + # would let a caller omit ``type`` from the body and slip past + # FEATURE_MAILBOX_ADMIN_CHANNELS even when that default type + # isn't in the allowlist. On UPDATE, ``type`` is made read-only + # by ``CreateOnlyFieldsMixin`` so it's never in ``attrs`` — fall + # through to the other validators which read the instance's + # existing type. if self.instance is None and "type" not in attrs: raise serializers.ValidationError( { diff --git a/src/backend/core/api/viewsets/inbound/mta.py b/src/backend/core/api/viewsets/inbound/mta.py index 6a18eba4e..34e88ffec 100644 --- a/src/backend/core/api/viewsets/inbound/mta.py +++ b/src/backend/core/api/viewsets/inbound/mta.py @@ -16,7 +16,7 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response -from core import models +from core import enums, models from core.mda.inbound import check_local_recipients, deliver_inbound_message from core.mda.raw_mime import remove_mime_headers @@ -238,7 +238,7 @@ def sanitize_header(header: str) -> str: # are the connecting SMTP peer (forwarded by the MTA). ``rcpt_to`` is # the actual RCPT TO and is set per recipient inside the loop. base_envelope = { - "origin": "mta", + "origin": enums.InboundOrigin.MTA, "mail_from": mta_metadata.get("sender", ""), "ip": mta_metadata.get("client_address", ""), "helo": mta_metadata.get("client_helo", ""), diff --git a/src/backend/core/api/viewsets/inbound/widget.py b/src/backend/core/api/viewsets/inbound/widget.py index db696f68e..0520784e5 100644 --- a/src/backend/core/api/viewsets/inbound/widget.py +++ b/src/backend/core/api/viewsets/inbound/widget.py @@ -15,7 +15,7 @@ from rest_framework.response import Response from rest_framework.throttling import SimpleRateThrottle -from core import models +from core import enums, models from core.api.permissions import IsAuthenticated from core.mda.inbound import deliver_inbound_message from core.mda.utils import current_sent_at @@ -242,7 +242,7 @@ def sanitize_header(header: str) -> str: compose_email(parsed_email, prepend_headers=prepend_headers), channel=channel, envelope={ - "origin": "widget", + "origin": enums.InboundOrigin.WIDGET, "mail_from": sender_email, "rcpt_to": target_email, "ip": request.META.get("REMOTE_ADDR", ""), diff --git a/src/backend/core/enums.py b/src/backend/core/enums.py index c64bc2fbb..8d4655b7f 100644 --- a/src/backend/core/enums.py +++ b/src/backend/core/enums.py @@ -85,6 +85,27 @@ class MessageDeliveryStatusChoices(models.IntegerChoices): CANCELLED = 5, "cancelled" +# Single source of truth for the "delivered"-class terminal statuses, so the +# SENT_INTERNAL/SENT_EXTERNAL footgun documented above is encoded once instead +# of re-derived (and mis-derived) at every call site. +DELIVERED_STATUSES = frozenset( + { + MessageDeliveryStatusChoices.SENT_INTERNAL, + MessageDeliveryStatusChoices.SENT_EXTERNAL, + } +) + + +def is_delivered(status) -> bool: + """Whether a delivery status is a terminal "delivered" one. + + Treats SENT_INTERNAL and SENT_EXTERNAL identically — the only correct way + to answer "did it send?" (see the FOOTGUN note on + ``MessageDeliveryStatusChoices``). + """ + return status in DELIVERED_STATUSES + + class MailDomainAccessRoleChoices(models.IntegerChoices): """Defines the unique roles a user can have to access a mail domain.""" @@ -235,6 +256,26 @@ class ChannelTypes(StrEnum): CALDAV = "caldav" +class InboundOrigin(StrEnum): + """Known ``InboundMessage.envelope["origin"]`` values — the trust + discriminator each ingest path sets EXPLICITLY (see ``is_internal``). + + ``StrEnum`` (not a Django ``TextChoices``): origin is a free-form key in a + JSONField, so adding one never requires a migration. Members ARE strings + (``InboundOrigin.MTA == "mta"``) so comparisons and dict values work + transparently. + + ``IMPORT`` is a known/reserved value: imports bypass the inbound queue and + so no producer sets it today, but it is kept here as the documented origin + for that path. + """ + + MTA = "mta" + INTERNAL = "internal" + WIDGET = "widget" + IMPORT = "import" + + class WebhookTrigger(StrEnum): """The lifecycle event that fires a webhook, stored as ``Channel.settings["trigger"]`` and surfaced verbatim in the diff --git a/src/backend/core/mda/dispatch_webhooks.py b/src/backend/core/mda/dispatch_webhooks.py index 972e1e831..323fa7fff 100644 --- a/src/backend/core/mda/dispatch_webhooks.py +++ b/src/backend/core/mda/dispatch_webhooks.py @@ -313,8 +313,15 @@ def _sanitize_url(url: str) -> str: if not parsed.hostname: return "" host = parsed.hostname - if parsed.port: - host = f"{host}:{parsed.port}" + try: + # ``.port`` re-parses the netloc and raises ValueError on a malformed + # port even though ``urlparse`` itself succeeded — swallow it and log + # host-only rather than let a bad URL crash the logging path. + port = parsed.port + except ValueError: + port = None + if port: + host = f"{host}:{port}" return f"{parsed.scheme}://{host}" @@ -1084,8 +1091,8 @@ def dispatch_webhook_task( Non-blocking webhooks can't influence delivery, so their network I/O runs here (default queue) instead of pinning the time-sensitive inbound pipeline worker. Best-effort and at-least-once: the message - is already handled, so any failure is logged and swallowed (matching - the previous inline non-blocking contract). The request is re-signed + is already handled, so any failure is logged and swallowed (a + non-blocking webhook never affects delivery). The request is re-signed here at send time, so the root secret never travels through the broker and the JWT TTL is measured from the actual POST. @@ -1100,6 +1107,25 @@ def dispatch_webhook_task( mailbox = models.Mailbox.objects.filter(id=mailbox_id).first() if channel is None or mailbox is None: return + # Re-validate the channel at send time: it may have been retyped, + # re-triggered, or re-scoped between the pipeline recording it and this + # task running. Skip (rather than post) if it no longer applies. The + # scope-matcher already filters to type=webhook channels covering this + # mailbox, so channel membership subsumes the type + mailbox checks. + applicable = {c.id for c in find_webhook_channels_for_mailbox(mailbox)} + trigger = (channel.settings or {}).get("trigger") + if ( + channel.id not in applicable + or trigger != enums.WebhookTrigger.MESSAGE_DELIVERED + ): + logger.warning( + "Webhook channel %s no longer applies to mailbox %s " + "(trigger=%r) — skipping dispatch", + channel_id, + mailbox_id, + trigger, + ) + return message = models.Message.objects.filter(id=message_id).first() if message is None or message.blob_id is None: logger.warning( diff --git a/src/backend/core/mda/inbound.py b/src/backend/core/mda/inbound.py index 5d702539c..abd8a7951 100644 --- a/src/backend/core/mda/inbound.py +++ b/src/backend/core/mda/inbound.py @@ -10,7 +10,7 @@ from jmap_email import JmapEmail, first_msgid -from core import models +from core import enums, models from core.mda.inbound_tasks import process_inbound_message_task from core.services.importer.labels import ( handle_duplicate_message, @@ -161,8 +161,6 @@ def deliver_inbound_message( (see ``InboundMessage.envelope``); its ``origin`` key is the explicit trust discriminator that drives ``is_internal`` — internal mail skips the spam steps while still firing user webhooks. - - raw_data is not parsed again, just stored as is. """ # --- 1. Find or Create Mailbox --- # try: @@ -215,7 +213,7 @@ def deliver_inbound_message( return bool(result) envelope = envelope or {} - is_internal = envelope.get("origin") == "internal" + is_internal = envelope.get("origin") == enums.InboundOrigin.INTERNAL # Internal mail is expected to reference the sender's already-committed # blob — that's the whole point (no second plaintext copy). Enforce the @@ -241,9 +239,9 @@ def deliver_inbound_message( channel=channel, ) logger.info( - "Queued inbound message %s (recipient: %s, origin: %s)", + "Queued inbound message %s (mailbox: %s, origin: %s)", inbound_message.id, - recipient_email, + mailbox.id, envelope.get("origin"), ) # Queue the task immediately for processing (no lag) diff --git a/src/backend/core/mda/inbound_auth.py b/src/backend/core/mda/inbound_auth.py index 8388a49f7..904dca3df 100644 --- a/src/backend/core/mda/inbound_auth.py +++ b/src/backend/core/mda/inbound_auth.py @@ -1,7 +1,7 @@ """Inbound sender authentication checks (DKIM / DMARC). -Returns the verdict the caller should stamp as ``X-StMsg-Sender-Auth``: - - ``None``: verified, do not add the header. +Returns the verdict the caller records in ``postmark["auth"]``: + - ``None``: verified, record nothing. - ``"none"``: cannot verify (missing DKIM, backend unreachable, no AR header from a trusted relay). Frontend shows a yellow "unverified" hint. - ``"fail"``: explicit forgery signal (DMARC fail). Frontend shows a red @@ -235,7 +235,7 @@ def check_inbound_authentication( spam_config: dict[str, Any], rspamd_result: dict[str, Any] | None = None, ) -> str | None: - """Return the ``X-StMsg-Sender-Auth`` verdict for this message. + """Return the sender-auth verdict for this message (``postmark["auth"]``). See module docstring for the rule set and supported backends. """ diff --git a/src/backend/core/mda/inbound_create.py b/src/backend/core/mda/inbound_create.py index 4b765bf83..89e85d5ae 100644 --- a/src/backend/core/mda/inbound_create.py +++ b/src/backend/core/mda/inbound_create.py @@ -124,7 +124,7 @@ def find_thread_for_inbound_message( return parent.thread # Found a match! # Strategy 2 (Fallback): If no subject match, return thread of the most recent parent message - # The list is ordered by -sent_at, so the first element is the latest match. + # The list is ordered by -created_at, so the first element is the latest match. return None # potential_parents.first().thread diff --git a/src/backend/core/mda/inbound_pipeline.py b/src/backend/core/mda/inbound_pipeline.py index d1193263a..0cbcf97ff 100644 --- a/src/backend/core/mda/inbound_pipeline.py +++ b/src/backend/core/mda/inbound_pipeline.py @@ -4,7 +4,7 @@ ``Message`` row" is a **Step**: a callable that takes an ``InboundContext`` and returns a ``Decision``. Steps may also mutate the context — set ``is_spam``, add ``labels``, cache ``rspamd_result``, -prepend an authentication header, etc. +record an auth verdict in ``postmark``, etc. ``build_inbound_pipeline`` assembles the ordered step list for a message — before-spam user webhooks, the hardcoded-rules and rspamd spam checks, the @@ -195,9 +195,12 @@ def _make_rspamd_step(spam_config: Dict[str, Any]) -> Step: (https://docs.rspamd.com/configuration/metrics/): * ``no action`` → deliver (is_spam=False). - * ``add header`` / ``rewrite subject`` / ``quarantine`` / ``reject`` → - spam verdict (is_spam=True → Junk). We can't honour ``reject`` at SMTP - time (already accepted), so it lands in Junk like the other markers. + * ``add header`` / ``rewrite subject`` → deliver to the inbox with a + graded ``postmark["spam"]`` marker (possible / likely) for the UI, not + hidden in Junk. + * ``quarantine`` / ``reject`` → spam verdict (is_spam=True → Junk). We + can't honour ``reject`` at SMTP time (already accepted), so it lands in + Junk. * ``greylist`` / ``soft reject`` → temporary failures, NOT verdicts: route onto our deferral path (RETRY). The condition (rate-limit, greylist, transient DNS) usually clears within the 5-min sweep; a persistent one @@ -293,14 +296,13 @@ def inbound_auth(ctx: InboundContext) -> Decision: # Widget submissions arrive over an unauthenticated web form, so # they carry the "none" baseline even when DKIM/DMARC verification # is disabled instance-wide (which is when ``verdict`` is falsy). - if (ctx.inbound_message.envelope or {}).get("origin") == "widget": + if (ctx.inbound_message.envelope or {}).get( + "origin" + ) == enums.InboundOrigin.WIDGET: ctx.postmark["auth"] = "none" return Decision.CONTINUE - # Record the verdict structurally instead of prepending it to the - # bytes: the ingest blob stays untouched (reused as Message.blob) and - # ``get_stmsg_headers`` surfaces it. ``verdict`` is already "none" - # (unverified) or "fail" (forged); a verified message returns no - # verdict and leaves ``auth`` absent. + # ``verdict`` is already "none" (unverified) or "fail" (forged); a + # verified message returns no verdict and leaves ``auth`` absent. ctx.postmark["auth"] = verdict return Decision.CONTINUE @@ -340,10 +342,9 @@ def build_inbound_pipeline(ctx: InboundContext) -> List[Step]: # Internal mailbox-to-mailbox mail is trusted and not externally # authenticated: run only the user-webhook steps. The spam steps # would no-op anyway (the task pre-sets is_spam=False), and the auth - # step would prepend a meaningless X-StMsg-Sender-Auth banner — which - # also mutates the bytes and defeats blob dedup with the sender — plus - # do needless DNS/rspamd work. Webhooks still fire on both phases so - # internal mail is indistinguishable from external to a consumer. + # step would record a meaningless auth verdict plus do needless + # DNS/rspamd work. Webhooks still fire on both phases so internal mail + # is indistinguishable from external to a consumer. if ctx.inbound_message.is_internal: return [ *webhook_steps_for_mailbox( @@ -449,15 +450,19 @@ def _resolve_assignable_users( for email in target_emails: bucket = by_email.get(email) or [] if not bucket: + # Don't log the raw webhook-supplied email (PII) — reference the + # thread instead. logger.warning( - "Webhook assignee email %s does not resolve to any user — skipping", - email, + "Webhook assignee email does not resolve to any user on " + "thread %s — skipping", + thread.id, ) continue if len(bucket) > 1: logger.warning( - "Webhook assignee email %s is ambiguous (multiple matches) — skipping", - email, + "Webhook assignee email is ambiguous (multiple matches) on " + "thread %s — skipping", + thread.id, ) continue user = bucket[0] @@ -474,9 +479,10 @@ def _resolve_assignable_users( ) for uid in candidate_ids: if uid not in assignable_ids: + # Reference the thread, not the user's email (PII). logger.warning( - "Webhook assignee %s lacks an assignable role on the thread — skipping", - candidate_users[uid].email, + "Webhook assignee lacks an assignable role on thread %s — skipping", + thread.id, ) return [ diff --git a/src/backend/core/mda/inbound_tasks.py b/src/backend/core/mda/inbound_tasks.py index 7e98bc87a..d3d1a808e 100644 --- a/src/backend/core/mda/inbound_tasks.py +++ b/src/backend/core/mda/inbound_tasks.py @@ -87,9 +87,11 @@ def _safe_finalize(label, inbound_message_id, gate, fn): ``gate`` short-circuits the call when the input collection is empty/false — same semantics as the inline ``if ctx.labels:`` - guards, just lifted out. Exceptions are logged but never - propagated: the message has already landed; failing the whole - task here would only confuse operators.""" + guards, just lifted out. ALL exceptions (including a Celery + ``SoftTimeLimitExceeded``) are logged and swallowed, never propagated: + these run AFTER the message has landed and its queue row is deleted, so + re-raising would make the task-level handler retry/abandon a row that no + longer exists. A dropped finalize side effect is the acceptable cost.""" if not gate: return try: @@ -139,7 +141,9 @@ def _handle_retry( def _retry_or_abandon( - inbound_message: models.InboundMessage, reason: str + inbound_message: models.InboundMessage, + reason: str, + blocking_webhook_results: Optional[Dict[Any, Any]] = None, ) -> Dict[str, Any]: """Bounded handling for a message that failed to be created/processed. @@ -152,9 +156,17 @@ def _retry_or_abandon( silently lose mail; instead an operator can inspect and replay the row from the Django admin, and ``logger.error`` raises a Sentry alert. ``error_message`` keeps the human-readable reason. + + ``blocking_webhook_results`` (when the failure happened AFTER the pipeline + already ran the blocking webhooks) is persisted on the retry path so the + next sweep replays those successes from cache instead of re-POSTing them. """ age = timezone.now() - inbound_message.created_at if age <= DEFERRAL_MAX_AGE: + if blocking_webhook_results: + persist_cached_webhook_results( + str(inbound_message.id), blocking_webhook_results + ) inbound_message.error_message = reason inbound_message.save(update_fields=["error_message"]) return { @@ -220,6 +232,9 @@ def process_inbound_message_task(self, inbound_message_id: str): return {"success": False, "error": "Message already being processed"} inbound_message: Optional[models.InboundMessage] = None + # Bound up-front so the except handlers below can safely read it even if a + # timeout/error fires before the pipeline builds it. + ctx: Optional[InboundContext] = None try: try: inbound_message = models.InboundMessage.objects.get(id=inbound_message_id) @@ -348,7 +363,7 @@ def process_inbound_message_task(self, inbound_message_id: str): is_spam=False if deferral_expired else bool(ctx.is_spam), is_trashed=ctx.mark_trashed, is_archived=ctx.mark_archived, - # Reuse the ingest blob (the bytes are never mutated now that + # Reuse the ingest blob (the bytes are never mutated — # verdicts go to postmark) and carry the pipeline's postmark. blob=inbound_message.blob, postmark=ctx.postmark, @@ -479,9 +494,12 @@ def process_inbound_message_task(self, inbound_message_id: str): } # Creation failed (transient DB error, constraint, …). Hold for a - # bounded retry rather than keeping the row forever. + # bounded retry rather than keeping the row forever — carrying the + # already-run blocking webhooks so the retry doesn't re-POST them. return _retry_or_abandon( - inbound_message, "Failed to create message from inbound message" + inbound_message, + "Failed to create message from inbound message", + blocking_webhook_results=ctx.blocking_webhook_results, ) except SoftTimeLimitExceeded: @@ -502,6 +520,7 @@ def process_inbound_message_task(self, inbound_message_id: str): inbound_message, f"Processing exceeded the {_INBOUND_TASK_SOFT_TIME_LIMIT}s " "soft time limit", + blocking_webhook_results=ctx.blocking_webhook_results if ctx else None, ) return {"success": False, "error": "soft_time_limit"} except Exception as e: @@ -511,7 +530,11 @@ def process_inbound_message_task(self, inbound_message_id: str): if inbound_message: # Same bounded-retry policy as a failed creation: a persistent # error must not pin the row (and re-fire webhooks) forever. - return _retry_or_abandon(inbound_message, str(e)) + return _retry_or_abandon( + inbound_message, + str(e), + blocking_webhook_results=ctx.blocking_webhook_results if ctx else None, + ) return {"success": False, "error": str(e)} finally: # Always release the lock @@ -589,8 +612,8 @@ def purge_abandoned_inbound_messages_task( Abandoned rows are deliberately kept (never deleted at abandon time) so the mail stays inspectable / replayable — see ``_retry_or_abandon``. But they must not accumulate forever: a sustained stream of unparseable / uncreatable - mail would otherwise grow this transient queue table (and its inline - ``raw_data`` bytes) without bound. This daily sweep deletes rows past the + mail would otherwise grow this transient queue table (and pin the blobs + it references) without bound. This daily sweep deletes rows past the retention window. Deletes in batches through ``QuerySet.delete()`` (not ``_raw_delete``) so diff --git a/src/backend/core/mda/outbound.py b/src/backend/core/mda/outbound.py index 7552df087..68209fb5a 100644 --- a/src/backend/core/mda/outbound.py +++ b/src/backend/core/mda/outbound.py @@ -19,7 +19,7 @@ ) from core import models -from core.enums import MessageDeliveryStatusChoices +from core.enums import InboundOrigin, MessageDeliveryStatusChoices from core.mda.inbound import check_local_recipient, deliver_inbound_message from core.mda.inline_images import ( extract_inline_images_html, @@ -696,7 +696,7 @@ def _mark_delivered( parsed_email, blob_content, envelope={ - "origin": "internal", + "origin": InboundOrigin.INTERNAL, "mail_from": message.sender.email, "rcpt_to": recipient_email, }, diff --git a/src/backend/core/mda/selfcheck.py b/src/backend/core/mda/selfcheck.py index 3055d9aba..55c88b29c 100644 --- a/src/backend/core/mda/selfcheck.py +++ b/src/backend/core/mda/selfcheck.py @@ -13,6 +13,7 @@ from jmap_email import body_text_joined from core import models +from core.enums import is_delivered from core.mda.draft import create_draft from core.mda.outbound import prepare_outbound_message, send_message from core.mda.selfcheck_reporting import ( @@ -91,13 +92,12 @@ def create_and_send_draft( # Check message delivery status recipient_status = message.recipients.first().delivery_status # pylint: disable=no-member - # ``!= SENT_EXTERNAL`` is safe here ONLY because - # ``send_message(force_mta_out=True)`` above forces the external MTA path, - # which yields SENT_EXTERNAL. A same-instance recipient delivered - # internally would be SENT_INTERNAL (also "delivered") and would wrongly - # fail this check — see the footgun note on MessageDeliveryStatusChoices. - # Don't drop force_mta_out without also accepting SENT_INTERNAL here. - if recipient_status != models.MessageDeliveryStatusChoices.SENT_EXTERNAL: + # Accept either "delivered"-class status via ``is_delivered``: the + # ``send_message(force_mta_out=True)`` above forces the external MTA path + # (SENT_EXTERNAL) today, but SENT_INTERNAL is equally "delivered", so the + # predicate stays correct even if force_mta_out is ever dropped — see the + # footgun note on MessageDeliveryStatusChoices. + if not is_delivered(recipient_status): raise SelfCheckError("Message not delivered") return message diff --git a/src/backend/core/mda/spam.py b/src/backend/core/mda/spam.py index c984d304b..0dc037e4e 100644 --- a/src/backend/core/mda/spam.py +++ b/src/backend/core/mda/spam.py @@ -45,6 +45,20 @@ def check_hardcoded_rules( key = key.lower().strip() value = value.lower().strip() + # ``Return-Path`` is baked into block 0 by the MDA from the *envelope* + # MAIL FROM, which is unauthenticated (a spammer sets it freely at SMTP + # time; the widget uses a raw form field) and unverified at this layer + # (no SPF/DMARC). Matching it as a trusted header would let a spoofed + # sender satisfy an ``action: ham`` allowlist and bypass the spam + # steps, so it is never eligible for a hardcoded-rule match. + if key == "return-path": + logger.warning( + "Ignoring spam rule #%d: 'return-path' is a spoofable envelope " + "value and cannot be used as a trusted header_match", + idx, + ) + continue + # Existence check first; the trusted value is read from the # Received-bounded blocks below. if not has_header(parsed_email, key): @@ -144,10 +158,13 @@ def call_rspamd( # error_message channel and let the caller decide (the inbound step # RETRYs rather than failing open). logger.exception("Error calling rspamd: %s", exc) - return None, str(exc), None + # Return a stable, sanitized token (exception class name only) — the + # full detail is logged locally above; the message can carry the + # endpoint URL / other details and is echoed downstream. + return None, type(exc).__name__, None except Exception as exc: logger.exception("Unexpected error calling rspamd: %s", exc) - return None, str(exc), None + return None, type(exc).__name__, None if not isinstance(result, dict): logger.warning("rspamd returned non-object body: %r", result) diff --git a/src/backend/core/migrations/0032_inboundmessage_blob_envelope.py b/src/backend/core/migrations/0032_inboundmessage_blob_envelope.py index 09792cee2..078475094 100644 --- a/src/backend/core/migrations/0032_inboundmessage_blob_envelope.py +++ b/src/backend/core/migrations/0032_inboundmessage_blob_envelope.py @@ -40,7 +40,11 @@ class Migration(migrations.Migration): migrations.AddField( model_name='inboundmessage', name='abandoned_at', - field=models.DateTimeField(blank=True, help_text='Set when processing was permanently abandoned after the retry window; the row is kept for inspection/replay and skipped by the retry sweep.', null=True, verbose_name='abandoned at'), + field=models.DateTimeField(blank=True, help_text='When processing was permanently abandoned; NULL while live.', null=True, verbose_name='abandoned at'), + ), + migrations.AddIndex( + model_name='inboundmessage', + index=models.Index(condition=models.Q(('abandoned_at__isnull', False)), fields=['abandoned_at'], name='messages_in_abandon_partial'), ), migrations.AddField( model_name='inboundmessage', diff --git a/src/backend/core/models.py b/src/backend/core/models.py index 4c7187667..a102303df 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -41,6 +41,7 @@ CompressionTypeChoices, CRUDAbilities, DKIMAlgorithmChoices, + InboundOrigin, MailboxAbilities, MailboxRoleChoices, MailDomainAbilities, @@ -2287,13 +2288,12 @@ class InboundMessage(BaseModel): related_name="inbound_messages", ) # Sole byte source: a content-addressed, encrypted, deduped Blob created - # at ingest. (External mail used to be stored inline as a plaintext - # ``raw_data`` BinaryField; now every path is blob-backed, so a message to - # N recipients shares ONE blob from the moment it's queued, and nothing - # sits in plaintext.) PROTECT mirrors ``Message.blob``: only the GC sweep - # deletes, and it clears references first; ``is_referenced`` counts this - # FK. Nullable at the DB level for migration safety, but always set by the - # ingest paths (``deliver_inbound_message``). + # at ingest, so a message to N recipients shares ONE blob from the moment + # it's queued and nothing sits in plaintext. PROTECT mirrors + # ``Message.blob``: only the GC sweep deletes, and it clears references + # first; ``is_referenced`` counts this FK. Nullable at the DB level for + # migration safety, but always set by the ingest paths + # (``deliver_inbound_message``). blob = models.ForeignKey( "Blob", on_delete=models.PROTECT, @@ -2343,11 +2343,7 @@ class InboundMessage(BaseModel): "abandoned at", null=True, blank=True, - help_text=( - "Set when processing was permanently abandoned after the retry " - "window; the row is kept for inspection/replay and skipped by the " - "retry sweep." - ), + help_text="When processing was permanently abandoned; NULL while live.", ) class Meta: @@ -2357,6 +2353,14 @@ class Meta: ordering = ["-created_at"] indexes = [ models.Index(fields=["created_at"]), + # Partial: only the rare abandoned rows are indexed (the column is + # NULL for every live/in-flight message), so live INSERTs skip it + # and the purge/retry sweeps still get a tiny index to scan. + models.Index( + fields=["abandoned_at"], + condition=models.Q(abandoned_at__isnull=False), + name="messages_in_abandon_partial", + ), ] def __str__(self): @@ -2372,7 +2376,7 @@ def is_internal(self) -> bool: fields, which would fail open to "trusted" for a stripped external message. """ - return (self.envelope or {}).get("origin") == "internal" + return (self.envelope or {}).get("origin") == InboundOrigin.INTERNAL def get_raw_bytes(self) -> bytes: """Return the raw message bytes from the backing blob. diff --git a/src/backend/core/services/ssrf.py b/src/backend/core/services/ssrf.py index a7ab3de91..a48052e05 100644 --- a/src/backend/core/services/ssrf.py +++ b/src/backend/core/services/ssrf.py @@ -250,6 +250,15 @@ def _request_with_redirects( re-issue the same verb rather than downgrading a 30x to GET), so a POST body reaches the final, validated destination intact. + ``kwargs`` (including any ``Authorization`` header and the POST body) + are re-sent verbatim on every hop, so a cross-host redirect forwards + the caller's credentials + payload to the redirect target. This is + intentional for webhook delivery — a receiver that 3xx-redirects (LB / + canonicaliser) must still get the signed body and its auth — and safe + because every hop is SSRF-validated to a public host and an HTTPS→HTTP + downgrade is refused. Callers that must not leak credentials across + hosts should not send a bearer credential through this session. + ``method`` is the lowercase session method name (``"get"`` / ``"post"``) — we call that bound method directly rather than ``Session.request`` so each verb keeps a distinct, individually @@ -276,6 +285,17 @@ def _request_with_redirects( return response next_url = urljoin(current_url, location) + # Refuse an HTTPS→HTTP downgrade: a redirect must not silently drop + # the connection from TLS to cleartext. Same-scheme or an HTTP→HTTPS + # upgrade is fine. + if ( + urlparse(current_url).scheme == "https" + and urlparse(next_url).scheme == "http" + ): + response.close() + raise SSRFValidationError( + "Refusing to follow HTTPS→HTTP redirect downgrade" + ) response.close() current_url = next_url diff --git a/src/backend/core/signals.py b/src/backend/core/signals.py index 7f852aaa3..8e8b00697 100644 --- a/src/backend/core/signals.py +++ b/src/backend/core/signals.py @@ -182,11 +182,10 @@ def delete_message_from_index(sender, instance, **kwargs): Pushes ``(thread_id, message_id)`` to the dedicated delete-message set so ``bulk_delete_messages_task`` can later issue a ``bulk delete by - _id`` (with the parent ``thread_id`` as routing). Replaces the - previous behaviour of scheduling a thread reindex and relying on a - ``delete_by_query`` orphan purge — far cheaper for the cluster, and - correct because Django fires ``post_delete`` for both direct and - cascaded message deletions. + _id`` (with the parent ``thread_id`` as routing). Cheaper for the + cluster than reindexing the thread and letting a ``delete_by_query`` + orphan purge catch up, and correct because Django fires ``post_delete`` + for both direct and cascaded message deletions. """ if not settings.OPENSEARCH_INDEX_THREADS: return @@ -204,9 +203,9 @@ def delete_thread_from_index(sender, instance, **kwargs): Only the parent doc is queued here — child message docs are picked up by ``delete_message_from_index`` via the cascaded ``post_delete`` - signal Django fires for each child. Splitting the two paths replaces - the previous all-in-one ``delete_by_query`` on ``thread_id`` with two - cheap ``bulk delete by _id`` requests. + signal Django fires for each child. Splitting parent and children into + two cheap ``bulk delete by _id`` requests avoids an all-in-one + ``delete_by_query`` on ``thread_id``. """ if not settings.OPENSEARCH_INDEX_THREADS: return diff --git a/src/backend/core/templates/admin/core/channel/change_form.html b/src/backend/core/templates/admin/core/channel/change_form.html index 0321eb4ef..df3682896 100644 --- a/src/backend/core/templates/admin/core/channel/change_form.html +++ b/src/backend/core/templates/admin/core/channel/change_form.html @@ -8,11 +8,12 @@ {% block object-tools-items %} {% if original and original.type == "api_key" or original and original.type == "webhook" %} +{% blocktranslate asvar regenerate_confirm %}Regenerate the secret for this channel? The previous secret will stop working immediately.{% endblocktranslate %}
  • + onsubmit="return confirm('{{ regenerate_confirm|escapejs }}');"> {% csrf_token %}