diff --git a/Makefile b/Makefile index 6f49a0911..e70473d9c 100644 --- a/Makefile +++ b/Makefile @@ -423,6 +423,10 @@ lint-jmap-email: build-python-base ## lint the jmap-email library (ruff check + @$(COMPOSE) run --build --rm --entrypoint pylint jmap-email-test jmap_email tests .PHONY: lint-jmap-email +format-jmap-email: build-python-base ## lint the jmap-email library (ruff check + format check + pylint) + @$(COMPOSE) run --build --rm --entrypoint ruff jmap-email-test format jmap_email tests +.PHONY: format-jmap-email + typecheck-jmap-email: build-python-base ## type-check the jmap-email library with ty (Astral, Rust) @$(COMPOSE) run --build --rm --entrypoint ty jmap-email-test check .PHONY: typecheck-jmap-email diff --git a/bin/release-jmap-email.sh b/bin/release-jmap-email.sh index 157f13af4..a47d17364 100755 --- a/bin/release-jmap-email.sh +++ b/bin/release-jmap-email.sh @@ -141,8 +141,9 @@ expected_wheel = { "jmap_email/__init__.py", "jmap_email/composer.py", "jmap_email/helpers.py", - "jmap_email/limits.py", + "jmap_email/options.py", "jmap_email/parser.py", + "jmap_email/preview.py", "jmap_email/types.py", "jmap_email/py.typed", f"jmap_email-{VERSION}.dist-info/METADATA", diff --git a/src/backend/core/mda/autoreply.py b/src/backend/core/mda/autoreply.py index d3a9d40ba..ded66a921 100644 --- a/src/backend/core/mda/autoreply.py +++ b/src/backend/core/mda/autoreply.py @@ -8,13 +8,8 @@ from django.db import transaction from django.utils import timezone -from jmap_email import ( - JmapEmail, - find_header, - find_headers, - first_address_email, - has_header, -) +from jmap_email import find_header, find_headers, first_address_email, has_header +from jmap_email.types import JmapEmail from core import models from core.enums import ( diff --git a/src/backend/core/mda/dispatch_webhooks.py b/src/backend/core/mda/dispatch_webhooks.py index c17ed5f86..1814e9120 100644 --- a/src/backend/core/mda/dispatch_webhooks.py +++ b/src/backend/core/mda/dispatch_webhooks.py @@ -49,7 +49,8 @@ from django.db.models import Q import jwt -from jmap_email import JmapEmail, parse_email +from jmap_email import parse_email +from jmap_email.types import JmapEmail from core import enums, models from core.mda.inbound_pipeline import ( diff --git a/src/backend/core/mda/inbound.py b/src/backend/core/mda/inbound.py index 95a473d44..64c7ee9a9 100644 --- a/src/backend/core/mda/inbound.py +++ b/src/backend/core/mda/inbound.py @@ -8,7 +8,8 @@ from django.core.exceptions import ValidationError from django.db.utils import Error as DjangoDbError -from jmap_email import JmapEmail, first_msgid +from jmap_email import first_msgid +from jmap_email.types import JmapEmail from core import enums, models from core.mda.inbound_tasks import process_inbound_message_task diff --git a/src/backend/core/mda/inbound_auth.py b/src/backend/core/mda/inbound_auth.py index 904dca3df..b5c67a6c7 100644 --- a/src/backend/core/mda/inbound_auth.py +++ b/src/backend/core/mda/inbound_auth.py @@ -39,7 +39,8 @@ import re from typing import Any -from jmap_email import JmapEmail, first_address_email +from jmap_email import first_address_email +from jmap_email.types import JmapEmail from core.mda.signing import verify_message_dkim from core.mda.utils import headers_blocks diff --git a/src/backend/core/mda/inbound_create.py b/src/backend/core/mda/inbound_create.py index 07aa02d6a..2295ae25f 100644 --- a/src/backend/core/mda/inbound_create.py +++ b/src/backend/core/mda/inbound_create.py @@ -14,12 +14,12 @@ from django.utils import timezone from jmap_email import ( - JmapEmail, first_address_email, first_address_name, first_msgid, sent_at_to_datetime, ) +from jmap_email.types import JmapEmail from core import enums, models from core.ai.call_label import assign_label_to_thread diff --git a/src/backend/core/mda/inbound_pipeline.py b/src/backend/core/mda/inbound_pipeline.py index 1b20f5ebc..6979fd309 100644 --- a/src/backend/core/mda/inbound_pipeline.py +++ b/src/backend/core/mda/inbound_pipeline.py @@ -33,7 +33,7 @@ from django.db.models.functions import Lower from django.utils import timezone -from jmap_email import JmapEmail +from jmap_email.types import JmapEmail from core import enums, models from core.mda import spam diff --git a/src/backend/core/mda/inbound_tasks.py b/src/backend/core/mda/inbound_tasks.py index 2cc2734ed..d6d71b9be 100644 --- a/src/backend/core/mda/inbound_tasks.py +++ b/src/backend/core/mda/inbound_tasks.py @@ -17,7 +17,8 @@ from celery.exceptions import SoftTimeLimitExceeded from celery.utils.log import get_task_logger -from jmap_email import JmapEmail, first_address_email, parse_email +from jmap_email import first_address_email, parse_email +from jmap_email.types import JmapEmail from core import models from core.mda.dispatch_webhooks import ( diff --git a/src/backend/core/mda/replies.py b/src/backend/core/mda/replies.py index ef3eced44..0ef99c53c 100644 --- a/src/backend/core/mda/replies.py +++ b/src/backend/core/mda/replies.py @@ -27,13 +27,13 @@ from typing import Any from jmap_email import ( - JmapEmail, first_msgid, format_address, format_address_list, is_valid_msg_id, msgid_chain, ) +from jmap_email.types import JmapEmail # Bracket-aware tokeniser for a wire-form ``References`` chain. Splitting # on whitespace would slice ```` into ```` and diff --git a/src/backend/core/mda/spam.py b/src/backend/core/mda/spam.py index 7be940365..9f6b9c146 100644 --- a/src/backend/core/mda/spam.py +++ b/src/backend/core/mda/spam.py @@ -15,7 +15,8 @@ from typing import Any, Dict, Optional, Tuple import requests -from jmap_email import JmapEmail, has_header +from jmap_email import has_header +from jmap_email.types import JmapEmail from core.mda.utils import headers_blocks diff --git a/src/backend/core/mda/utils.py b/src/backend/core/mda/utils.py index 833eea724..a2ebba434 100644 --- a/src/backend/core/mda/utils.py +++ b/src/backend/core/mda/utils.py @@ -22,7 +22,8 @@ from django.utils import timezone -from jmap_email import JmapEmail, body_part_text, decode_rfc2047_header +from jmap_email import body_part_text, decode_rfc2047_header +from jmap_email.types import JmapEmail __all__ = [ "SNIPPET_MAX_LENGTH", diff --git a/src/backend/core/mda/webhook_payload.py b/src/backend/core/mda/webhook_payload.py index 0c1cf79cb..8cc3fbd00 100644 --- a/src/backend/core/mda/webhook_payload.py +++ b/src/backend/core/mda/webhook_payload.py @@ -11,7 +11,7 @@ from datetime import datetime, timezone from typing import Any, Dict, Optional -from jmap_email import JmapEmail +from jmap_email.types import JmapEmail def _utcdate(value: Any) -> Optional[str]: diff --git a/src/backend/core/models.py b/src/backend/core/models.py index cb796c7c3..eeb704e3b 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -30,7 +30,8 @@ import pyzstd from encrypted_fields.fields import EncryptedJSONField, EncryptedTextField -from jmap_email import EmailHeader, JmapEmail, body_part_text, parse_email +from jmap_email import body_part_text, parse_email +from jmap_email.types import EmailHeader, JmapEmail from timezone_field import TimeZoneField from core.enums import ( diff --git a/src/backend/core/services/exporter/tasks.py b/src/backend/core/services/exporter/tasks.py index 6ec071f48..a5efa091c 100644 --- a/src/backend/core/services/exporter/tasks.py +++ b/src/backend/core/services/exporter/tasks.py @@ -12,7 +12,8 @@ from django.db.models import OuterRef, Subquery from celery.utils.log import get_task_logger -from jmap_email import JmapEmail, compose_email, parse_email +from jmap_email import compose_email, parse_email +from jmap_email.types import JmapEmail from sentry_sdk import capture_exception from core.api.utils import generate_presigned_url diff --git a/src/backend/core/services/importer/labels.py b/src/backend/core/services/importer/labels.py index 2ef65b3ee..31346a247 100644 --- a/src/backend/core/services/importer/labels.py +++ b/src/backend/core/services/importer/labels.py @@ -2,7 +2,7 @@ import logging -from jmap_email import JmapEmail +from jmap_email.types import JmapEmail from core import models from core.mda.utils import gmail_labels diff --git a/src/backend/core/services/importer/mbox.py b/src/backend/core/services/importer/mbox.py index f8914de8e..a58d0a75c 100644 --- a/src/backend/core/services/importer/mbox.py +++ b/src/backend/core/services/importer/mbox.py @@ -13,7 +13,7 @@ from datetime import timezone as dt_timezone from celery.utils.log import get_task_logger -from jmap_email.parser import parse_date +from jmap_email import parse_date from core.services.s3_seekable import BUFFER_CENTERED, S3SeekableReader diff --git a/src/backend/core/services/importer/pst.py b/src/backend/core/services/importer/pst.py index 011688a6c..dba8d7e1b 100644 --- a/src/backend/core/services/importer/pst.py +++ b/src/backend/core/services/importer/pst.py @@ -20,13 +20,8 @@ from django.conf import settings import pypff -from jmap_email import ( - EmailAddress, - compose_email, - is_valid_msg_id, - parse_address, - parse_addresses, -) +from jmap_email import compose_email, is_valid_msg_id, parse_address, parse_addresses +from jmap_email.types import EmailAddress from core.services.s3_seekable import BUFFER_NONE, S3SeekableReader diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index c8e5bdee4..68eb9575c 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -144,6 +144,7 @@ select = [ "S", # flake8-bandit "SLF", # flake8-self "T20", # flake8-print + "TID251", # flake8-tidy-imports: banned-api only (not TID252 relative-imports) "F" # pyflakes ] @@ -153,6 +154,16 @@ sections = { messages=["core"], django=["django"] } known-third-party = ["dns"] extra-standard-library = ["tomllib"] +[tool.ruff.lint.flake8-tidy-imports.banned-api] +# jmap-email's public API is the top-level `jmap_email` package; the +# TypedDict shapes live in `jmap_email.types`. Everything else is private +# implementation — reaching into it couples us to internal layout. +"jmap_email.parser".msg = "Import from the top-level `jmap_email` package (shapes: `jmap_email.types`)." +"jmap_email.helpers".msg = "Import from the top-level `jmap_email` package (shapes: `jmap_email.types`)." +"jmap_email.composer".msg = "Import from the top-level `jmap_email` package (shapes: `jmap_email.types`)." +"jmap_email.options".msg = "Import from the top-level `jmap_email` package (shapes: `jmap_email.types`)." +"jmap_email.preview".msg = "Import from the top-level `jmap_email` package (shapes: `jmap_email.types`)." + [tool.ruff.lint.per-file-ignores] "**/tests/*" = ["S", "SLF"] diff --git a/src/jmap-email/CHANGELOG.md b/src/jmap-email/CHANGELOG.md index c62542620..8cdc84824 100644 --- a/src/jmap-email/CHANGELOG.md +++ b/src/jmap-email/CHANGELOG.md @@ -5,9 +5,67 @@ All notable changes to `jmap-email` are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +## [0.2.0] - 2026-07-22 + +### Added + +- `ParseOptions.max_preview_chars` (default 256, the RFC 8621 §4.1.4 + ceiling) caps the length of the server-set `preview` excerpt. Lower it + via `options=` to produce a shorter preview (e.g. a list-view snippet). +- `ParseOptions.max_preview_scan_bytes` (default 128 KiB) bounds the body + text scanned for the `preview`, so a body that is almost entirely markup + (which never reaches the text budget) can't run the extractor away. +- `preview_text` helper (also used internally for the `preview` field): + turns HTML/markdown body text into a single, display-ready line. It strips + HTML (dropping `

Real text

" + assert preview_text(html) == "Real text" + + def test_drops_head_title_metadata(self): + # is <head> metadata (browser-tab text), not visible body — + # some senders even put an image URL there. It must not leak. + html = ( + "<head><title>http://img.example/logo.jpg" + "Bonjour, voici le message." + ) + assert preview_text(html) == "Bonjour, voici le message." + + def test_drops_unclosed_script_payload(self): + """A truncated/unclosed ``after") == "after" + + def test_handles_gt_inside_comments_and_attributes(self): + """``>`` inside a comment body or a quoted attribute value does + not terminate the markup early.""" + assert preview_text("a b") == "a b" + assert preview_text('a>bcaption') == "caption" + + def test_preserves_plain_text_comparisons(self): + """``<``/``>`` used as comparison signs in plain text are not + tags — ``<`` before a non-letter stays text.""" + assert preview_text("x < 5 and y > 3") == "x < 5 and y > 3" + assert preview_text("Prix < 100 euros, remise > 10%") == ( + "Prix < 100 euros, remise > 10%" + ) + + def test_lt_glued_to_letter_reads_as_markup(self): + """``<`` immediately followed by a letter opens a tag — same + reading as browsers. Pinned as the accepted trade-off of the + tolerant-parser approach: ``x`` and ```` are markdown autolinks, + not tags — the URL / address survives the tag strip.""" + assert preview_text("see ") == ( + "see https://example.com/a" + ) + assert preview_text("Courriel: ") == ( + "Courriel: contact@brigny.fr" + ) + + def test_autolink_with_trailing_slash(self): + """A URL ending in ``/`` tokenises as a self-closing tag + (``handle_startendtag``), not a start tag — it must still unwrap.""" + assert preview_text("end here") == ( + "end https://example.com/ here" + ) + + def test_autolink_decodes_entities(self): + """The unwrapped URL is HTML-unescaped, matching the entity decoding + the parser applies to ordinary text (`&` -> `&`).""" + assert preview_text("q x") == ( + "q https://e.com/p?a=1&b=2 x" + ) + + def test_non_letter_email_autolink_is_unwrapped_post_strip(self): + """An email autolink whose local-part starts with a non-letter + (`<123@host.com>`, `<+promo@x.com>`) is read as literal text by the + parser (``<`` before a non-letter is text), so the in-parser tag path + never sees it. The bounded post-strip pass unwraps it instead.""" + assert preview_text("x <123@host.com> y") == "x 123@host.com y" + assert preview_text("x <+promo@shop.com> y") == "x +promo@shop.com y" + + def test_lt_not_autolink_is_untouched(self): + """A bare ``<`` that isn't an autolink stays put — the post-strip + unwrap only fires on URL/email shapes.""" + assert preview_text("i <3 you") == "i <3 you" + assert preview_text("x < 5 and y > 3") == "x < 5 and y > 3" + + def test_strips_hard_break_backslashes(self): + """Trailing-backslash hard breaks, with LF or CRLF endings.""" + assert preview_text("tableau.\\\n\\\nCordialement") == "tableau. Cordialement" + assert preview_text("tableau.\\\r\n\\\r\nCordialement") == ( + "tableau. Cordialement" + ) + + def test_passes_through_multibyte_unicode(self): + """Emoji, CJK, RTL and combining sequences are content, not markup — + they survive unchanged, including alongside HTML and entities.""" + assert preview_text("Hello 👋🏽 world 😀") == "Hello 👋🏽 world 😀" + assert preview_text("你好，世界！これはテスト") == "你好，世界！これはテスト" + assert preview_text("مرحبا بالعالم") == "مرحبا بالعالم" + # decomposed (e + combining acute) is kept code-point-for-code-point + assert preview_text("cafe\u0301") == "cafe\u0301" + assert preview_text("

Prix: 5€ 😀 & plus

") == "Prix: 5€ 😀 & plus" + + def test_collapses_unicode_whitespace(self): + """``\\s`` matches Unicode whitespace, so NBSP / line-separator / + ideographic space collapse like ASCII runs do.""" + assert preview_text("a   b") == "a b" + assert preview_text("a
b　c") == "a b c" + + def test_truncation_counts_code_points(self): + """Truncation is by Unicode code point (RFC 8621 §4.1.4 counts + characters). Single-code-point astral chars are never split.""" + out = preview_text("😀" * 300) + assert len(out) == 256 + assert out == "😀" * 256 + + def test_truncation_may_split_a_grapheme_cluster(self): + """Accepted edge: a multi-code-point grapheme (ZWJ emoji, combining + sequence) straddling the max_chars boundary is cut mid-cluster — + code-point truncation, not grapheme-aware. Pinned as deliberate; the + invariant that matters is never exceeding the budget.""" + family = "👨‍👩‍👧‍👦" # 7 code points, 1 grapheme + out = preview_text(("x" * 253) + family) + assert len(out) == 256 # never exceeds the budget + assert out == ("x" * 253) + family[:3] # cut mid-cluster + + def test_cleans_before_truncating(self): + """Heavy leading syntax must not consume the preview budget: + a ~200-char HTML figure prefix still yields the full caption + and following prose.""" + figure = ( + '
Jean-Baptiste-Camille_Corot.jpg' + "
Tableau de Jean-Baptiste Corot - Fontainebleau" + "
" + ) + content = f"{figure}\n\n# Voici un message\n\nJe te présente ce tableau." + assert preview_text(content) == ( + "Tableau de Jean-Baptiste Corot - Fontainebleau " + "Voici un message Je te présente ce tableau." + ) + + def test_truncates_to_max_length(self): + assert len(preview_text("x" * 600)) == 256 + assert preview_text("y" * 40, max_chars=10) == "y" * 10 + + def test_syntax_heavy_bodies_still_fill_the_budget(self): + """The extractor budgets what it *collects*, but the stages after it + only shrink the text, so a body whose text carries indentation or + markdown used to yield a preview well short of ``max_chars`` — 79% on + a pretty-printed newsletter, 50% on markdown. ``_COLLECT_SLACK`` is + what closes that gap; each of these bodies has far more than 256 + characters of prose to offer, so a short result means the collection + budget ran out before the cleaning did. + + ``>= 255`` rather than ``== 256``: truncation can land on a space, + which the trailing ``rstrip`` then removes. One character of slop, not + the 50-plus this test exists to catch.""" + newsletter = "\n" + "\n".join( + f" " for i in range(200) + ) + assert len(preview_text(newsletter, content_type="text/html")) >= 255 + assert len(preview_text("**mot** " * 300)) >= 255 + assert len(preview_text("### Titre\n- item de liste\n" * 200)) >= 255 + # A reply that quotes every other line: the fresh half alone is far + # longer than the budget, so dropped quotes must not shorten it. + reply = "\n".join( + f"ligne fraiche numero {i}" if i % 2 else f"> ligne citee {i}" + for i in range(200) + ) + assert len(preview_text(reply)) >= 255 + + def test_budget_fill_is_bounded_by_available_text_not_by_slack(self): + """The slack bounds shrinkage, not availability: a reply with only a + short fresh part previews at that length under any slack, because + there is no more unquoted text to find. Not a shortfall.""" + reply = "Ma reponse fraiche.\n" + "\n".join( + f"> ancienne ligne citee numero {i}" for i in range(80) + ) + assert preview_text(reply) == "Ma reponse fraiche." + + def test_no_trailing_whitespace_after_truncation(self): + out = preview_text("word " * 100) + assert out == out.rstrip() + + def test_large_body_previews_from_the_head(self): + """Self-interrupt path: a body far larger than the budget (its tail + is never parsed) gives the same preview as a small body with the + same leading content (cleaned in full).""" + unit = "Bonjour, ceci est le debut du message. \n" + small = unit * 10 # ~400 B: below the budget, cleaned in full + big = unit * 20000 # ~800 KB: the parser stops after the head + assert preview_text(big) == preview_text(small) + assert preview_text(big).startswith("Bonjour, ceci est le debut") + + def test_large_syntax_heavy_body_still_finds_the_text(self): + """Heavy leading markup collected before the budget still cleans + away, leaving the real text at the top.""" + # Mostly markdown/HTML noise, a little real text near the top. + noise = ("> \n# \n- \n
\n**\n") * 5000 + body = "Texte utile en tete du message.\n" + noise + assert preview_text(body) == "Texte utile en tete du message." + + def test_empty_input(self): + assert preview_text("") == "" + + +class TestPreviewHardening: + """Security / robustness of the preview: bounded work, no control chars, + no escape sequences, single line, quoted-reply skipping.""" + + # ── scan cap (DoS bound) ─────────────────────────────────────────── + def test_scan_cap_excludes_text_beyond_the_cap(self): + # Text past ``max_scan_bytes`` is never scanned, so it can't appear. + assert preview_text("" * 3000 + "HIDDEN", max_scan_bytes=1000) == "" + assert "visible" in preview_text( + "visible " + "" * 50, max_scan_bytes=1000 + ) + + def test_markup_bomb_is_bounded(self): + # A body that is almost entirely markup never reaches the text budget; + # the scan cap must still bound the work (regression for the DoS). + import time + + start = time.perf_counter() + preview_text("" * 500_000, max_scan_bytes=64 * 1024) + assert (time.perf_counter() - start) < 1.0 # was seconds before the cap + + # ── control chars / escape sequences / single line ───────────────── + def test_strips_c0_del_c1_control_chars(self): + # Non-whitespace controls (NUL, BEL, BS, ESC, DEL, C1) are deleted. + assert preview_text("a\x00b\x07c\x08d\x1be\x7ff\x9fg") == "abcdefg" + # Separator controls (US, 0x1F) are whitespace -> collapse to a space. + assert preview_text("abc\x1fdef") == "abc def" + + def test_strips_ansi_csi_and_osc_escape_sequences(self): + assert preview_text("a\x1b[31mRED\x1b[0mb") == "aREDb" + assert preview_text("x\x1b]0;window-title\x07 y") == "x y" + + def test_strips_zero_width_preheader_spacers(self): + # Senders pad the inbox-preview line with invisible format chars: soft + # hyphen (00AD), ZWSP (200B), ZWNJ (200C), word joiner (2060), BOM + # (FEFF). They render as nothing and must be removed. The ZERO WIDTH + # JOINER (U+200D) is kept — emoji sequences rely on it. + assert preview_text("Preheader\u200c \u00ad \u200b done") == "Preheader done" + assert preview_text("\ufeffHello\u200bworld") == "Helloworld" + assert preview_text("a\u2060b") == "ab" + fam = "\U0001f468\u200d\U0001f469" # man-ZWJ-woman + assert preview_text(f"x {fam} y") == f"x {fam} y" + + def test_control_chars_from_autolink_entities_are_stripped(self): + # ```` (BEL) decodes then is deleted; ``�`` is HTML-remapped to + # U+FFFD (printable) by the parser. Never a raw control char. + out = preview_text(" z") + assert "\x07" not in out and "\x00" not in out + + def test_preview_is_always_single_line(self): + assert "\n" not in preview_text("a\nb\r\nc
d…e") + assert preview_text("a\nb\r\nc") == "a b c" + + # ── E1 / E3 ──────────────────────────────────────────────────────── + def test_uppercase_scheme_autolink_is_kept(self): + assert preview_text("see x") == ( + "see HTTPS://Example.com/A x" + ) + assert preview_text("") == "MAILTO:a@b.c" + + def test_nonpositive_max_chars_yields_empty(self): + assert preview_text("hello", max_chars=0) == "" + assert preview_text("hello", max_chars=-5) == "" + + # ── setext headers ───────────────────────────────────────────────── + def test_strips_setext_headers(self): + assert preview_text("Title\n====\nbody text") == "Title body text" + assert preview_text("Title\n----\nbody text") == "Title body text" + + # ── quoted-reply skipping ────────────────────────────────────────── + def test_drops_plaintext_quoted_lines(self): + assert preview_text("My fresh reply\n> old thread\n> more old") == ( + "My fresh reply" + ) + + def test_attribution_lines_are_not_cut(self): + # Locale/client attribution phrases ("On … wrote:", "Original + # Message") are deliberately NOT matched — only structural signals + # (`>` lines, `
`) are language-neutral enough for the lib. + # (Content after an un-quoted attribution line survives.) + assert preview_text("Short reply On Mon, Bob wrote: quoted stuff") == ( + "Short reply On Mon, Bob wrote: quoted stuff" + ) + + def test_suppresses_html_blockquote_content(self): + assert ( + preview_text("

fresh reply

quoted history
") + == "fresh reply" + ) + + def test_autolink_inside_blockquote_does_not_leak(self): + # The suppression guard in _tag_or_autolink must stop an autolink + # inside quoted (blockquote) HTML from leaking into the preview. + assert ( + preview_text( + "

hi

see
" + ) + == "hi" + ) + + def test_gt_entity_starting_an_html_line_is_not_a_quote_marker(self): + # The `>`-quoted-line drop is a *text/plain* signal: in HTML, quoted + # history is carried by
(suppressed above), never by a + # leading `>`. `convert_charrefs` decodes `>` to a literal `>`, so + # running the text/plain filter over HTML would mistake visible prose + # opening with a chevron — "> 100 EUR", "> 50% off", a decorative + # bullet — for quoted history and delete the whole line. Here that + # line is the entire body, so the preview would come back empty. + assert ( + preview_text( + "
> 100 EUR de remise sur votre commande
", + content_type="text/html", + ) + == "> 100 EUR de remise sur votre commande" + ) + + def test_gt_entity_does_not_silently_drop_a_pretty_printed_html_line(self): + # Same cause, quieter symptom — and the realistic trigger. What makes + # a chevron look like a quote is the newline in front of it, so it + # bites indented HTML: the source newlines survive the strip (they + # live inside a text node), and every mail-template engine emits + # pretty-printed HTML. The preview stays non-empty, so nothing would + # look broken — the chevron line would just be gone from the middle. + assert ( + preview_text( + "
\n" + "

Offre du mois

\n" + " > 100 EUR de remise\n" + "

Voir la boutique

\n" + "
", + content_type="text/html", + ) + == "Offre du mois > 100 EUR de remise Voir la boutique" + ) + + def test_html_keeps_markdown_characters_literal(self): + # The other half of the same rule: `*`, `_`, backticks and `[…](…)` + # are html2text syntax in a text/plain body but ordinary characters a + # sender typed in an HTML one. Stripping them there corrupts real + # content — "2*3=6" must not become "23=6". + assert ( + preview_text("

Tarif: 2*3=6, C* language

", content_type="text/html") + == "Tarif: 2*3=6, C* language" + ) + + def test_content_type_parameters_are_ignored(self): + # Real parts carry MIME parameters; only the leading type decides. + assert ( + preview_text("

2*3=6

", content_type="text/html; charset=utf-8") + == "2*3=6" + ) + assert preview_text("

2*3=6

", content_type=" TEXT/HTML ") == "2*3=6" + # Separator-less header: malformed, but common enough in real mail + # that it must not silently fall back to the text/plain stages. + assert ( + preview_text("

2*3=6

", content_type="text/html charset=utf-8") + == "2*3=6" + ) + + def test_html_in_a_parameter_value_does_not_trigger_the_html_path(self): + # The gate disables cleaning, so it stays anchored at the start: a + # part that merely mentions text/html in a parameter is not HTML. + assert ( + preview_text( + "Tarif: 2*3=6", + content_type='application/octet-stream; name="text/html"', + ) + == "Tarif: 23=6" + ) + + def test_undeclared_html_still_gets_the_thorough_cleaning(self): + # The default is text/plain on purpose: an unrecognised or absent type + # (importer blob, text/markdown, hand-built part) must not silently + # opt out of markdown and quote cleaning just because it contains + # angle brackets. Callers opt into the literal reading explicitly. + assert preview_text("

Tarif: 2*3=6

") == "Tarif: 23=6" + assert preview_text("

2*3=6

", content_type="text/markdown") == "23=6" + + def test_plaintext_quote_and_markdown_stripping_are_unchanged(self): + # The text/plain path must not regress: quoted history is still + # dropped whole (not just its marker), markdown is still stripped. + assert ( + preview_text("My fresh reply\n> old thread", content_type="text/plain") + == "My fresh reply" + ) + assert preview_text("**bold** text", content_type="text/plain") == "bold text" + + # ── word boundaries (pending-space coalescing) ───────────────────── + def test_tags_preserve_word_boundary_without_extra_spaces(self): + assert preview_text("alpha
beta") == "alpha beta" + assert preview_text("

one

two

") == "one two" + assert preview_text("
solo
") == "solo" + + +class TestPreviewParserQuirks: + """Pins for the stdlib ``html.parser`` behaviours the preview relies on — + documented so a Python upgrade that changes them fails loudly.""" + + def test_output_is_plaintext_not_html_safe(self): + # The preview is a text string and may carry <, >, & verbatim; it is + # NOT HTML-escaped (consumers must escape before HTML rendering). + assert preview_text("x < 5 & y > 3") == "x < 5 & y > 3" + assert preview_text("Tom & Jerry <3") == "Tom & Jerry <3" + + def test_processing_instructions_are_dropped(self): + assert preview_text("ab") == "a b" + assert preview_text("ab") == "a b" + + def test_declarations_and_doctype_are_dropped(self): + assert preview_text("

hi

") == "hi" + assert preview_text("ab") == "a b" + + def test_deep_nesting_does_not_recurse_or_crash(self): + # HTMLParser is iterative — deep nesting must not blow the stack. + assert preview_text("
" * 20_000 + "deep" + "
" * 20_000) == "deep" + + def test_numeric_char_ref_control_handling(self): + # � is HTML5-remapped to U+FFFD (printable);  (BEL) decodes then + # is stripped as a control char. Never a raw NUL/BEL in the output. + out = preview_text("x�yz") + assert "\x00" not in out and "\x07" not in out + assert out == "x�yz" + + def test_unclosed_style_and_blockquote_suppress_to_eof(self): + assert preview_text("keep", + "", + "", + 'a>b', + "", + "", + "
Ligne {i} du bulletin mensuel