From e771cd5eb036f7896fa11d9d2b2774669d1f3758 Mon Sep 17 00:00:00 2001 From: Ondra Zahradnik Date: Tue, 16 Jun 2026 10:56:57 +0200 Subject: [PATCH 1/2] Cap outliers by percentile winsorization --- docs/stats/outliers.md | 74 ++++++++++ mkdocs.yml | 1 + src/epstats/server/req.py | 63 ++++++-- src/epstats/toolkit/metric.py | 57 +++++++- src/epstats/toolkit/parser.py | 34 ++++- .../toolkit/test_outlier_winsorization.py | 136 ++++++++++++++++++ 6 files changed, 352 insertions(+), 13 deletions(-) create mode 100644 docs/stats/outliers.md create mode 100644 tests/epstats/toolkit/test_outlier_winsorization.py diff --git a/docs/stats/outliers.md b/docs/stats/outliers.md new file mode 100644 index 0000000..52ba69d --- /dev/null +++ b/docs/stats/outliers.md @@ -0,0 +1,74 @@ +# Outlier Winsorization + +Some metrics are heavy-tailed: revenue, time spent, number of actions per user, etc. A handful of +extreme units (e.g. a single user who spends 1000× the typical amount) can dominate the per-variant +mean and, even more so, the variance. When such outliers are concentrated in one variant, the +[by-unit evaluation][epstats.toolkit.experiment.Experiment.evaluate_by_unit] of that metric becomes +unreliable — the test loses power and the estimated effect is distorted. + +Ep-Stats can make the evaluation robust to these outliers by **winsorizing** the per-unit values +before the metric is aggregated. + +## What winsorization does + +Winsorization **caps** (clips) extreme per-unit values to a percentile threshold instead of letting +them through unchanged. For example, capping the upper `1%` replaces every per-unit value above the +99th percentile with the 99th-percentile value. + +Two design choices make this safe for experiment evaluation: + +1. **The threshold is computed once from the pooled data across all variants and applied + identically to every variant.** Because the very same absolute cap is used in every arm, + winsorization is a single fixed transformation of the metric — it cannot, on its own, introduce a + difference between variants. Computing the threshold *separately per variant* would cut the tail + of each arm at a different absolute value and could bias (or even create) the measured effect, so + Ep-Stats deliberately does **not** do that. + +2. **Units are capped, not removed.** Unlike trimming, the units stay in the sample, so the + denominator and the sample size are preserved. This keeps more information and avoids removing a + different number of units from each arm. + +## Upper vs. lower tail + +Heavy-tailed metrics typically only have outliers in the **upper** tail; the lower tail is usually a +pile of legitimate zeros (most users do not convert). Winsorizing the lower tail of such a metric +would floor those real zeros and bias the mean upward. For that reason upper-tail winsorization is +the common case and lower-tail winsorization is opt-in and off by default. + +## Configuration + +Winsorization is configured per [`Metric`][epstats.toolkit.metric.Metric] through two optional +parameters, each expressed as a percentage (`0`–`50`) of the tail to cap: + +- `outlier_upper_percentile` — cap the upper tail. `1` caps everything above the 99th pooled + percentile down to the 99th-percentile value. +- `outlier_lower_percentile` — cap the lower tail (analogous, off by default). + +```python +from epstats.toolkit import Metric + +Metric( + 1, + 'Revenue per User', + 'value(test_unit_type.unit.revenue)', + 'count(test_unit_type.unit.exposure)', + # cap the most extreme 1% of users to the pooled 99th percentile + outlier_upper_percentile=1, +) +``` + +Over the HTTP API the same parameters are available on the metric definition as +`outlier_upper_percentile` and `outlier_lower_percentile`. + +## Scope and caveats + +- Winsorization only applies to [`evaluate_by_unit`][epstats.toolkit.experiment.Experiment.evaluate_by_unit], + because it needs the individual per-unit values. The pre-aggregated + [`evaluate_agg`][epstats.toolkit.experiment.Experiment.evaluate_agg] path has no per-unit data and + is unaffected. +- Winsorization changes the estimand slightly: you are estimating the mean of the *capped* metric + rather than of the raw metric. This is the intended trade-off — a small, controlled bias in + exchange for robustness to extreme values. Keep the capped fraction small (typically `≤ 1%`). +- The standard deviation reported after winsorization is the ordinary (capped) sample standard + deviation. With large samples and a small capped fraction the difference from a dedicated + winsorized-variance estimator is negligible. diff --git a/mkdocs.yml b/mkdocs.yml index 4b14774..1af2315 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,6 +28,7 @@ nav: - CTR Metric Redefined: 'stats/ctr.md' - Multiple Comparison Correction: 'stats/multiple.md' - Compound Metrics: 'stats/compound_metrics.md' + - Outlier Winsorization: 'stats/outliers.md' - API: - Metric: 'api/metric.md' - Experiment: 'api/experiment.md' diff --git a/src/epstats/server/req.py b/src/epstats/server/req.py index 682321d..e5b4905 100644 --- a/src/epstats/server/req.py +++ b/src/epstats/server/req.py @@ -49,6 +49,27 @@ class Metric(BaseModel): description=f"""The minimum effect of interest is the smallest relative difference that is meaningful to detect, defining it allows us to estimate the size of the sample data required to reach {DEFAULT_POWER:.0%} power.""", ) + outlier_upper_percentile: Optional[float] = Field( + None, + ge=0, + lt=50, + title="Upper-tail winsorization percentile", + description="""Percentage (`0`-`50`) of the upper tail to winsorize (cap) before the metric is evaluated. + E.g. `1` caps every per-unit value above the 99th percentile down to the 99th percentile value. The threshold + is computed once from the pooled per-unit values across all variants and applied identically to every variant, + so it makes the evaluation robust to extreme outliers without biasing the comparison between variants. Units + are kept (only their extreme values are capped). Applies only to by-unit evaluation. Defaults to no + winsorization.""", + ) + outlier_lower_percentile: Optional[float] = Field( + None, + ge=0, + lt=50, + title="Lower-tail winsorization percentile", + description="""Percentage (`0`-`50`) of the lower tail to winsorize (floor), analogous to + `outlier_upper_percentile`. Usually left empty because heavy-tailed metrics only have outliers in the upper + tail. Applies only to by-unit evaluation. Defaults to no winsorization.""", + ) @model_validator(mode="after") def check_nominator_denominator(self): @@ -60,10 +81,14 @@ def check_nominator_denominator(self): try: parser = Parser(nominator, denominator) if not parser.get_goals(): - raise ValueError("We expect the metric to have at least one goal in nominator and denominator") + raise ValueError( + "We expect the metric to have at least one goal in nominator and denominator" + ) return self except ParseException as e: - raise ValueError(f"Cannot parse nominator '{nominator}' or '{denominator}' because of '{e}'") + raise ValueError( + f"Cannot parse nominator '{nominator}' or '{denominator}' because of '{e}'" + ) def to_metric(self): return EvMetric( @@ -72,6 +97,8 @@ def to_metric(self): nominator=self.nominator, denominator=self.denominator, minimum_effect=self.minimum_effect, + outlier_upper_percentile=self.outlier_upper_percentile, + outlier_lower_percentile=self.outlier_lower_percentile, ) @@ -125,7 +152,9 @@ def _validate_nominator_or_denominator(value, which): try: parser = Parser(value, value) if not parser.get_goals(): - raise ValueError(f"We expect the check to have at least one goal in {which}") + raise ValueError( + f"We expect the check to have at least one goal in {which}" + ) return value except ParseException as e: raise ValueError(f"Cannot parse '{value}' because of '{e}'") @@ -163,7 +192,9 @@ class Filter(BaseModel): value: Optional[List[Any]] = Field(None, title="List of possible values") - goal: Optional[str] = Field(None, title="Specify goals if filter scope is `trigger`") + goal: Optional[str] = Field( + None, title="Specify goals if filter scope is `trigger`" + ) @model_validator(mode="after") def check_trigger(self): @@ -197,7 +228,9 @@ class Experiment(BaseModel): ) variants: Optional[List[str]] = Field( - None, title="Variants", description="""List of experiment variants to evaluate for.""" + None, + title="Variants", + description="""List of experiment variants to evaluate for.""", ) unit_type: str = Field( @@ -305,8 +338,12 @@ def date_to_must_be_date(cls, value): @model_validator(mode="after") def check_date_from_to(self): - if self.date_for is not None and (self.date_from is None or self.date_to is None): - raise ValueError("date_for requires date_from and date_to to be present as well") + if self.date_for is not None and ( + self.date_from is None or self.date_to is None + ): + raise ValueError( + "date_for requires date_from and date_to to be present as well" + ) if self.date_from is not None and self.date_to is not None: try: df = datetime.strptime(self.date_from, "%Y-%m-%d") # noqa: PD901 @@ -319,11 +356,17 @@ def check_date_from_to(self): except ValueError: raise ValueError("cannot parse date_for") if dfor < df: - raise ValueError("we expect date_for to be greater or equal to date_from") + raise ValueError( + "we expect date_for to be greater or equal to date_from" + ) if dfor > dt: - raise ValueError("we expect date_for to be less or equal to date_to") + raise ValueError( + "we expect date_for to be less or equal to date_to" + ) if dt < df: - raise ValueError("we expect date_to to be greater or equal to date_from") + raise ValueError( + "we expect date_to to be greater or equal to date_from" + ) return self diff --git a/src/epstats/toolkit/metric.py b/src/epstats/toolkit/metric.py index 9c85adf..185b831 100644 --- a/src/epstats/toolkit/metric.py +++ b/src/epstats/toolkit/metric.py @@ -20,6 +20,8 @@ def __init__( metric_format: str = "{:.2%}", metric_value_multiplier: int = 1, minimum_effect: Optional[float] = None, + outlier_upper_percentile: Optional[float] = None, + outlier_lower_percentile: Optional[float] = None, ): """ Constructor of the general metric definition. @@ -37,6 +39,19 @@ def __init__( denominator: definition of denominator metric_format: specify format of the metric, e.g. '${:,.1f}' for RPM metric_value_multiplier: specify multiplier, e.g. 1000 for RPM + minimum_effect: minimum (relative) effect of interest used to estimate the required sample size + outlier_upper_percentile: percentage (`0`-`50`) of the upper tail to winsorize (cap) before the + metric is evaluated. E.g. `1` caps every per-unit value above the 99th percentile down to the + 99th percentile value. The threshold is computed once from the pooled per-unit values across + all variants and applied identically to every variant, so it makes the evaluation robust to + extreme outliers without biasing the comparison between variants. Unlike trimming, units are + kept (only their extreme values are capped), so the denominator / sample size is preserved. + Applies only to [`evaluate_by_unit`][epstats.toolkit.experiment.Experiment.evaluate_by_unit]. + `None` (default) disables upper-tail winsorization. + outlier_lower_percentile: percentage (`0`-`50`) of the lower tail to winsorize (floor), analogous + to `outlier_upper_percentile`. Usually left as `None` because heavy-tailed metrics (revenue, + time spent, ...) typically only have outliers in the upper tail while the lower tail is made of + legitimate zeros. Usage: @@ -48,6 +63,12 @@ def __init__( 'count(test_unit_type.global.exposure)') ``` """ + self._validate_outlier_percentile( + "outlier_upper_percentile", outlier_upper_percentile + ) + self._validate_outlier_percentile( + "outlier_lower_percentile", outlier_lower_percentile + ) self.id = id self.name = name @@ -58,6 +79,15 @@ def __init__( self.metric_format = metric_format self.metric_value_multiplier = metric_value_multiplier self.minimum_effect = minimum_effect + self.outlier_upper_percentile = outlier_upper_percentile + self.outlier_lower_percentile = outlier_lower_percentile + + @staticmethod + def _validate_outlier_percentile(name: str, percentile: Optional[float]) -> None: + if percentile is not None and not 0 <= percentile < 50: + raise ValueError( + f"`{name}` must be in `[0, 50)` but `{percentile}` received." + ) def get_goals(self) -> Set: """ @@ -97,7 +127,18 @@ def get_evaluate_columns_by_unit(self, goals: pd.DataFrame) -> np.array: numpy array of shape (variants, metrics) where metrics are in order of (count, sum_value, sum_sqr_value) """ - return self._parser.evaluate_by_unit(goals) + lower_quantile, upper_quantile = self._get_outlier_quantiles() + return self._parser.evaluate_by_unit(goals, lower_quantile, upper_quantile) + + def _get_outlier_quantiles(self) -> (float, float): + """ + Translate the upper/lower outlier percentiles (percentage winsorized from each tail) into + the `(lower_quantile, upper_quantile)` pair used to cap per-unit values. Returns `(0.0, 1.0)` + (no winsorization) when outlier handling is disabled. + """ + lower_quantile = (self.outlier_lower_percentile or 0) / 100 + upper_quantile = 1 - (self.outlier_upper_percentile or 0) / 100 + return lower_quantile, upper_quantile class SimpleMetric(Metric): @@ -115,6 +156,8 @@ def __init__( metric_format: str = "{:.2%}", metric_value_multiplier: int = 1, minimum_effect: Optional[float] = None, + outlier_upper_percentile: Optional[float] = None, + outlier_lower_percentile: Optional[float] = None, ): """ Constructor of the simplified metric definition. @@ -149,4 +192,14 @@ def __init__( num = "value" + "(" + unit_type + "." + agg_type + "." + numerator + ")" den = "value" + "(" + unit_type + "." + agg_type + "." + denominator + ")" - super().__init__(id, name, num, den, metric_format, metric_value_multiplier, minimum_effect) + super().__init__( + id, + name, + num, + den, + metric_format, + metric_value_multiplier, + minimum_effect, + outlier_upper_percentile, + outlier_lower_percentile, + ) diff --git a/src/epstats/toolkit/parser.py b/src/epstats/toolkit/parser.py index 3fc76ed..d778c4b 100644 --- a/src/epstats/toolkit/parser.py +++ b/src/epstats/toolkit/parser.py @@ -112,13 +112,22 @@ def evaluate_agg(self, goals: pd.DataFrame): count = self._denominator_expr.evaluate_agg(goals) return count, sum_value, sum_sqr_value - def evaluate_by_unit(self, goals: pd.DataFrame): + def evaluate_by_unit( + self, + goals: pd.DataFrame, + lower_quantile: float = 0.0, + upper_quantile: float = 1.0, + ): """ Get `count`, `sum_value`, `sum_value_sqr` numpy array of shape (variants, metrics) after evaluating nominator and denominator expressions from goals aggregated by unit. Arguments: goals: one row per experiment variant + lower_quantile: per-unit nominator values below this pooled quantile (in `[0, 1]`) are capped + (winsorized) to it. `0.0` caps nothing. + upper_quantile: per-unit nominator values above this pooled quantile (in `[0, 1]`) are capped + (winsorized) to it. `1.0` caps nothing. See [`Experiment.evaluate_agg`][epstats.toolkit.experiment.Experiment.evaluate_by_unit] for details on `goals` at input. @@ -130,6 +139,10 @@ def evaluate_by_unit(self, goals: pd.DataFrame): value_variants, value = self._nominator_expr.evaluate_by_unit( goals, "sum_value" ) + + if lower_quantile > 0.0 or upper_quantile < 1.0: + value = self._winsorize(value, lower_quantile, upper_quantile) + value_df = ( pd.DataFrame( { @@ -152,6 +165,25 @@ def evaluate_by_unit(self, goals: pd.DataFrame): metrics_df = value_df.join(count_df) return metrics_df["count"], metrics_df["sum_value"], metrics_df["sum_sqr_value"] + @staticmethod + def _winsorize( + value: pd.Series, + lower_quantile: float, + upper_quantile: float, + ) -> pd.Series: + """ + Cap per-unit `value`s to the `[lower_quantile, upper_quantile]` range (winsorization). + + The capping thresholds are computed once from the pooled per-unit values across all variants + and applied identically to every variant. Because the same absolute threshold is used for all + variants, winsorization makes the evaluation robust to extreme outliers without biasing the + comparison between variants. Units are kept (only their extreme values are capped), so the + denominator / sample size is preserved. + """ + lower = value.quantile(lower_quantile) if lower_quantile > 0.0 else None + upper = value.quantile(upper_quantile) if upper_quantile < 1.0 else None + return value.clip(lower=lower, upper=upper) + def get_goals(self) -> Set: """ Get set of goals that appear in `nominator` and `denominator` expressions as `EpGoal` instances. diff --git a/tests/epstats/toolkit/test_outlier_winsorization.py b/tests/epstats/toolkit/test_outlier_winsorization.py new file mode 100644 index 0000000..e71756f --- /dev/null +++ b/tests/epstats/toolkit/test_outlier_winsorization.py @@ -0,0 +1,136 @@ +import pandas as pd +import pytest + +from src.epstats.toolkit.experiment import Experiment +from src.epstats.toolkit.metric import Metric, SimpleMetric + + +def _by_unit_goals(units_per_variant: dict) -> pd.DataFrame: + """ + Build a flat by-unit goals data frame (one row per unit and goal) from a mapping + `{variant: [per-unit conversion value, ...]}`. Every unit gets one exposure. + """ + rows = [] + unit_id = 0 + for variant, values in units_per_variant.items(): + for value in values: + unit_id += 1 + common = dict( + exp_id="test-outlier", + exp_variant_id=variant, + unit_type="test_unit_type", + agg_type="unit", + dimension="", + dimension_value="", + unit_id=unit_id, + ) + rows.append({**common, "goal": "exposure", "count": 1, "sum_value": 1}) + rows.append( + {**common, "goal": "conversion", "count": 1, "sum_value": value} + ) + return pd.DataFrame(rows) + + +def _evaluate(goals, outlier_upper_percentile=None, outlier_lower_percentile=None): + metric = Metric( + 1, + "Average Bookings", + "value(test_unit_type.unit.conversion)", + "count(test_unit_type.unit.exposure)", + outlier_upper_percentile=outlier_upper_percentile, + outlier_lower_percentile=outlier_lower_percentile, + ) + experiment = Experiment( + "test-outlier", "a", [metric], [], unit_type="test_unit_type" + ) + return experiment.evaluate_by_unit(goals).metrics.set_index("exp_variant_id") + + +def test_upper_winsorization_caps_outlier_and_keeps_units(): + # Variant `a` is clean, variant `b` has a single extreme outlier. + goals = _by_unit_goals({"a": [10] * 100, "b": [10] * 99 + [100_000]}) + + metrics = _evaluate(goals, outlier_upper_percentile=1) + + # The outlier in `b` is capped to the pooled upper threshold (10), but the unit is *kept*, + # so the count / sample size is unchanged (this is the key difference from trimming). + assert metrics.loc["a", "count"] == 100 + assert metrics.loc["a", "mean"] == pytest.approx(10) + assert metrics.loc["b", "count"] == 100 + assert metrics.loc["b", "mean"] == pytest.approx(10) + + +def test_pooled_threshold_is_applied_identically_to_all_variants(): + # Each variant has the same clean values 1..50 plus a single (different) extreme outlier. + clean = list(range(1, 51)) + goals = _by_unit_goals({"a": clean + [10_000], "b": clean + [20_000]}) + + metrics = _evaluate(goals, outlier_upper_percentile=2) + + # The cap is the pooled 98th percentile (= 50), so both arms' outliers are capped to the same + # absolute value. The two variants therefore evaluate identically despite different raw outliers. + expected_sum = sum(clean) + 50 # the outlier is capped down to 50 + assert metrics.loc["a", "count"] == 51 + assert metrics.loc["b", "count"] == 51 + assert metrics.loc["a", "sum_value"] == pytest.approx(expected_sum) + assert metrics.loc["b", "sum_value"] == pytest.approx(expected_sum) + assert metrics.loc["a", "mean"] == pytest.approx(metrics.loc["b", "mean"]) + + +def test_lower_winsorization_floors_low_values(): + goals = _by_unit_goals({"a": [10] * 100, "b": [10] * 99 + [-100_000]}) + + metrics = _evaluate(goals, outlier_lower_percentile=1) + + # The extreme low value in `b` is floored to the pooled lower threshold (10), unit kept. + assert metrics.loc["b", "count"] == 100 + assert metrics.loc["b", "mean"] == pytest.approx(10) + + +def test_winsorization_disabled_keeps_raw_values(): + goals = _by_unit_goals({"a": [10] * 100, "b": [10] * 99 + [100_000]}) + + metrics = _evaluate(goals) + + # Without winsorization the outlier inflates the mean of `b`. + assert metrics.loc["b", "count"] == 100 + assert metrics.loc["b", "mean"] == pytest.approx((10 * 99 + 100_000) / 100) + + +def test_disabled_matches_zero_percentiles(): + goals = _by_unit_goals({"a": list(range(1, 101)), "b": list(range(50, 150))}) + + disabled = _evaluate(goals) + zero = _evaluate(goals, outlier_upper_percentile=0, outlier_lower_percentile=0) + + pd.testing.assert_frame_equal(disabled, zero) + + +@pytest.mark.parametrize("percentile", [-1, 50, 60, 100]) +@pytest.mark.parametrize( + "param", ["outlier_upper_percentile", "outlier_lower_percentile"] +) +def test_outlier_percentile_out_of_range_raises(param, percentile): + with pytest.raises(ValueError): + Metric( + 1, + "Average Bookings", + "value(test_unit_type.unit.conversion)", + "count(test_unit_type.unit.exposure)", + **{param: percentile}, + ) + + +def test_simple_metric_passes_outlier_percentiles(): + metric = SimpleMetric( + 1, + "Average Bookings", + "conversion", + "exposure", + unit_type="test_unit_type", + outlier_upper_percentile=5, + outlier_lower_percentile=2, + ) + assert metric.outlier_upper_percentile == 5 + assert metric.outlier_lower_percentile == 2 + assert metric._get_outlier_quantiles() == (0.02, 0.95) From d03aeb6afc54e4acedb02b6e17c288f5424e8372 Mon Sep 17 00:00:00 2001 From: Ondra Zahradnik Date: Tue, 16 Jun 2026 11:01:24 +0200 Subject: [PATCH 2/2] Fix flaky timestamp comparison in winsorization test `test_disabled_matches_zero_percentiles` compared the full metrics DataFrames from two evaluations, including the `timestamp` column that is stamped from `datetime.now()`. When the two calls straddled a one-second boundary (as on the 3.12 CI runner) the timestamps differed and the test failed. Drop the non-deterministic `timestamp` column before comparing. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/epstats/toolkit/test_outlier_winsorization.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/epstats/toolkit/test_outlier_winsorization.py b/tests/epstats/toolkit/test_outlier_winsorization.py index e71756f..5de71fb 100644 --- a/tests/epstats/toolkit/test_outlier_winsorization.py +++ b/tests/epstats/toolkit/test_outlier_winsorization.py @@ -100,8 +100,11 @@ def test_winsorization_disabled_keeps_raw_values(): def test_disabled_matches_zero_percentiles(): goals = _by_unit_goals({"a": list(range(1, 101)), "b": list(range(50, 150))}) - disabled = _evaluate(goals) - zero = _evaluate(goals, outlier_upper_percentile=0, outlier_lower_percentile=0) + # `timestamp` is stamped from `datetime.now()` at evaluation time, so drop it before comparing. + disabled = _evaluate(goals).drop(columns="timestamp") + zero = _evaluate( + goals, outlier_upper_percentile=0, outlier_lower_percentile=0 + ).drop(columns="timestamp") pd.testing.assert_frame_equal(disabled, zero)