From 87a0bd67cbb7c49153bdf642f24254c3473108ba Mon Sep 17 00:00:00 2001 From: caterryan Date: Thu, 9 Jul 2026 16:31:24 -0500 Subject: [PATCH 1/3] feat(signal-generator): add simulated gaps --- influxdata/library/plugin_library.json | 4 +- influxdata/signal_generator/GAP_DESIGN.md | 562 ++++++++++++++++++ .../GAP_IMPLEMENTATION_PLAN.md | 258 ++++++++ influxdata/signal_generator/README.md | 53 +- influxdata/signal_generator/manifest.toml | 4 +- .../signal_generator/signal_generator.py | 244 +++++++- .../signal_generator/test_signal_generator.py | 462 +++++++++++++- 7 files changed, 1571 insertions(+), 16 deletions(-) create mode 100644 influxdata/signal_generator/GAP_DESIGN.md create mode 100644 influxdata/signal_generator/GAP_IMPLEMENTATION_PLAN.md diff --git a/influxdata/library/plugin_library.json b/influxdata/library/plugin_library.json index b089733..327158b 100644 --- a/influxdata/library/plugin_library.json +++ b/influxdata/library/plugin_library.json @@ -244,12 +244,12 @@ { "name": "Signal Generator", "path": "influxdata/signal_generator/signal_generator.py", - "description": "[BETA] Requires InfluxDB 3.8.2+. Generates configurable waveform-based time-series data on a schedule for demos, testing, dashboards, alerts, and downstream plugin validation without requiring an external data source.", + "description": "[BETA] Requires InfluxDB 3.8.2+. Generates configurable waveform-based time-series data on a schedule for demos, testing, dashboards, alerts, and downstream plugin validation without requiring an external data source. Simulates data-source outage gaps by default.", "author": "InfluxData", "docs_file_link": "https://github.com/influxdata/influxdb3_plugins/blob/main/influxdata/signal_generator/README.md", "required_plugins": [], "required_libraries": [], - "last_update": "2026-05-15", + "last_update": "2026-07-09", "trigger_types_supported": ["scheduler"] }, { diff --git a/influxdata/signal_generator/GAP_DESIGN.md b/influxdata/signal_generator/GAP_DESIGN.md new file mode 100644 index 0000000..06dfcdd --- /dev/null +++ b/influxdata/signal_generator/GAP_DESIGN.md @@ -0,0 +1,562 @@ +# Data Gap Design + +## Purpose + +Add simulated temporary disconnections to the signal generator so generated data +contains realistic missing spans. A gap represents a data source outage: points +inside the outage window are absent from output. + +Gaps are not a waveform type. They affect whether points are written, not the +signal value calculation. + +## Important Context + +This design depends on existing Signal Generator behavior documented in: + +- `README.md`: The plugin currently advertises gap-filling behavior, meaning it + generates continuous data between scheduled executions. Simulated gaps are an + intentional exception to that behavior and must be documented as such (see + [Breaking Change and Documentation Impact](#breaking-change-and-documentation-impact)). +- `README.md`: The first scheduled execution initializes cache state and writes + no data. Gap scheduling preserves this startup behavior. +- `README.md`: `points_per_second` controls generated timestamp resolution. + Gap behavior must work equally well when each scheduled call writes many + points or only one point. +- `README.md` (Timestamp jitter section): Timestamp jitter is configured as + flat trigger arguments, is enabled by default, and uses a seed for + reproducibility. Gap configuration follows the same style. Jitter applies + after nominal timestamp generation; simulated gaps apply after jitter so + filtering uses final recorded timestamps. +- `signal_generator.py`: Cache state currently stores `signal_gen:last_time` as + the unjittered generation boundary. Gaps must not change that meaning. +- `signal_generator.py`: `_rng_for_timestamp` already derives per-timestamp + jitter from a `blake2b` digest of `(seed, timestamp)`, so draws are a pure + function of their inputs. The gap schedule uses the same primitive, keyed by + gap index instead of timestamp. +- `signal_generator.py`: Trigger arguments are parsed at scheduled-call runtime, + so invalid gap configuration must be reported at runtime and should produce no + writes for that call. +- `signal_generator.py`: Waveforms are composed independently from output + configuration. Gaps stay outside waveform configuration and are not a + waveform type. + +## User Model + +Users configure explicit bounds on how long outages last and how often they +begin. Both are randomized within those bounds, so output does not look +mechanically periodic. + +Gaps are enabled by default. This is a breaking change for existing triggers; +see [Breaking Change and Documentation Impact](#breaking-change-and-documentation-impact) +for the required mitigations. Users who need continuous output can disable +gaps with `gap_enabled=false`. + +## Configuration Shape + +Gaps are configured with flat trigger arguments: + +| Argument | Type | Default | Description | +|----------|------|---------|-------------| +| `gap_enabled` | boolean string | `true` | Enables or disables simulated gaps. | +| `gap_min_duration_seconds` | float string | `10.0` | Shortest outage duration, in seconds. | +| `gap_max_duration_seconds` | float string | `30.0` | Longest outage duration, in seconds. | +| `gap_min_interval_seconds` | float string | `120.0` | Shortest time between consecutive gap starts, in seconds. | +| `gap_max_interval_seconds` | float string | `240.0` | Longest time between consecutive gap starts, in seconds. | +| `gap_seed` | integer string | none | Optional seed for reproducible gap timing. | + +Example: + +```text +gap_min_duration_seconds=10,gap_max_duration_seconds=30,gap_min_interval_seconds=120,gap_max_interval_seconds=240,gap_seed=123 +``` + +The min/max bounds are hard guarantees, not soft targets: + +- every gap's duration lies in + `[gap_min_duration_seconds, gap_max_duration_seconds]`; +- the time between consecutive gap starts lies in + `[gap_min_interval_seconds, gap_max_interval_seconds]`. + +Internally the schedule derives a grid pitch and a per-gap start offset from +the interval bounds: + +```text +pitch = (gap_min_interval_seconds + gap_max_interval_seconds) / 2 +offset = (gap_max_interval_seconds - gap_min_interval_seconds) / 4 + +start(n) = n * pitch + uniform(-offset, +offset) +duration(n) = uniform(gap_min_duration_seconds, gap_max_duration_seconds) +``` + +The offset half-range is a quarter of the interval span, not half, because the +spacing between consecutive starts is `pitch` plus the *difference of two +independent offsets*: that difference spans twice the offset half-range, so a +quarter-span offset makes consecutive spacing land exactly in +`[gap_min_interval_seconds, gap_max_interval_seconds]`. See +[Gap Schedule Algorithm](#gap-schedule-algorithm). + +Out-of-range values are rejected by validation and logged as errors; they are +never silently corrected or clamped. + +## Default Behavior + +Default gaps should be visible but not dominant: + +```text +gap_min_duration_seconds = 10.0 +gap_max_duration_seconds = 30.0 +gap_min_interval_seconds = 120.0 +gap_max_interval_seconds = 240.0 +``` + +This yields outage durations from 10 to 30 seconds, and gap starts spaced 2 to +4 minutes apart (usually close to 3 minutes). Expected data loss is +`mean duration / mean interval` = 20/180 ≈ 11% of points, and at least +`gap_min_interval_seconds - gap_max_duration_seconds` = 90 seconds of +continuous data are guaranteed between consecutive gaps. At the default +`points_per_second=1.0`, a typical gap removes about 20 points. + +The defaults are intentionally moderate: users should notice missing spans in +query results and dashboards without losing most generated data. + +## Disabling Gaps + +Set `gap_enabled=false` to preserve continuous output. This is the only +supported disable mechanism. + +Setting both duration bounds to `0` is valid and produces empty gap windows +(effectively no gaps), but `gap_enabled=false` is the explicit, documented +form. + +## Gap Schedule Algorithm + +The gap schedule is a **jittered grid**: absolute time is divided into strata +of width `pitch`, anchored at Unix epoch 0, and stratum `n` contains exactly +one gap whose start offset and duration are derived from a hash of +`(seed, n)`. Every gap window is a pure function of +`(gap_seed, configuration, gap_index)` — no sequential state, no replay from +an anchor. + +This is the same construction the plugin already uses for per-timestamp jitter +(`_rng_for_timestamp`), keyed by gap index instead of timestamp: + +```python +def _rng_for_gap_index(seed, n): + payload = f"{seed}:gap:{n}".encode("ascii") + digest = hashlib.blake2b(payload, digest_size=8).digest() + return random.Random(int.from_bytes(digest, "big")) + +def gap_window(seed, n): + """Gap n's half-open [start, end) window, in epoch seconds. + Draw order is fixed: start offset first, then duration.""" + rng = _rng_for_gap_index(seed, n) + start = n * pitch + rng.uniform(-offset, +offset) + duration = rng.uniform(min_duration, max_duration) + return (start, start + duration) +``` + +The `"gap"` tag in the payload domain-separates gap draws from any other +hash-derived draws using the same numeric key space. The draw order (offset, +then duration) is part of the reproducibility contract: reordering the draws +changes every schedule produced by a given seed. + +### Membership test + +Filtering asks, per point: is final timestamp `t` inside any gap window? Under +the validation constraint below, at most two gap indices can contain `t`, so +membership is O(1): + +```python +def in_gap(seed, t): + n_hi = int((t + offset) // pitch) + for n in (n_hi, n_hi - 1): + if n < 0: + continue + start, end = gap_window(seed, n) + if start <= t < end: + return True + return False +``` + +### Properties + +These follow from the construction and were verified by simulation over 300k +gap windows and 200k random membership queries against brute force: + +- **Batch-size invariance**: membership depends only on the point's final + timestamp, never on scheduled-call boundaries or how many points a call + generates. Filtering one point per call and filtering thousands per call + produce identical output. In particular, a trigger generating one point per + execution does *not* roll a fresh gap decision each execution — the known + failure mode of per-call probability draws, where outage frequency scales + with trigger frequency instead of simulated time. +- **Reproducibility**: same seed and configuration produce bit-identical gap + windows, across calls, restarts, and batch sizes. +- **No persistent state**: nothing about the schedule is stored. The only new + cache entry is the auto-generated seed when `gap_seed` is omitted (see + [Cache Semantics](#cache-semantics)). +- **Bounded spacing**: consecutive gap starts are spaced + `pitch + (offset_next - offset_prev)`, and each offset is at most a quarter + of the interval span, so spacing always lies in + `[gap_min_interval_seconds, gap_max_interval_seconds]` — the bounds the + argument names promise. Verified: 300k consecutive spacings at defaults all + fell within [120, 240], mean 180. +- **Non-overlap**: gaps cannot overlap when the validation constraint + `gap_min_interval_seconds >= gap_max_duration_seconds` holds, and at least + `gap_min_interval_seconds - gap_max_duration_seconds` seconds of data are + guaranteed between consecutive gaps (90 seconds at defaults). +- **Spacing distribution**: within its [min, max] bounds, spacing follows a + triangular distribution peaked at the midpoint, not a uniform one, and every + gap stays within `±offset` of its own grid position, so there is no + cumulative drift — the schedule is quasi-periodic rather than a renewal + process with independent inter-arrival times. The min/max bounds are exact; + only the shape inside them is not uniform. This trade-off is deliberate; see + [Alternative Schedule Algorithms Considered](#alternative-schedule-algorithms-considered). +- **Configuration is part of the schedule identity**: changing any interval or + duration bound re-derives the entire schedule. An in-progress gap does not + survive a configuration change. + +### Prior art + +This lattice-plus-keyed-hash construction is standard for reproducible, +random-access randomness over an unbounded domain: jittered stratified +sampling (one sample per stratum, offset from the stratum anchor), stateless +procedural world generation (cell coordinates hashed to a per-cell seed, with +a proven bounded influence radius that makes constant-neighborhood queries +correct), Pixar's correlated multi-jittered sampling (any sample computable +directly from its index), and counter-based RNGs such as Philox (the Nth draw +is a stateless function of key and counter). Chaos-testing tools document the +anti-patterns this design avoids: per-request probability draws couple fault +frequency to request frequency, and re-seeding a sequential RNG per evaluation +collapses to a constant decision. + +## Pipeline Semantics + +Gaps are applied after signal generation and timestamp jitter: + +```text +nominal timestamps -> signal values -> jittered timestamps -> gap filtering -> write +``` + +This ordering matters. Jitter determines the final recorded timestamp. Gap +filtering then removes any point whose final timestamp lands inside a simulated +outage window. + +The result is a source-disconnection model: points inside the outage do not +exist in output. + +## Gap Window Semantics + +A gap is a half-open time window: + +```text +[gap_start, gap_end) +``` + +Any point with a final timestamp inside that window is removed. Points exactly +at `gap_end` are kept. This mirrors the existing timestamp generation convention +that avoids duplicate boundary handling across adjacent windows. + +The same seed controls both the start-offset and duration draws, preserving a +single reproducibility knob. + +Example, at defaults with some seed (`pitch` = 180, `offset` = 30; gap indices +are large in practice because the grid is anchored at epoch — small indices +shown for readability): + +```text +gap_min_interval_seconds = 120 +gap_max_interval_seconds = 240 +gap_min_duration_seconds = 10 +gap_max_duration_seconds = 30 + +gap n nominal start n*180 actual start duration +1000 180000 179973 27 +1001 180180 180198 13 +1002 180360 180351 22 +``` + +Nominal positions are a fixed grid; each gap's start shifts by at most +`offset` around its own grid position. Consecutive starts here are spaced +225s and 153s apart — always within [120, 240]. + +## Randomness + +Gap randomness is based on absolute simulated time, not on write-cycle +boundaries. + +This is required for low-volume configurations such as one point per scheduled +call. A per-call random decision would make gaps correlate with trigger +executions and could produce "all or nothing" behavior each cycle. +Absolute-time gap windows produce the same outage pattern whether points are +written one at a time or batched many per call. + +When `gap_seed` is set, the gap schedule is reproducible: the same seed and +configuration produce the same gap windows regardless of scheduled-call batch +size, restarts, or catch-up after downtime. + +When `gap_seed` is omitted, the plugin generates a random integer seed during +the first-run initialization and stores it in the cache (see +[Cache Semantics](#cache-semantics)). All subsequent calls read the cached +seed and take the same code path as an explicit seed, so an outage that begins +in one execution continues coherently in later executions until its duration +ends. If the cache is lost (for example, across a server restart that clears +the trigger cache), a new seed is generated and the schedule changes: an +in-progress gap may end early or new gaps may appear at different times. This +is acceptable for demo data; users who need schedule stability across restarts +should set `gap_seed` explicitly. + +## Cache Semantics + +The cache continues to represent generated-time progress, not written-time +progress. `signal_gen:last_time` keeps its existing meaning: the unjittered +generation boundary. + +One new cache key is added: + +- `signal_gen:gap_seed`: the auto-generated gap seed, written once during + first-run initialization when `gap_seed` is not configured. Never written + when `gap_seed` is configured. + +When points are removed by gaps, the plugin still advances its generation +boundary. Missing data must remain missing; it must not be backfilled during +the next scheduled call. + +Catch-up after downtime follows the same rule: when a call generates a long +range of points (because `last_time` is far in the past), the absolute-time +gap schedule applies to that entire range, so backfilled spans contain the +same gaps they would have contained if the plugin had been running. + +## Validation + +Valid gap configuration: + +- `gap_enabled` parses as a boolean when provided. +- `gap_min_duration_seconds` parses as a finite float with value `>= 0`. +- `gap_max_duration_seconds` parses as a finite float with value + `>= gap_min_duration_seconds`. +- `gap_min_interval_seconds` parses as a finite float with value `> 0`. +- `gap_max_interval_seconds` parses as a finite float with value + `>= gap_min_interval_seconds`. +- `gap_min_interval_seconds >= gap_max_duration_seconds`. +- `gap_seed` is omitted or parses as an integer. + +Finiteness matters: `inf` passes a bare `>= 0` check and would produce a +permanent outage. The existing `points_per_second` and jitter validation +already require finite values; gap validation matches. + +`gap_min_interval_seconds >= gap_max_duration_seconds` is the load-bearing +rule, and reads as its own explanation: the shortest time between two outage +starts must fit the longest outage. It guarantees: + +- gap windows never overlap (worst case: gap `n` ends at + `n*pitch + offset + max_duration`, gap `n+1` starts at + `(n+1)*pitch - offset`, and the difference is + `pitch - 2*offset - max_duration = min_interval - max_duration >= 0`); +- the membership test only ever needs to inspect two candidate indices; +- a minimum of `gap_min_interval_seconds - gap_max_duration_seconds` seconds + of continuous data between consecutive gaps. + +Defaults satisfy it: `120 >= 30`. At exact equality gaps may touch +back-to-back (the half-open windows still never overlap or double-remove a +boundary point); simulation at the equality boundary over 300k gaps confirmed +no overlap. + +Invalid configuration logs an error and writes nothing for that scheduled +call. Runtime validation is the enforcement point because trigger arguments +are not validated at trigger-creation time. Values are rejected, not clamped. + +## Interaction With Jitter + +Gaps operate on jittered timestamps. A point near a gap boundary can be moved +into or out of the gap by timestamp jitter. + +This is desirable: from the user's perspective, gaps describe missing output +time ranges, and jitter describes the final recorded timestamp. Filtering after +jitter makes the final written timeline match the configured outage windows. + +`gap_seed` and `jitter_seed` are independent knobs, and the gap draws are +domain-separated from jitter draws by the `"gap"` payload tag, so identical +seed values cannot produce correlated draws. + +## Interaction With Waveforms + +Waveforms still define what values would have existed at each generated time. +Gaps only remove points from output. + +Deterministic waveforms remain anchored to absolute time. Stochastic waveforms +continue to use their own seed behavior. Gap configuration should not alter +waveform values outside the missing windows. + +## Observability + +Normal successful runs should report how many points were written. When gaps +remove points, logs should make that visible without becoming noisy. + +Useful summary fields: + +- generated point count +- removed-by-gap point count +- written point count +- number of gap windows intersecting the call window + +This helps users distinguish "no points because interval too short" from +"points generated but all removed by simulated gap." + +## Breaking Change and Documentation Impact + +Enabling gaps by default changes the plugin's advertised core behavior. +Shipping this requires, in the same change: + +- Rewrite the README "Gap-filling" headline bullet: continuous generation + between executions still holds, but simulated outage gaps are now present by + default and are a feature, not a bug. +- Add a README troubleshooting entry for "missing spans in query results" + explaining simulated gaps and `gap_enabled=false`. +- Call out explicitly that existing triggers pick up default gaps on upgrade + with no configuration change, and that downstream demos using deadman or + no-data alerts will start firing on the simulated outages. That may be + desirable (it exercises those alerts) but must not be a surprise. +- Add the six `gap_*` arguments to the plugin docstring's + `scheduled_args_config` metadata (required for InfluxDB 3 Explorer UI + integration). +- Update `influxdata/library/plugin_library.json` metadata for the new + version. +- Note the behavior change in the release/PR notes. + +## Test Plan + +Beyond unit tests for parsing and validation (each rule above, valid and +invalid, including `inf`/`nan`, swapped min/max, and the +`min_interval >= max_duration` rule): + +- **Batch invariance**: generate a fixed time range one point per call vs. + all points in one call; written timestamps must be identical. +- **Membership correctness**: compare `in_gap` against a brute-force scan of + all gap windows over a large sampled range. +- **Bounded spacing**: assert consecutive gap starts are always spaced within + `[gap_min_interval_seconds, gap_max_interval_seconds]` and durations within + `[gap_min_duration_seconds, gap_max_duration_seconds]` over a large index + range. +- **Non-overlap**: assert windows are disjoint over a large index range at + the validation boundary (`gap_min_interval_seconds == + gap_max_duration_seconds`) and at defaults. +- **Reproducibility**: same seed and config produce identical windows across + separate processes; different seeds produce different windows. +- **Gap spanning call boundaries**: a gap that starts in one call's window + and ends in a later call's window removes points in both calls. +- **Gap covering an entire call window**: call writes zero points, logs the + removed-by-gap count, and still advances `last_time`. +- **Catch-up**: after simulated downtime, the backfilled range contains gaps + at the same absolute positions as an uninterrupted run with the same seed. +- **Unseeded stability**: with `gap_seed` omitted, the cached auto-seed makes + consecutive calls share one schedule. +- **Disabled**: `gap_enabled=false` writes all generated points. +- **First-run init**: unchanged — no writes, cache initialized (including + `signal_gen:gap_seed` when unseeded). + +## Alternative Schedule Algorithms Considered + +### Sequential renewal process + +Draw each inter-gap interval from +`uniform(gap_min_interval_seconds, gap_max_interval_seconds)` and accumulate: +gap `n+1` starts where gap `n`'s interval ends. This gives independent, +identically distributed (uniform) spacing — but start times are cumulative +sums, so answering "is timestamp T inside a gap?" requires replaying every +draw from an anchor (millions of draws per call for an epoch anchor) or +persisting RNG state in the cache. Persisted state breaks batch invariance and +catch-up determinism, and grows the cache contract. Renewal processes do not +admit O(1) random access; this disqualifies them here. The jittered grid keeps +the same [min, max] spacing bounds and mean, trading uniform spacing for a +triangular distribution inside the bounds. + +### Per-call probability draw + +Each scheduled call rolls "does an outage start now?". Rejected outright: gap +frequency then scales with trigger frequency rather than simulated time, and a +one-point-per-call configuration degenerates to all-or-nothing per cycle. +Chaos-testing tools that use per-request draws document exactly this coupling. + +### Stateful burst-loss model (Gilbert-Elliott) + +A two-state good/bad Markov chain produces realistic burst losses, but the +decision at each point depends on hidden state carried from the previous +point — sequential by construction, with the same random-access and +persistence problems as the renewal process, plus a distribution concept the +plugin does not otherwise expose. + +## Alternative Configuration Shapes Considered + +### Center Plus Jitter + +```text +gap_duration_seconds=20 +gap_duration_jitter_seconds=10 +gap_interval_seconds=180 +gap_interval_jitter_seconds=60 +``` + +This matches the timestamp-jitter vocabulary, but the interval arguments are +misleading under a jittered-grid schedule: `gap_interval_jitter_seconds` would +bound each gap's offset from its grid position, while the spacing between +consecutive starts varies by twice that amount (180±60 config actually yields +60–300s spacing). The non-overlap validation rule is also harder to state +(`interval - 2*jitter >= duration + duration_jitter`). Min/max bounds are +directly checkable guarantees, and their validation rule +(`min_interval >= max_duration`) is self-explanatory. + +### Probability Per Second + +```text +gap_start_probability_per_second=0.005 +gap_duration_seconds=20 +gap_duration_jitter_seconds=10 +``` + +This maps cleanly to a stochastic process, but users tend to think in "roughly +every few minutes" rather than per-second probabilities. + +### Average and Distribution + +```text +gap_avg_duration_seconds=20 +gap_avg_interval_seconds=180 +gap_distribution=exponential +``` + +This can produce realistic outage timing, with many short intervals and a few +long quiet periods. It is harder to explain, validate, and test, and it adds a +distribution concept that the plugin does not otherwise expose. + +### Fixed Plus Random Factor + +```text +gap_duration_seconds=20 +gap_duration_random_factor=0.5 +gap_interval_seconds=180 +gap_interval_random_factor=0.25 +``` + +This is compact, but less obvious than duration and jitter expressed directly in +seconds. + +## Chosen Design + +Use a jittered-grid schedule over absolute time — one gap per `pitch`-wide +stratum, with start offset and duration derived from a `blake2b` hash of +`(seed, gap_index)` — configured with explicit min/max bounds: + +```text +gap_min_duration_seconds +gap_max_duration_seconds +gap_min_interval_seconds +gap_max_interval_seconds +gap_seed +gap_enabled +``` + +The bounds are hard guarantees on outage duration and on spacing between +consecutive outage starts. All gap parameters stay optional while default gaps +remain visible, and the schedule is reproducible, stateless, O(1) to query, +and independent of how many points each scheduled call generates. diff --git a/influxdata/signal_generator/GAP_IMPLEMENTATION_PLAN.md b/influxdata/signal_generator/GAP_IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000..c3e9eb7 --- /dev/null +++ b/influxdata/signal_generator/GAP_IMPLEMENTATION_PLAN.md @@ -0,0 +1,258 @@ +# Data Gap Implementation Plan + +Implements `GAP_DESIGN.md` (jittered-grid gap schedule, min/max configuration +shape). Read that document first; this plan only sequences the work and maps +it onto files. Where this plan and the design doc disagree, the design doc +wins. + +## Files touched + +| File | Change | +|------|--------| +| `signal_generator.py` | Gap config parsing, schedule functions, filtering, entry-point wiring, docstring metadata | +| `test_signal_generator.py` | New gap tests; update existing entry-point tests that assert exact output (gaps are on by default) | +| `README.md` | Gap section, headline bullet rewrite, parameter table, troubleshooting entry | +| `manifest.toml` | Version bump `0.2.0` → `0.3.0` | +| `influxdata/library/plugin_library.json` | Description mentions simulated gaps; `last_update` | + +## Step 1: Gap config parsing (`signal_generator.py`) + +Add defaults next to the existing jitter constants: + +```python +DEFAULT_GAP_MIN_DURATION_SECONDS = 10.0 +DEFAULT_GAP_MAX_DURATION_SECONDS = 30.0 +DEFAULT_GAP_MIN_INTERVAL_SECONDS = 120.0 +DEFAULT_GAP_MAX_INTERVAL_SECONDS = 240.0 +``` + +Add `parse_gap_config(args)` following the `parse_jitter_config` contract — +returns `(config, None)` on success or `(None, error_message)` on failure, +error messages prefixed `"Invalid gap config: "`. Config is a tuple: + +```python +(enabled, min_duration, max_duration, min_interval, max_interval, pitch, offset, seed) +``` + +with `pitch = (min_interval + max_interval) / 2` and +`offset = (max_interval - min_interval) / 4` precomputed at parse time. +`seed` is the configured `gap_seed` or `None` (resolved later, Step 3). + +Rules (all from the design doc's Validation section): + +- `gap_enabled`: accept `"true"`/`"false"` case-insensitive; default `True`; + anything else is an error. No boolean parser exists yet — add a small + `_parse_bool` helper. +- All four numeric args parse via `float(raw)` and must be finite + (`math.isfinite`), matching the existing jitter/pps checks. +- `min_duration >= 0`, `max_duration >= min_duration`, `min_interval > 0`, + `max_interval >= min_interval`, and the load-bearing rule + `min_interval >= max_duration`. +- `gap_seed` parses with the same integer coercion used for `jitter_seed` + (reject bools and non-integral floats). +- Validate every provided argument even when `gap_enabled=false`; disabled + only skips filtering, not validation. + +## Step 2: Schedule and filtering functions (`signal_generator.py`) + +Place next to `_rng_for_timestamp` — same primitive, keyed by gap index, with +`"gap"` domain separation: + +```python +def _rng_for_gap_index(seed, n): + payload = f"{seed}:gap:{n}".encode("ascii") + digest = hashlib.blake2b(payload, digest_size=8).digest() + return random.Random(int.from_bytes(digest, "big")) +``` + +`gap_window(gap_config, n)` — draw order is a reproducibility contract: +start offset first, then duration: + +```python +rng = _rng_for_gap_index(seed, n) +start = n * pitch + rng.uniform(-offset, +offset) +duration = rng.uniform(min_duration, max_duration) +return (start, start + duration) +``` + +`in_gap(gap_config, t)` — check candidate indices +`n_hi = int((t + offset) // pitch)` and `n_hi - 1`, skipping `n < 0`; +half-open comparison `start <= t < end`. + +`apply_gaps(points, gap_config)` — runs after `apply_timestamp_jitter`, tests +each point's final (jittered) timestamp, returns +`(kept_points, removed_count, gap_window_count)` where `gap_window_count` is +the number of gap indices whose window intersects +`[min(point ts), max(point ts)]` (iterate the index range +`int((t_min - offset - max_duration) // pitch)` to `int((t_max + offset) // pitch)`; +the range is tiny relative to any realistic call window). Memoize +`gap_window` per index in a local dict for the duration of the call so a +window shared by thousands of points hashes once. + +## Step 3: Seed resolution and cache (`signal_generator.py`) + +```python +GAP_SEED_CACHE_KEY = "signal_gen:gap_seed" +``` + +`resolve_gap_seed(cache, configured_seed)`: + +- configured seed set → return it, never touch the cache; +- else cached value present → return it; +- else generate `random.getrandbits(63)`, `cache.put`, return it. + +Called on every scheduled call when gaps are enabled (including first-run +init, so the schedule exists before the first data write). Cache loss simply +regenerates — documented behavior. + +`signal_gen:last_time` semantics unchanged: always advances to `now`, even +when every generated point was removed by a gap. No backfill. + +## Step 4: Entry-point wiring (`process_scheduled_call`) + +Parse order: waveforms → output → pps → jitter → **gap** (new). A gap config +error logs `f"[{task_id}] {gap_error}"` and returns before touching the +cache, matching the invalid-jitter behavior exactly (existing test pattern: +cache stays `None`). + +First-run init block: additionally call `resolve_gap_seed` when gaps are +enabled, then return as today. + +Generation path, after `apply_timestamp_jitter`: + +```python +generated = len(points) +if gap_enabled: + points, removed, gap_windows = apply_gaps(points, gap_config) +``` + +Then write as today (an empty kept-list skips the write and still advances +the cache — `write_points` already returns 0 for no lines). Success log +becomes, when gaps removed anything: + +```text +Wrote {written} points to {m}.{f} ({generated} generated, {removed} removed by {gap_windows} gap windows) +``` + +and stays the current one-liner when `removed == 0`, keeping logs quiet in +the common case. + +## Step 5: Docstring metadata + +Add the six `gap_*` arguments to `scheduled_args_config` (name, example, +description, `"required": false` — same shape as the jitter entries). +Examples: `gap_enabled` → `"false"`, durations → `"10"`/`"30"`, intervals → +`"120"`/`"240"`, `gap_seed` → `"123"`. Descriptions state the min/max bound +guarantees and that gaps are on by default. + +## Step 6: Tests (`test_signal_generator.py`) + +### Fix existing tests first — gaps are on by default + +Any entry-point test asserting exact point counts or timestamps now runs with +an unseeded (random) gap schedule and becomes nondeterministic. Add +`"gap_enabled": "false"` to the args of: + +- `test_process_scheduled_call_second_run` (currently passes no args — give + it `{"gap_enabled": "false"}`) +- `test_process_scheduled_call_custom_pps` +- `test_process_scheduled_call_target_database` +- `test_process_scheduled_call_jitter_zero_keeps_regular_timestamps` +- `test_process_scheduled_call_seeded_jitter_offsets_timestamps` +- `test_process_scheduled_call_seeded_jitter_varies_across_one_point_calls` +- `test_process_scheduled_call_default_jitter_offsets_timestamps` + +`test_process_scheduled_call_first_run` and `..._updates_cache` assert only +cache/no-write behavior and can stay as-is; extend `first_run` to also assert +`signal_gen:gap_seed` is populated. + +### New parsing/validation tests (mirror the jitter test style) + +- defaults (no args → enabled, 10/30/120/240, derived pitch 180 / offset 30) +- explicit values round-trip; `gap_enabled` `"false"`/`"TRUE"`/garbage +- each rejection: non-numeric, `nan`, `inf`, negative min duration, + `max_duration < min_duration`, `min_interval <= 0`, + `max_interval < min_interval`, `min_interval < max_duration`, + non-integer `gap_seed` +- disabled-but-invalid still errors + +### New schedule-function tests (design doc Test Plan section) + +- **Bounded spacing + non-overlap**: iterate `gap_window` over a few thousand + indices at defaults and at the equality boundary + (`min_interval == max_duration`); assert spacing within `[min_i, max_i]`, + durations within `[min_d, max_d]`, windows disjoint. +- **Membership correctness**: `in_gap` vs brute-force scan over the covering + index range for a few thousand sampled timestamps. +- **Reproducibility**: same seed → identical windows; different seeds differ. +- **Determinism pin**: one exact-value test in the style of + `test_apply_timestamp_jitter_seeded_exact_offsets` — compute + `gap_window(seed=7, n=...)` once after implementation, pin the constants. + Guards the draw-order and payload-format contract against refactors. + +### New entry-point tests + +- **Batch invariance**: fixed range, seeded gaps and seeded jitter: one call + covering 60 s vs 60 calls of 1 s; identical written timestamps. (Choose a + `gap_seed` whose window lands inside the range so the test is meaningful — + find it by scanning indices near the chosen epoch time.) +- **Gap spanning call boundaries**: points removed in both adjacent calls. +- **Gap covering entire call window**: zero written, cache still advances, + info log contains the removed-by-gap count. +- **Catch-up**: init, then a single call far in the future; removed spans + match the seeded schedule positions. +- **Unseeded stability**: two consecutive calls share the cached auto-seed + (assert cache key written once and schedule consistent across the calls). +- **Disabled**: `gap_enabled=false` writes all generated points. + +## Step 7: README + +Per the design doc's Breaking Change section: + +- Rewrite the "Gap-filling" headline bullet; add a "Simulated gaps" bullet. +- Add the six parameters to the optional-parameters table. +- Add a "Simulated gaps" section: min/max bound guarantees, on-by-default, + `gap_enabled=false`, seed behavior, upgrade note (existing triggers pick up + gaps; deadman/no-data alert demos will fire). +- Troubleshooting entry: "missing spans in query results" → expected default + behavior, how to disable, how to distinguish from real failures via the + removed-by-gap log line. +- Output schema / timestamp note: mention missing spans. + +## Step 8: Versioning and library metadata + +- `manifest.toml`: `version = "0.3.0"` (behavior-changing default). +- `plugin_library.json` signal_generator entry: append simulated-gap mention + to description, update `last_update`. +- PR notes: call out the default-on behavior change. + +## Step 9: Verify + +1. `pytest influxdata/signal_generator/test_signal_generator.py` — all pass. +2. Manual smoke: `influxdb3 test` or Docker Compose flow per repo README — + default trigger for ~10 minutes, query for missing spans near the 3-minute + cadence; then `gap_enabled=false` run shows continuous data. +3. Grep logs for the removed-by-gap summary line during the smoke run. + +## Sequencing + +Steps 1–3 are pure additions (safe to land in one commit with their unit +tests). Step 4 flips the default and must land together with the Step 6 test +updates and Steps 5, 7, 8 in the same PR — never ship the wiring without the +README/metadata changes. + +## Risks / notes + +- **Performance**: 2 blake2b digests per point worst case; memoization in + `apply_gaps` reduces this to ~2 per gap index per call. Catch-up over days + of downtime at high `points_per_second` is the worst case and stays linear + in point count. +- **Float precision**: gap indices near current epoch are ~10^7; `n * pitch` + products are exact well past 2^53 — no precision concern. +- **Contract freeze**: payload format `f"{seed}:gap:{n}"`, digest size 8, and + draw order (offset, duration) are frozen once released — changing any of + them silently reshuffles every seeded schedule. The determinism-pin test + exists to catch this. +- **Do not** add gap logic to `generate_timestamps` or waveform evaluation; + filtering happens strictly on final jittered timestamps + (design doc Pipeline Semantics). diff --git a/influxdata/signal_generator/README.md b/influxdata/signal_generator/README.md index 2bc8b9f..4d16418 100644 --- a/influxdata/signal_generator/README.md +++ b/influxdata/signal_generator/README.md @@ -10,7 +10,8 @@ The Signal Generator Plugin lets new users produce realistic time-series data fo - **Composable waveforms**: Mix sine, square, triangle, sawtooth, noise, and spike signals by stacking them - **Realistic signals**: Default preset produces a signal centered at 30 with a slow sine trend, Gaussian noise, and occasional large spikes — immediately useful for alert testing - **Timestamp jitter**: Offsets point timestamps by default to simulate sensors that do not sample at perfectly uniform intervals -- **Gap-filling**: Generates continuous, gapless data between executions regardless of trigger schedule +- **Simulated gaps**: Simulates temporary data-source outages by default — bounded windows with no written points, useful for testing deadman alerts and gap handling. Set `gap_enabled=false` for continuous output +- **Catch-up generation**: Generates data for the full span between executions regardless of trigger schedule — the only missing spans are the simulated gaps - **Flexible output**: Configure measurement name, field name, and custom tags per trigger ## Important CLI limitation @@ -44,6 +45,12 @@ This plugin has no required parameters. | `points_per_second`| float | `1.0` | Data point resolution in points per second. Controls how many points are generated per second of elapsed time. | | `jitter_amplitude_seconds` | float | 10% of point interval | Maximum timestamp offset in seconds. Set to `0` to disable timestamp jitter. | | `jitter_seed` | integer | *(none)* | Optional seed for reproducible per-timestamp jitter, mainly useful for tests. | +| `gap_enabled` | boolean | `true` | Enables simulated data-source outage gaps. Set to `false` for continuous output. | +| `gap_min_duration_seconds` | float | `10.0` | Shortest simulated outage duration, in seconds. | +| `gap_max_duration_seconds` | float | `30.0` | Longest simulated outage duration, in seconds. | +| `gap_min_interval_seconds` | float | `120.0` | Shortest time between consecutive outage starts, in seconds. Must be at least `gap_max_duration_seconds`. | +| `gap_max_interval_seconds` | float | `240.0` | Longest time between consecutive outage starts, in seconds. | +| `gap_seed` | integer | *(none)* | Optional seed for reproducible gap timing. | | `target_database` | string | *(none)* | Optional target database. If omitted, writes to the trigger's own database. | ### Timestamp jitter @@ -59,6 +66,28 @@ Each generated timestamp receives an independent random offset from `[-jitter_am Set `jitter_amplitude_seconds=0` to preserve perfectly regular timestamps. The plugin rejects jitter settings that could produce duplicate or reordered timestamps. +### Simulated gaps + +Simulated gaps are enabled by default. The plugin periodically simulates a data-source outage: points whose final (jittered) timestamps fall inside an outage window are not written, leaving realistic missing spans in the output. + +With the default configuration, each outage lasts 10–30 seconds, consecutive outages start 2–4 minutes apart, and about 11% of generated points are removed. The min/max bounds are hard guarantees: + +- Every outage duration lies in `[gap_min_duration_seconds, gap_max_duration_seconds]`. +- The time between consecutive outage starts lies in `[gap_min_interval_seconds, gap_max_interval_seconds]`. +- At least `gap_min_interval_seconds - gap_max_duration_seconds` seconds of continuous data separate consecutive outages (90 seconds at defaults). + +Gap windows are anchored to absolute time, so the same outage pattern is produced whether each scheduled execution writes one point or thousands, and catch-up after downtime contains the same gaps an uninterrupted run would have. When `gap_seed` is set, gap timing is fully reproducible. When omitted, the plugin generates a seed during first-run initialization and caches it, so the schedule stays coherent across executions; a restart that clears the trigger cache starts a new schedule. + +Missing spans are never backfilled: the generation boundary advances even when every point in a window was removed. Runs that removed points log a summary, for example: + +```text +Wrote 585 points to signal.value (600 generated, 15 removed by 1 gap windows) +``` + +Set `gap_enabled=false` to disable simulated gaps entirely. + +**Upgrade note:** existing triggers pick up default gaps on upgrade with no configuration change. Downstream demos that use deadman or no-data alerts will begin firing on the simulated outages — useful for exercising those alerts, but set `gap_enabled=false` if you need continuous data. + ### Waveform types Waveform configuration is supplied as a JSON array. Each object requires a `type` key; all other parameters are optional and fall back to defaults. @@ -350,6 +379,7 @@ Example with `tags={"host": "server01", "region": "us-west"}`: - Nanosecond precision Unix epoch timestamps. Each point's timestamp reflects simulated sample time, not the wall-clock time of the write. - Timestamp jitter is enabled by default, so adjacent timestamps are usually close to, but not exactly on, the nominal cadence grid. Set `jitter_amplitude_seconds=0` for regular intervals. +- Simulated gaps are enabled by default, so output contains periodic missing spans of 10–30 seconds. Set `gap_enabled=false` for continuous data. ### Line protocol examples @@ -428,14 +458,15 @@ Entry point for scheduled triggers. Orchestrates the full execution: parses conf Key operations: -1. Parses waveform, output, resolution, and timestamp jitter configuration from trigger arguments +1. Parses waveform, output, resolution, timestamp jitter, and gap configuration from trigger arguments 2. Reads `last_time` from the trigger-local cache -3. On first run: stores current time and returns without writing (initialization) +3. On first run: stores current time (and the auto-generated gap seed when gaps are enabled and unseeded) and returns without writing (initialization) 4. Generates timestamps in the half-open interval `(last_time, now]` 5. Evaluates the combined waveform at each nominal timestamp 6. Applies timestamp jitter without changing field values -7. Writes all points using `write_sync` with `no_sync=True` -8. Updates `last_time` in the cache +7. Removes points whose final timestamps fall inside simulated gap windows (see [Simulated gaps](#simulated-gaps)) +8. Writes the remaining points using `write_sync` with `no_sync=True` +9. Updates `last_time` in the cache #### Waveform factories @@ -453,6 +484,10 @@ Generates timestamps in the half-open interval `(start, end]`. The interval is e Offsets generated point timestamps after signal evaluation. Values are unchanged; only timestamps are moved. +#### `gap_window(gap_config, n)` / `in_gap(gap_config, t)` / `apply_gaps(points, gap_config)` + +Implements the simulated gap schedule: absolute time is divided into fixed strata, and stratum `n` contains one gap whose start offset and duration are derived from a `blake2b` hash of `(seed, n)`. Every window is a pure function of seed, configuration, and gap index, so results are identical regardless of how many points each scheduled call generates. `apply_gaps` filters final (jittered) timestamps and reports how many points were removed. See `GAP_DESIGN.md` for the full design. + ## Troubleshooting ### Common issues @@ -482,6 +517,12 @@ Look for a log entry containing `"Signal generator initializing"` — this confi echo '[{"type": "sine"}, {"type": "noise"}]' | python3 -m json.tool ``` +#### Issue: Missing spans in query results + +**Cause**: Simulated gaps are enabled by default. The plugin periodically simulates a data-source outage and writes no points inside the outage window. + +**Solution**: This is expected behavior. To confirm a missing span is a simulated gap rather than a real failure, check the logs — runs that removed points report `... (N generated, M removed by K gap windows)`. Set `gap_enabled=false` in the trigger arguments for continuous output. + #### Issue: No points generated (interval too short) **Cause**: The trigger fires more frequently than `1 / points_per_second` seconds, so no timestamps fall in the interval. @@ -510,7 +551,7 @@ influxdb3 query \ **Cause**: Deterministic waveforms (sine, square, triangle, sawtooth) are anchored to absolute Unix time, so they are always at the correct phase for a given wall-clock time. If the signal appears discontinuous, check that the `frequency` parameter is the same before and after the restart. -**Cause of gaps in data**: If the trigger was disabled or InfluxDB was stopped, the cache retains `last_time` from the last successful execution. On restart, the plugin generates all missing points from `last_time` to the current time, filling the gap automatically. +**Cause of gaps in data**: Short gaps (10–30 seconds by default) are simulated outages; see [Simulated gaps](#simulated-gaps). For downtime-related gaps: if the trigger was disabled or InfluxDB was stopped, the cache retains `last_time` from the last successful execution, and on restart the plugin generates all missing points from `last_time` to the current time — the caught-up span contains the same simulated gap windows an uninterrupted run would have. ### Viewing logs diff --git a/influxdata/signal_generator/manifest.toml b/influxdata/signal_generator/manifest.toml index e64ca79..5607dd6 100644 --- a/influxdata/signal_generator/manifest.toml +++ b/influxdata/signal_generator/manifest.toml @@ -2,8 +2,8 @@ manifest_schema_version = "1.2" [plugin] name = "signal_generator" -version = "0.2.0" -description = "Generates configurable waveform-based time-series data on a schedule for demos, testing, dashboards, alerts, and downstream plugin validation without requiring an external data source." +version = "0.3.0" +description = "Generates configurable waveform-based time-series data on a schedule for demos, testing, dashboards, alerts, and downstream plugin validation without requiring an external data source. Simulates data-source outage gaps by default." triggers = ["process_scheduled_call"] homepage = "https://www.influxdata.com/" repository = "https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/signal_generator" diff --git a/influxdata/signal_generator/signal_generator.py b/influxdata/signal_generator/signal_generator.py index 80804db..ec69192 100644 --- a/influxdata/signal_generator/signal_generator.py +++ b/influxdata/signal_generator/signal_generator.py @@ -44,6 +44,42 @@ "description": "Optional integer seed for reproducible per-timestamp jitter.", "required": false }, + { + "name": "gap_enabled", + "example": "false", + "description": "Enables simulated data-source outage gaps. Enabled by default; set to 'false' for continuous output.", + "required": false + }, + { + "name": "gap_min_duration_seconds", + "example": "10", + "description": "Shortest simulated outage duration in seconds. Defaults to 10.", + "required": false + }, + { + "name": "gap_max_duration_seconds", + "example": "30", + "description": "Longest simulated outage duration in seconds. Defaults to 30.", + "required": false + }, + { + "name": "gap_min_interval_seconds", + "example": "120", + "description": "Shortest time between consecutive outage starts in seconds. Must be at least gap_max_duration_seconds. Defaults to 120.", + "required": false + }, + { + "name": "gap_max_interval_seconds", + "example": "240", + "description": "Longest time between consecutive outage starts in seconds. Defaults to 240.", + "required": false + }, + { + "name": "gap_seed", + "example": "123", + "description": "Optional integer seed for reproducible gap timing.", + "required": false + }, { "name": "target_database", "example": "my_db", @@ -181,6 +217,11 @@ def _validate_params(params): DEFAULT_JITTER_INTERVAL_FRACTION = 0.10 MIN_JITTER_GAP_SECONDS = 1e-6 +DEFAULT_GAP_MIN_DURATION_SECONDS = 10.0 +DEFAULT_GAP_MAX_DURATION_SECONDS = 30.0 +DEFAULT_GAP_MIN_INTERVAL_SECONDS = 120.0 +DEFAULT_GAP_MAX_INTERVAL_SECONDS = 240.0 + # --------------------------------------------------------------------------- # Config parsing @@ -306,6 +347,101 @@ def parse_jitter_config(args, points_per_second): return ((amplitude, seed), None) +def _parse_bool(raw): + """Parse a boolean trigger argument. Returns None on error.""" + if isinstance(raw, bool): + return raw + if isinstance(raw, str): + lowered = raw.strip().lower() + if lowered == "true": + return True + if lowered == "false": + return False + return None + + +def parse_gap_config(args): + """Parse simulated gap config from trigger args. + Returns (config_dict, None) on success, or (None, error_message) on error. + All provided arguments are validated even when gaps are disabled.""" + if args is None: + args = {} + + raw_enabled = args.get("gap_enabled") + if raw_enabled is None: + enabled = True + else: + enabled = _parse_bool(raw_enabled) + if enabled is None: + return (None, "Invalid gap config: gap_enabled must be 'true' or 'false'") + + bounds = {} + for name, default in ( + ("gap_min_duration_seconds", DEFAULT_GAP_MIN_DURATION_SECONDS), + ("gap_max_duration_seconds", DEFAULT_GAP_MAX_DURATION_SECONDS), + ("gap_min_interval_seconds", DEFAULT_GAP_MIN_INTERVAL_SECONDS), + ("gap_max_interval_seconds", DEFAULT_GAP_MAX_INTERVAL_SECONDS), + ): + raw = args.get(name) + if raw is None: + bounds[name] = default + continue + try: + value = float(raw) + except (ValueError, TypeError): + return (None, f"Invalid gap config: {name} must be a float") + if not math.isfinite(value): + return (None, f"Invalid gap config: {name} must be finite") + bounds[name] = value + + min_duration = bounds["gap_min_duration_seconds"] + max_duration = bounds["gap_max_duration_seconds"] + min_interval = bounds["gap_min_interval_seconds"] + max_interval = bounds["gap_max_interval_seconds"] + + if min_duration < 0: + return (None, "Invalid gap config: gap_min_duration_seconds must be >= 0") + if max_duration < min_duration: + return (None, "Invalid gap config: gap_max_duration_seconds must be >= gap_min_duration_seconds") + if min_interval <= 0: + return (None, "Invalid gap config: gap_min_interval_seconds must be > 0") + if max_interval < min_interval: + return (None, "Invalid gap config: gap_max_interval_seconds must be >= gap_min_interval_seconds") + if min_interval < max_duration: + return ( + None, + f"Invalid gap config: gap_min_interval_seconds ({min_interval}s) must be >= " + f"gap_max_duration_seconds ({max_duration}s) so outages cannot overlap", + ) + + raw_seed = args.get("gap_seed") + if raw_seed is None: + seed = None + else: + try: + if isinstance(raw_seed, bool): + raise ValueError + if isinstance(raw_seed, float) and not raw_seed.is_integer(): + raise ValueError + seed = int(raw_seed) + except (ValueError, TypeError): + return (None, "Invalid gap config: gap_seed must be an integer") + + return ( + { + "enabled": enabled, + "min_duration": min_duration, + "max_duration": max_duration, + "min_interval": min_interval, + "max_interval": max_interval, + "pitch": (min_interval + max_interval) / 2, + "offset": (max_interval - min_interval) / 4, + "seed": seed, + }, + None, + ) + + # --------------------------------------------------------------------------- # Time series generation # --------------------------------------------------------------------------- @@ -356,6 +492,77 @@ def apply_timestamp_jitter(points, amplitude_seconds, seed=None): return jittered +# --------------------------------------------------------------------------- +# Simulated gaps (see GAP_DESIGN.md) +# --------------------------------------------------------------------------- + +def _rng_for_gap_index(seed, n): + payload = f"{seed}:gap:{n}".encode("ascii") + digest = hashlib.blake2b(payload, digest_size=8).digest() + return random.Random(int.from_bytes(digest, "big")) + + +def gap_window(gap_config, n): + """Gap n's half-open [start, end) window in epoch seconds. + The draw order (start offset, then duration) and the payload format in + _rng_for_gap_index are a reproducibility contract: changing either + reshuffles every seeded schedule.""" + rng = _rng_for_gap_index(gap_config["seed"], n) + start = n * gap_config["pitch"] + rng.uniform(-gap_config["offset"], gap_config["offset"]) + duration = rng.uniform(gap_config["min_duration"], gap_config["max_duration"]) + return (start, start + duration) + + +def in_gap(gap_config, t, window_cache=None): + """Return True if timestamp t falls inside a gap window. + Under the min_interval >= max_duration validation rule, only two gap + indices can contain t, so this is O(1).""" + n_hi = int((t + gap_config["offset"]) // gap_config["pitch"]) + for n in (n_hi, n_hi - 1): + if n < 0: + continue + if window_cache is None: + window = gap_window(gap_config, n) + else: + window = window_cache.get(n) + if window is None: + window = gap_window(gap_config, n) + window_cache[n] = window + start, end = window + if start <= t < end: + return True + return False + + +def apply_gaps(points, gap_config): + """Remove points whose final timestamp falls inside a gap window. + Returns (kept_points, removed_count, gap_window_count) where + gap_window_count is the number of gap windows intersecting the time span + of the given points.""" + if not points: + return ([], 0, 0) + + window_cache = {} + kept = [p for p in points if not in_gap(gap_config, p[0], window_cache)] + removed = len(points) - len(kept) + + timestamps = [t for t, _ in points] + t_min, t_max = min(timestamps), max(timestamps) + pitch = gap_config["pitch"] + offset = gap_config["offset"] + n_lo = int((t_min - offset - gap_config["max_duration"]) // pitch) + n_hi = int((t_max + offset) // pitch) + gap_window_count = 0 + for n in range(max(0, n_lo), n_hi + 1): + window = window_cache.get(n) + if window is None: + window = gap_window(gap_config, n) + start, end = window + if start <= t_max and end > t_min: + gap_window_count += 1 + return (kept, removed, gap_window_count) + + # --------------------------------------------------------------------------- # Batch write helper # --------------------------------------------------------------------------- @@ -426,6 +633,7 @@ def write_points(influxdb3_local, points, measurement, field, tags, target_datab # --------------------------------------------------------------------------- CACHE_KEY = "signal_gen:last_time" +GAP_SEED_CACHE_KEY = "signal_gen:gap_seed" def get_last_time(cache): @@ -436,6 +644,20 @@ def set_last_time(cache, t): cache.put(CACHE_KEY, t) +def resolve_gap_seed(cache, configured_seed): + """Return the gap seed: the configured value, else the cached auto-seed, + else a newly generated auto-seed persisted to the cache. Cache loss means + a new schedule (documented in GAP_DESIGN.md).""" + if configured_seed is not None: + return configured_seed + cached = cache.get(GAP_SEED_CACHE_KEY) + if cached is not None: + return int(cached) + seed = random.getrandbits(63) + cache.put(GAP_SEED_CACHE_KEY, seed) + return seed + + # --------------------------------------------------------------------------- # Entry point # --------------------------------------------------------------------------- @@ -468,10 +690,17 @@ def process_scheduled_call(influxdb3_local, call_time, args=None): influxdb3_local.error(f"[{task_id}] {jitter_error}") return + gap_config, gap_error = parse_gap_config(args) + if gap_config is None: + influxdb3_local.error(f"[{task_id}] {gap_error}") + return + # --- Check cache for last execution time --- last_time = get_last_time(influxdb3_local.cache) if last_time is None: influxdb3_local.info(f"[{task_id}] Signal generator initializing, first data on next execution") + if gap_config["enabled"]: + resolve_gap_seed(influxdb3_local.cache, gap_config["seed"]) set_last_time(influxdb3_local.cache, now) return @@ -488,9 +717,22 @@ def process_scheduled_call(influxdb3_local, call_time, args=None): jitter_amplitude_seconds, jitter_seed = jitter_config points = apply_timestamp_jitter(points, jitter_amplitude_seconds, jitter_seed) + generated = len(points) + removed = 0 + gap_window_count = 0 + if gap_config["enabled"]: + gap_config["seed"] = resolve_gap_seed(influxdb3_local.cache, gap_config["seed"]) + points, removed, gap_window_count = apply_gaps(points, gap_config) + try: written = write_points(influxdb3_local, points, measurement, field, tags, target_database) - influxdb3_local.info(f"[{task_id}] Wrote {written} points to {measurement}.{field}") + if removed: + influxdb3_local.info( + f"[{task_id}] Wrote {written} points to {measurement}.{field} " + f"({generated} generated, {removed} removed by {gap_window_count} gap windows)" + ) + else: + influxdb3_local.info(f"[{task_id}] Wrote {written} points to {measurement}.{field}") except Exception as e: influxdb3_local.error(f"[{task_id}] Batch write failed: {e}") diff --git a/influxdata/signal_generator/test_signal_generator.py b/influxdata/signal_generator/test_signal_generator.py index 1130003..10d0f9c 100644 --- a/influxdata/signal_generator/test_signal_generator.py +++ b/influxdata/signal_generator/test_signal_generator.py @@ -748,6 +748,8 @@ def test_process_scheduled_call_first_run(monkeypatch): # First run: no writes, only cache initialization assert len(mock.written_lines) == 0 assert mock.cache.get("signal_gen:last_time") is not None + # Gaps are enabled by default and unseeded, so init stores an auto-seed + assert mock.cache.get("signal_gen:gap_seed") is not None def test_process_scheduled_call_second_run(monkeypatch): @@ -755,12 +757,13 @@ def test_process_scheduled_call_second_run(monkeypatch): monkeypatch.setattr(signal_generator, "LineBuilder", MockLineBuilder) from signal_generator import process_scheduled_call mock = MockInfluxDB3Local() + args = {"gap_enabled": "false"} # Simulate first run first_time = datetime(2026, 4, 10, 12, 0, 0) - process_scheduled_call(mock, first_time) + process_scheduled_call(mock, first_time, args) # Second run 5 seconds later — should produce 5 points at 1 pps second_time = datetime(2026, 4, 10, 12, 0, 5) - process_scheduled_call(mock, second_time) + process_scheduled_call(mock, second_time, args) assert count_written_points(mock) == 5 @@ -769,7 +772,7 @@ def test_process_scheduled_call_custom_pps(monkeypatch): monkeypatch.setattr(signal_generator, "LineBuilder", MockLineBuilder) from signal_generator import process_scheduled_call mock = MockInfluxDB3Local() - args = {"points_per_second": "2"} + args = {"points_per_second": "2", "gap_enabled": "false"} first_time = datetime(2026, 4, 10, 12, 0, 0) process_scheduled_call(mock, first_time, args) # Second run 5 seconds later — 2 pps * 5s = 10 points @@ -783,7 +786,7 @@ def test_process_scheduled_call_target_database(monkeypatch): monkeypatch.setattr(signal_generator, "LineBuilder", MockLineBuilder) from signal_generator import process_scheduled_call mock = MockInfluxDB3Local() - args = {"target_database": "other_db"} + args = {"target_database": "other_db", "gap_enabled": "false"} first_time = datetime(2026, 4, 10, 12, 0, 0) process_scheduled_call(mock, first_time, args) second_time = datetime(2026, 4, 10, 12, 0, 5) @@ -831,6 +834,7 @@ def test_process_scheduled_call_jitter_zero_keeps_regular_timestamps(monkeypatch args = { "waveforms": '[{"type":"constant","value":1.0}]', "jitter_amplitude_seconds": "0", + "gap_enabled": "false", } first_time = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) process_scheduled_call(mock, first_time, args) @@ -853,6 +857,7 @@ def test_process_scheduled_call_seeded_jitter_offsets_timestamps(monkeypatch): "waveforms": '[{"type":"constant","value":1.0}]', "jitter_amplitude_seconds": "0.1", "jitter_seed": "7", + "gap_enabled": "false", } first_time = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) process_scheduled_call(mock, first_time, args) @@ -874,6 +879,7 @@ def test_process_scheduled_call_seeded_jitter_varies_across_one_point_calls(monk "waveforms": '[{"type":"constant","value":1.0}]', "jitter_amplitude_seconds": "0.1", "jitter_seed": "7", + "gap_enabled": "false", } first_time = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) process_scheduled_call(mock, first_time, args) @@ -910,7 +916,7 @@ def test_process_scheduled_call_default_jitter_offsets_timestamps(monkeypatch): monkeypatch.setattr(signal_generator, "LineBuilder", MockLineBuilder) from signal_generator import process_scheduled_call mock = MockInfluxDB3Local() - args = {"waveforms": '[{"type":"constant","value":1.0}]'} + args = {"waveforms": '[{"type":"constant","value":1.0}]', "gap_enabled": "false"} first_time = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) process_scheduled_call(mock, first_time, args) second_time = datetime(2026, 4, 10, 12, 0, 3, tzinfo=timezone.utc) @@ -928,3 +934,449 @@ def test_process_scheduled_call_default_jitter_offsets_timestamps(monkeypatch): earlier < later for earlier, later in zip(timestamps, timestamps[1:]) ) + + +# --------------------------------------------------------------------------- +# Gap config parsing tests +# --------------------------------------------------------------------------- + +def make_gap_config(seed=123, **args): + """Build a parsed gap config for schedule-function tests.""" + from signal_generator import parse_gap_config + config, error = parse_gap_config({k: str(v) for k, v in args.items()}) + assert error is None, error + config["seed"] = seed + return config + + +def test_parse_gap_config_defaults(): + from signal_generator import parse_gap_config + config, error = parse_gap_config(None) + assert error is None + assert config["enabled"] is True + assert config["min_duration"] == 10.0 + assert config["max_duration"] == 30.0 + assert config["min_interval"] == 120.0 + assert config["max_interval"] == 240.0 + assert config["pitch"] == 180.0 + assert config["offset"] == 30.0 + assert config["seed"] is None + + +def test_parse_gap_config_custom_values(): + from signal_generator import parse_gap_config + config, error = parse_gap_config({ + "gap_min_duration_seconds": "5", + "gap_max_duration_seconds": "15", + "gap_min_interval_seconds": "60", + "gap_max_interval_seconds": "100", + "gap_seed": "42", + }) + assert error is None + assert config["min_duration"] == 5.0 + assert config["max_duration"] == 15.0 + assert config["min_interval"] == 60.0 + assert config["max_interval"] == 100.0 + assert config["pitch"] == 80.0 + assert config["offset"] == 10.0 + assert config["seed"] == 42 + + +def test_parse_gap_config_enabled_false(): + from signal_generator import parse_gap_config + config, error = parse_gap_config({"gap_enabled": "false"}) + assert error is None + assert config["enabled"] is False + + +def test_parse_gap_config_enabled_case_insensitive(): + from signal_generator import parse_gap_config + config, error = parse_gap_config({"gap_enabled": "TRUE"}) + assert error is None + assert config["enabled"] is True + config, error = parse_gap_config({"gap_enabled": "False"}) + assert error is None + assert config["enabled"] is False + + +def test_parse_gap_config_enabled_garbage(): + from signal_generator import parse_gap_config + config, error = parse_gap_config({"gap_enabled": "yes"}) + assert config is None + assert "Invalid gap config" in error + + +def test_parse_gap_config_non_numeric_duration(): + from signal_generator import parse_gap_config + config, error = parse_gap_config({"gap_min_duration_seconds": "abc"}) + assert config is None + assert "Invalid gap config" in error + + +def test_parse_gap_config_nan_rejected(): + from signal_generator import parse_gap_config + config, error = parse_gap_config({"gap_max_interval_seconds": "nan"}) + assert config is None + assert "Invalid gap config" in error + + +def test_parse_gap_config_inf_rejected(): + from signal_generator import parse_gap_config + config, error = parse_gap_config({"gap_max_duration_seconds": "inf"}) + assert config is None + assert "Invalid gap config" in error + + +def test_parse_gap_config_negative_min_duration(): + from signal_generator import parse_gap_config + config, error = parse_gap_config({"gap_min_duration_seconds": "-1"}) + assert config is None + assert "Invalid gap config" in error + + +def test_parse_gap_config_max_duration_below_min(): + from signal_generator import parse_gap_config + config, error = parse_gap_config({ + "gap_min_duration_seconds": "20", + "gap_max_duration_seconds": "10", + }) + assert config is None + assert "Invalid gap config" in error + + +def test_parse_gap_config_zero_min_interval(): + from signal_generator import parse_gap_config + config, error = parse_gap_config({ + "gap_min_interval_seconds": "0", + "gap_max_interval_seconds": "10", + }) + assert config is None + assert "Invalid gap config" in error + + +def test_parse_gap_config_max_interval_below_min(): + from signal_generator import parse_gap_config + config, error = parse_gap_config({ + "gap_min_interval_seconds": "200", + "gap_max_interval_seconds": "100", + }) + assert config is None + assert "Invalid gap config" in error + + +def test_parse_gap_config_min_interval_below_max_duration(): + from signal_generator import parse_gap_config + config, error = parse_gap_config({ + "gap_max_duration_seconds": "30", + "gap_min_interval_seconds": "25", + "gap_max_interval_seconds": "240", + }) + assert config is None + assert "Invalid gap config" in error + + +def test_parse_gap_config_min_interval_equal_max_duration_valid(): + from signal_generator import parse_gap_config + config, error = parse_gap_config({ + "gap_max_duration_seconds": "30", + "gap_min_interval_seconds": "30", + "gap_max_interval_seconds": "90", + }) + assert error is None + assert config["min_interval"] == 30.0 + + +def test_parse_gap_config_non_integer_seed(): + from signal_generator import parse_gap_config + config, error = parse_gap_config({"gap_seed": "1.5"}) + assert config is None + assert "Invalid gap config" in error + + +def test_parse_gap_config_disabled_still_validates(): + from signal_generator import parse_gap_config + config, error = parse_gap_config({ + "gap_enabled": "false", + "gap_min_duration_seconds": "abc", + }) + assert config is None + assert "Invalid gap config" in error + + +# --------------------------------------------------------------------------- +# Gap schedule function tests +# --------------------------------------------------------------------------- + +def test_gap_window_bounded_spacing_and_no_overlap(): + from signal_generator import gap_window + config = make_gap_config(seed=123) + starts = [] + prev_end = None + for n in range(5000): + start, end = gap_window(config, n) + assert 10.0 <= end - start <= 30.0 + if prev_end is not None: + assert start >= prev_end + starts.append(start) + prev_end = end + spacings = [b - a for a, b in zip(starts, starts[1:])] + assert min(spacings) >= 120.0 + assert max(spacings) <= 240.0 + + +def test_gap_window_no_overlap_at_equality_boundary(): + # min_interval == max_duration: gaps may touch but never overlap + from signal_generator import gap_window + config = make_gap_config( + seed=7, + gap_min_duration_seconds=5, + gap_max_duration_seconds=30, + gap_min_interval_seconds=30, + gap_max_interval_seconds=90, + ) + prev_end = None + for n in range(5000): + start, end = gap_window(config, n) + if prev_end is not None: + assert start >= prev_end - 1e-9 + prev_end = end + + +def test_in_gap_matches_brute_force(): + import random as pyrandom + from signal_generator import gap_window, in_gap + config = make_gap_config(seed=123) + rng = pyrandom.Random(9) + for _ in range(5000): + t = rng.uniform(0, 5000 * config["pitch"]) + n_center = int(t // config["pitch"]) + brute = any( + gap_window(config, n)[0] <= t < gap_window(config, n)[1] + for n in range(max(0, n_center - 3), n_center + 4) + ) + assert in_gap(config, t) == brute + + +def test_gap_window_reproducible_across_seeds(): + from signal_generator import gap_window + config_a = make_gap_config(seed=123) + config_b = make_gap_config(seed=123) + config_c = make_gap_config(seed=124) + windows_a = [gap_window(config_a, n) for n in range(100)] + windows_b = [gap_window(config_b, n) for n in range(100)] + windows_c = [gap_window(config_c, n) for n in range(100)] + assert windows_a == windows_b + assert windows_a != windows_c + + +def test_gap_window_pinned_values(): + # Freezes the hash payload format and draw order (offset, then duration). + # If this fails, every seeded gap schedule in the wild has changed. + from signal_generator import gap_window + config = make_gap_config(seed=7) + start, end = gap_window(config, 1000) + assert abs(start - 180011.7885872335) < 1e-9 + assert abs(end - 180037.7907425121) < 1e-9 + + +def test_apply_gaps_empty_points(): + from signal_generator import apply_gaps + config = make_gap_config(seed=123) + assert apply_gaps([], config) == ([], 0, 0) + + +def test_apply_gaps_half_open_boundaries(): + from signal_generator import apply_gaps, gap_window + config = make_gap_config(seed=123) + start, end = gap_window(config, 50) + points = [(start, 1.0), ((start + end) / 2, 2.0), (end, 3.0)] + kept, removed, window_count = apply_gaps(points, config) + # start and midpoint are inside [start, end); the point exactly at end is kept + assert kept == [(end, 3.0)] + assert removed == 2 + assert window_count >= 1 + + +def test_apply_gaps_window_count(): + from signal_generator import apply_gaps, gap_window + config = make_gap_config(seed=123) + t0 = 1775822400.0 + points = [(t0 + i, 1.0) for i in range(1, 601)] + _, _, window_count = apply_gaps(points, config) + expected = 0 + n_center = int(t0 // config["pitch"]) + for n in range(max(0, n_center - 3), n_center + 8): + start, end = gap_window(config, n) + if start <= t0 + 600 and end > t0 + 1: + expected += 1 + assert window_count == expected + assert window_count >= 2 # a 600s span always intersects multiple windows + + +# --------------------------------------------------------------------------- +# Entry point gap tests +# --------------------------------------------------------------------------- + +GAP_TEST_ARGS = { + "waveforms": '[{"type":"constant","value":1.0}]', + "jitter_amplitude_seconds": "0", + "gap_seed": "123", +} + + +def test_process_scheduled_call_default_gaps_remove_points(monkeypatch): + import signal_generator + monkeypatch.setattr(signal_generator, "LineBuilder", MockLineBuilder) + from signal_generator import process_scheduled_call + mock = MockInfluxDB3Local() + first_time = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + process_scheduled_call(mock, first_time, GAP_TEST_ARGS) + second_time = datetime(2026, 4, 10, 12, 10, 0, tzinfo=timezone.utc) + process_scheduled_call(mock, second_time, GAP_TEST_ARGS) + written = count_written_points(mock) + # A 600s span always contains at least one full gap window (10-30s) + assert 0 < written < 600 + assert any("removed by" in log for log in mock.logs["info"]) + + +def test_process_scheduled_call_gap_batch_invariance(monkeypatch): + import signal_generator + monkeypatch.setattr(signal_generator, "LineBuilder", MockLineBuilder) + from signal_generator import process_scheduled_call + first_time = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + + mock_one = MockInfluxDB3Local() + process_scheduled_call(mock_one, first_time, GAP_TEST_ARGS) + process_scheduled_call( + mock_one, datetime(2026, 4, 10, 12, 10, 0, tzinfo=timezone.utc), GAP_TEST_ARGS + ) + + mock_many = MockInfluxDB3Local() + process_scheduled_call(mock_many, first_time, GAP_TEST_ARGS) + for seconds in range(10, 601, 10): + call_time = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + call_time = datetime.fromtimestamp( + call_time.timestamp() + seconds, tz=timezone.utc + ) + process_scheduled_call(mock_many, call_time, GAP_TEST_ARGS) + + assert written_timestamps_ns(mock_one) == written_timestamps_ns(mock_many) + assert 0 < count_written_points(mock_one) < 600 + + +def test_process_scheduled_call_gap_spans_call_boundary(monkeypatch): + import math + import signal_generator + monkeypatch.setattr(signal_generator, "LineBuilder", MockLineBuilder) + from signal_generator import gap_window, process_scheduled_call + config = make_gap_config(seed=123) + t0 = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc).timestamp() + + # Find a gap window comfortably inside (t0+60, t0+540) + n = int(t0 // config["pitch"]) + while True: + start, end = gap_window(config, n) + if start > t0 + 60 and end < t0 + 540: + break + n += 1 + boundary = int(start + (end - start) / 2) # call boundary inside the gap + + mock = MockInfluxDB3Local() + process_scheduled_call(mock, datetime.fromtimestamp(t0, tz=timezone.utc), GAP_TEST_ARGS) + + process_scheduled_call( + mock, datetime.fromtimestamp(boundary, tz=timezone.utc), GAP_TEST_ARGS + ) + written_first = count_written_points(mock) + generated_first = boundary - int(t0) + assert written_first < generated_first # gap removed points before the boundary + + process_scheduled_call( + mock, datetime.fromtimestamp(t0 + 600, tz=timezone.utc), GAP_TEST_ARGS + ) + written_second = count_written_points(mock) - written_first + generated_second = int(t0 + 600) - boundary + assert written_second < generated_second # and after the boundary + + # No written timestamp falls inside the gap window + for ts_ns in written_timestamps_ns(mock): + ts = ts_ns / 1_000_000_000 + assert not (start <= ts < end) + + +def test_process_scheduled_call_gap_covers_entire_call(monkeypatch): + import math + import signal_generator + monkeypatch.setattr(signal_generator, "LineBuilder", MockLineBuilder) + from signal_generator import gap_window, process_scheduled_call + config = make_gap_config(seed=123) + t0 = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc).timestamp() + n = int(t0 // config["pitch"]) + 1 + start, end = gap_window(config, n) + + init_time = math.ceil(start) + call_time = init_time + 4 # entire (init_time, call_time] sits inside the gap + assert call_time < end + + mock = MockInfluxDB3Local() + process_scheduled_call(mock, datetime.fromtimestamp(init_time, tz=timezone.utc), GAP_TEST_ARGS) + process_scheduled_call(mock, datetime.fromtimestamp(call_time, tz=timezone.utc), GAP_TEST_ARGS) + + assert count_written_points(mock) == 0 + assert any("removed by" in log for log in mock.logs["info"]) + # Generation boundary still advances: the missing span is never backfilled + assert mock.cache.get("signal_gen:last_time") == call_time + + +def test_process_scheduled_call_gap_catch_up_matches_schedule(monkeypatch): + import signal_generator + monkeypatch.setattr(signal_generator, "LineBuilder", MockLineBuilder) + from signal_generator import in_gap, process_scheduled_call + config = make_gap_config(seed=123) + t0 = int(datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc).timestamp()) + + mock = MockInfluxDB3Local() + process_scheduled_call(mock, datetime.fromtimestamp(t0, tz=timezone.utc), GAP_TEST_ARGS) + # Single catch-up call one hour later + process_scheduled_call( + mock, datetime.fromtimestamp(t0 + 3600, tz=timezone.utc), GAP_TEST_ARGS + ) + + expected = [ + t * 1_000_000_000 + for t in range(t0 + 1, t0 + 3601) + if not in_gap(config, float(t)) + ] + assert written_timestamps_ns(mock) == expected + assert len(expected) < 3600 + + +def test_process_scheduled_call_unseeded_gap_seed_is_stable(monkeypatch): + import signal_generator + monkeypatch.setattr(signal_generator, "LineBuilder", MockLineBuilder) + from signal_generator import process_scheduled_call + mock = MockInfluxDB3Local() + args = {"waveforms": '[{"type":"constant","value":1.0}]'} + first_time = datetime(2026, 4, 10, 12, 0, 0, tzinfo=timezone.utc) + process_scheduled_call(mock, first_time, args) + seed_after_init = mock.cache.get("signal_gen:gap_seed") + assert isinstance(seed_after_init, int) + for seconds in (10, 20, 30): + call_time = datetime.fromtimestamp( + first_time.timestamp() + seconds, tz=timezone.utc + ) + process_scheduled_call(mock, call_time, args) + assert mock.cache.get("signal_gen:gap_seed") == seed_after_init + + +def test_process_scheduled_call_gap_config_error_writes_nothing(monkeypatch): + import signal_generator + monkeypatch.setattr(signal_generator, "LineBuilder", MockLineBuilder) + from signal_generator import process_scheduled_call + mock = MockInfluxDB3Local() + args = {"gap_min_duration_seconds": "abc"} + call_time = datetime(2026, 4, 10, 12, 0, 0) + process_scheduled_call(mock, call_time, args) + assert len(mock.written_lines) == 0 + assert mock.cache.get("signal_gen:last_time") is None + assert any("Invalid gap config" in log for log in mock.logs["error"]) From 1258cac58b8085b7a6129e18350869f9a2e4f250 Mon Sep 17 00:00:00 2001 From: caterryan Date: Fri, 10 Jul 2026 11:58:56 -0500 Subject: [PATCH 2/3] Delete influxdata/signal_generator/GAP_DESIGN.md --- influxdata/signal_generator/GAP_DESIGN.md | 562 ---------------------- 1 file changed, 562 deletions(-) delete mode 100644 influxdata/signal_generator/GAP_DESIGN.md diff --git a/influxdata/signal_generator/GAP_DESIGN.md b/influxdata/signal_generator/GAP_DESIGN.md deleted file mode 100644 index 06dfcdd..0000000 --- a/influxdata/signal_generator/GAP_DESIGN.md +++ /dev/null @@ -1,562 +0,0 @@ -# Data Gap Design - -## Purpose - -Add simulated temporary disconnections to the signal generator so generated data -contains realistic missing spans. A gap represents a data source outage: points -inside the outage window are absent from output. - -Gaps are not a waveform type. They affect whether points are written, not the -signal value calculation. - -## Important Context - -This design depends on existing Signal Generator behavior documented in: - -- `README.md`: The plugin currently advertises gap-filling behavior, meaning it - generates continuous data between scheduled executions. Simulated gaps are an - intentional exception to that behavior and must be documented as such (see - [Breaking Change and Documentation Impact](#breaking-change-and-documentation-impact)). -- `README.md`: The first scheduled execution initializes cache state and writes - no data. Gap scheduling preserves this startup behavior. -- `README.md`: `points_per_second` controls generated timestamp resolution. - Gap behavior must work equally well when each scheduled call writes many - points or only one point. -- `README.md` (Timestamp jitter section): Timestamp jitter is configured as - flat trigger arguments, is enabled by default, and uses a seed for - reproducibility. Gap configuration follows the same style. Jitter applies - after nominal timestamp generation; simulated gaps apply after jitter so - filtering uses final recorded timestamps. -- `signal_generator.py`: Cache state currently stores `signal_gen:last_time` as - the unjittered generation boundary. Gaps must not change that meaning. -- `signal_generator.py`: `_rng_for_timestamp` already derives per-timestamp - jitter from a `blake2b` digest of `(seed, timestamp)`, so draws are a pure - function of their inputs. The gap schedule uses the same primitive, keyed by - gap index instead of timestamp. -- `signal_generator.py`: Trigger arguments are parsed at scheduled-call runtime, - so invalid gap configuration must be reported at runtime and should produce no - writes for that call. -- `signal_generator.py`: Waveforms are composed independently from output - configuration. Gaps stay outside waveform configuration and are not a - waveform type. - -## User Model - -Users configure explicit bounds on how long outages last and how often they -begin. Both are randomized within those bounds, so output does not look -mechanically periodic. - -Gaps are enabled by default. This is a breaking change for existing triggers; -see [Breaking Change and Documentation Impact](#breaking-change-and-documentation-impact) -for the required mitigations. Users who need continuous output can disable -gaps with `gap_enabled=false`. - -## Configuration Shape - -Gaps are configured with flat trigger arguments: - -| Argument | Type | Default | Description | -|----------|------|---------|-------------| -| `gap_enabled` | boolean string | `true` | Enables or disables simulated gaps. | -| `gap_min_duration_seconds` | float string | `10.0` | Shortest outage duration, in seconds. | -| `gap_max_duration_seconds` | float string | `30.0` | Longest outage duration, in seconds. | -| `gap_min_interval_seconds` | float string | `120.0` | Shortest time between consecutive gap starts, in seconds. | -| `gap_max_interval_seconds` | float string | `240.0` | Longest time between consecutive gap starts, in seconds. | -| `gap_seed` | integer string | none | Optional seed for reproducible gap timing. | - -Example: - -```text -gap_min_duration_seconds=10,gap_max_duration_seconds=30,gap_min_interval_seconds=120,gap_max_interval_seconds=240,gap_seed=123 -``` - -The min/max bounds are hard guarantees, not soft targets: - -- every gap's duration lies in - `[gap_min_duration_seconds, gap_max_duration_seconds]`; -- the time between consecutive gap starts lies in - `[gap_min_interval_seconds, gap_max_interval_seconds]`. - -Internally the schedule derives a grid pitch and a per-gap start offset from -the interval bounds: - -```text -pitch = (gap_min_interval_seconds + gap_max_interval_seconds) / 2 -offset = (gap_max_interval_seconds - gap_min_interval_seconds) / 4 - -start(n) = n * pitch + uniform(-offset, +offset) -duration(n) = uniform(gap_min_duration_seconds, gap_max_duration_seconds) -``` - -The offset half-range is a quarter of the interval span, not half, because the -spacing between consecutive starts is `pitch` plus the *difference of two -independent offsets*: that difference spans twice the offset half-range, so a -quarter-span offset makes consecutive spacing land exactly in -`[gap_min_interval_seconds, gap_max_interval_seconds]`. See -[Gap Schedule Algorithm](#gap-schedule-algorithm). - -Out-of-range values are rejected by validation and logged as errors; they are -never silently corrected or clamped. - -## Default Behavior - -Default gaps should be visible but not dominant: - -```text -gap_min_duration_seconds = 10.0 -gap_max_duration_seconds = 30.0 -gap_min_interval_seconds = 120.0 -gap_max_interval_seconds = 240.0 -``` - -This yields outage durations from 10 to 30 seconds, and gap starts spaced 2 to -4 minutes apart (usually close to 3 minutes). Expected data loss is -`mean duration / mean interval` = 20/180 ≈ 11% of points, and at least -`gap_min_interval_seconds - gap_max_duration_seconds` = 90 seconds of -continuous data are guaranteed between consecutive gaps. At the default -`points_per_second=1.0`, a typical gap removes about 20 points. - -The defaults are intentionally moderate: users should notice missing spans in -query results and dashboards without losing most generated data. - -## Disabling Gaps - -Set `gap_enabled=false` to preserve continuous output. This is the only -supported disable mechanism. - -Setting both duration bounds to `0` is valid and produces empty gap windows -(effectively no gaps), but `gap_enabled=false` is the explicit, documented -form. - -## Gap Schedule Algorithm - -The gap schedule is a **jittered grid**: absolute time is divided into strata -of width `pitch`, anchored at Unix epoch 0, and stratum `n` contains exactly -one gap whose start offset and duration are derived from a hash of -`(seed, n)`. Every gap window is a pure function of -`(gap_seed, configuration, gap_index)` — no sequential state, no replay from -an anchor. - -This is the same construction the plugin already uses for per-timestamp jitter -(`_rng_for_timestamp`), keyed by gap index instead of timestamp: - -```python -def _rng_for_gap_index(seed, n): - payload = f"{seed}:gap:{n}".encode("ascii") - digest = hashlib.blake2b(payload, digest_size=8).digest() - return random.Random(int.from_bytes(digest, "big")) - -def gap_window(seed, n): - """Gap n's half-open [start, end) window, in epoch seconds. - Draw order is fixed: start offset first, then duration.""" - rng = _rng_for_gap_index(seed, n) - start = n * pitch + rng.uniform(-offset, +offset) - duration = rng.uniform(min_duration, max_duration) - return (start, start + duration) -``` - -The `"gap"` tag in the payload domain-separates gap draws from any other -hash-derived draws using the same numeric key space. The draw order (offset, -then duration) is part of the reproducibility contract: reordering the draws -changes every schedule produced by a given seed. - -### Membership test - -Filtering asks, per point: is final timestamp `t` inside any gap window? Under -the validation constraint below, at most two gap indices can contain `t`, so -membership is O(1): - -```python -def in_gap(seed, t): - n_hi = int((t + offset) // pitch) - for n in (n_hi, n_hi - 1): - if n < 0: - continue - start, end = gap_window(seed, n) - if start <= t < end: - return True - return False -``` - -### Properties - -These follow from the construction and were verified by simulation over 300k -gap windows and 200k random membership queries against brute force: - -- **Batch-size invariance**: membership depends only on the point's final - timestamp, never on scheduled-call boundaries or how many points a call - generates. Filtering one point per call and filtering thousands per call - produce identical output. In particular, a trigger generating one point per - execution does *not* roll a fresh gap decision each execution — the known - failure mode of per-call probability draws, where outage frequency scales - with trigger frequency instead of simulated time. -- **Reproducibility**: same seed and configuration produce bit-identical gap - windows, across calls, restarts, and batch sizes. -- **No persistent state**: nothing about the schedule is stored. The only new - cache entry is the auto-generated seed when `gap_seed` is omitted (see - [Cache Semantics](#cache-semantics)). -- **Bounded spacing**: consecutive gap starts are spaced - `pitch + (offset_next - offset_prev)`, and each offset is at most a quarter - of the interval span, so spacing always lies in - `[gap_min_interval_seconds, gap_max_interval_seconds]` — the bounds the - argument names promise. Verified: 300k consecutive spacings at defaults all - fell within [120, 240], mean 180. -- **Non-overlap**: gaps cannot overlap when the validation constraint - `gap_min_interval_seconds >= gap_max_duration_seconds` holds, and at least - `gap_min_interval_seconds - gap_max_duration_seconds` seconds of data are - guaranteed between consecutive gaps (90 seconds at defaults). -- **Spacing distribution**: within its [min, max] bounds, spacing follows a - triangular distribution peaked at the midpoint, not a uniform one, and every - gap stays within `±offset` of its own grid position, so there is no - cumulative drift — the schedule is quasi-periodic rather than a renewal - process with independent inter-arrival times. The min/max bounds are exact; - only the shape inside them is not uniform. This trade-off is deliberate; see - [Alternative Schedule Algorithms Considered](#alternative-schedule-algorithms-considered). -- **Configuration is part of the schedule identity**: changing any interval or - duration bound re-derives the entire schedule. An in-progress gap does not - survive a configuration change. - -### Prior art - -This lattice-plus-keyed-hash construction is standard for reproducible, -random-access randomness over an unbounded domain: jittered stratified -sampling (one sample per stratum, offset from the stratum anchor), stateless -procedural world generation (cell coordinates hashed to a per-cell seed, with -a proven bounded influence radius that makes constant-neighborhood queries -correct), Pixar's correlated multi-jittered sampling (any sample computable -directly from its index), and counter-based RNGs such as Philox (the Nth draw -is a stateless function of key and counter). Chaos-testing tools document the -anti-patterns this design avoids: per-request probability draws couple fault -frequency to request frequency, and re-seeding a sequential RNG per evaluation -collapses to a constant decision. - -## Pipeline Semantics - -Gaps are applied after signal generation and timestamp jitter: - -```text -nominal timestamps -> signal values -> jittered timestamps -> gap filtering -> write -``` - -This ordering matters. Jitter determines the final recorded timestamp. Gap -filtering then removes any point whose final timestamp lands inside a simulated -outage window. - -The result is a source-disconnection model: points inside the outage do not -exist in output. - -## Gap Window Semantics - -A gap is a half-open time window: - -```text -[gap_start, gap_end) -``` - -Any point with a final timestamp inside that window is removed. Points exactly -at `gap_end` are kept. This mirrors the existing timestamp generation convention -that avoids duplicate boundary handling across adjacent windows. - -The same seed controls both the start-offset and duration draws, preserving a -single reproducibility knob. - -Example, at defaults with some seed (`pitch` = 180, `offset` = 30; gap indices -are large in practice because the grid is anchored at epoch — small indices -shown for readability): - -```text -gap_min_interval_seconds = 120 -gap_max_interval_seconds = 240 -gap_min_duration_seconds = 10 -gap_max_duration_seconds = 30 - -gap n nominal start n*180 actual start duration -1000 180000 179973 27 -1001 180180 180198 13 -1002 180360 180351 22 -``` - -Nominal positions are a fixed grid; each gap's start shifts by at most -`offset` around its own grid position. Consecutive starts here are spaced -225s and 153s apart — always within [120, 240]. - -## Randomness - -Gap randomness is based on absolute simulated time, not on write-cycle -boundaries. - -This is required for low-volume configurations such as one point per scheduled -call. A per-call random decision would make gaps correlate with trigger -executions and could produce "all or nothing" behavior each cycle. -Absolute-time gap windows produce the same outage pattern whether points are -written one at a time or batched many per call. - -When `gap_seed` is set, the gap schedule is reproducible: the same seed and -configuration produce the same gap windows regardless of scheduled-call batch -size, restarts, or catch-up after downtime. - -When `gap_seed` is omitted, the plugin generates a random integer seed during -the first-run initialization and stores it in the cache (see -[Cache Semantics](#cache-semantics)). All subsequent calls read the cached -seed and take the same code path as an explicit seed, so an outage that begins -in one execution continues coherently in later executions until its duration -ends. If the cache is lost (for example, across a server restart that clears -the trigger cache), a new seed is generated and the schedule changes: an -in-progress gap may end early or new gaps may appear at different times. This -is acceptable for demo data; users who need schedule stability across restarts -should set `gap_seed` explicitly. - -## Cache Semantics - -The cache continues to represent generated-time progress, not written-time -progress. `signal_gen:last_time` keeps its existing meaning: the unjittered -generation boundary. - -One new cache key is added: - -- `signal_gen:gap_seed`: the auto-generated gap seed, written once during - first-run initialization when `gap_seed` is not configured. Never written - when `gap_seed` is configured. - -When points are removed by gaps, the plugin still advances its generation -boundary. Missing data must remain missing; it must not be backfilled during -the next scheduled call. - -Catch-up after downtime follows the same rule: when a call generates a long -range of points (because `last_time` is far in the past), the absolute-time -gap schedule applies to that entire range, so backfilled spans contain the -same gaps they would have contained if the plugin had been running. - -## Validation - -Valid gap configuration: - -- `gap_enabled` parses as a boolean when provided. -- `gap_min_duration_seconds` parses as a finite float with value `>= 0`. -- `gap_max_duration_seconds` parses as a finite float with value - `>= gap_min_duration_seconds`. -- `gap_min_interval_seconds` parses as a finite float with value `> 0`. -- `gap_max_interval_seconds` parses as a finite float with value - `>= gap_min_interval_seconds`. -- `gap_min_interval_seconds >= gap_max_duration_seconds`. -- `gap_seed` is omitted or parses as an integer. - -Finiteness matters: `inf` passes a bare `>= 0` check and would produce a -permanent outage. The existing `points_per_second` and jitter validation -already require finite values; gap validation matches. - -`gap_min_interval_seconds >= gap_max_duration_seconds` is the load-bearing -rule, and reads as its own explanation: the shortest time between two outage -starts must fit the longest outage. It guarantees: - -- gap windows never overlap (worst case: gap `n` ends at - `n*pitch + offset + max_duration`, gap `n+1` starts at - `(n+1)*pitch - offset`, and the difference is - `pitch - 2*offset - max_duration = min_interval - max_duration >= 0`); -- the membership test only ever needs to inspect two candidate indices; -- a minimum of `gap_min_interval_seconds - gap_max_duration_seconds` seconds - of continuous data between consecutive gaps. - -Defaults satisfy it: `120 >= 30`. At exact equality gaps may touch -back-to-back (the half-open windows still never overlap or double-remove a -boundary point); simulation at the equality boundary over 300k gaps confirmed -no overlap. - -Invalid configuration logs an error and writes nothing for that scheduled -call. Runtime validation is the enforcement point because trigger arguments -are not validated at trigger-creation time. Values are rejected, not clamped. - -## Interaction With Jitter - -Gaps operate on jittered timestamps. A point near a gap boundary can be moved -into or out of the gap by timestamp jitter. - -This is desirable: from the user's perspective, gaps describe missing output -time ranges, and jitter describes the final recorded timestamp. Filtering after -jitter makes the final written timeline match the configured outage windows. - -`gap_seed` and `jitter_seed` are independent knobs, and the gap draws are -domain-separated from jitter draws by the `"gap"` payload tag, so identical -seed values cannot produce correlated draws. - -## Interaction With Waveforms - -Waveforms still define what values would have existed at each generated time. -Gaps only remove points from output. - -Deterministic waveforms remain anchored to absolute time. Stochastic waveforms -continue to use their own seed behavior. Gap configuration should not alter -waveform values outside the missing windows. - -## Observability - -Normal successful runs should report how many points were written. When gaps -remove points, logs should make that visible without becoming noisy. - -Useful summary fields: - -- generated point count -- removed-by-gap point count -- written point count -- number of gap windows intersecting the call window - -This helps users distinguish "no points because interval too short" from -"points generated but all removed by simulated gap." - -## Breaking Change and Documentation Impact - -Enabling gaps by default changes the plugin's advertised core behavior. -Shipping this requires, in the same change: - -- Rewrite the README "Gap-filling" headline bullet: continuous generation - between executions still holds, but simulated outage gaps are now present by - default and are a feature, not a bug. -- Add a README troubleshooting entry for "missing spans in query results" - explaining simulated gaps and `gap_enabled=false`. -- Call out explicitly that existing triggers pick up default gaps on upgrade - with no configuration change, and that downstream demos using deadman or - no-data alerts will start firing on the simulated outages. That may be - desirable (it exercises those alerts) but must not be a surprise. -- Add the six `gap_*` arguments to the plugin docstring's - `scheduled_args_config` metadata (required for InfluxDB 3 Explorer UI - integration). -- Update `influxdata/library/plugin_library.json` metadata for the new - version. -- Note the behavior change in the release/PR notes. - -## Test Plan - -Beyond unit tests for parsing and validation (each rule above, valid and -invalid, including `inf`/`nan`, swapped min/max, and the -`min_interval >= max_duration` rule): - -- **Batch invariance**: generate a fixed time range one point per call vs. - all points in one call; written timestamps must be identical. -- **Membership correctness**: compare `in_gap` against a brute-force scan of - all gap windows over a large sampled range. -- **Bounded spacing**: assert consecutive gap starts are always spaced within - `[gap_min_interval_seconds, gap_max_interval_seconds]` and durations within - `[gap_min_duration_seconds, gap_max_duration_seconds]` over a large index - range. -- **Non-overlap**: assert windows are disjoint over a large index range at - the validation boundary (`gap_min_interval_seconds == - gap_max_duration_seconds`) and at defaults. -- **Reproducibility**: same seed and config produce identical windows across - separate processes; different seeds produce different windows. -- **Gap spanning call boundaries**: a gap that starts in one call's window - and ends in a later call's window removes points in both calls. -- **Gap covering an entire call window**: call writes zero points, logs the - removed-by-gap count, and still advances `last_time`. -- **Catch-up**: after simulated downtime, the backfilled range contains gaps - at the same absolute positions as an uninterrupted run with the same seed. -- **Unseeded stability**: with `gap_seed` omitted, the cached auto-seed makes - consecutive calls share one schedule. -- **Disabled**: `gap_enabled=false` writes all generated points. -- **First-run init**: unchanged — no writes, cache initialized (including - `signal_gen:gap_seed` when unseeded). - -## Alternative Schedule Algorithms Considered - -### Sequential renewal process - -Draw each inter-gap interval from -`uniform(gap_min_interval_seconds, gap_max_interval_seconds)` and accumulate: -gap `n+1` starts where gap `n`'s interval ends. This gives independent, -identically distributed (uniform) spacing — but start times are cumulative -sums, so answering "is timestamp T inside a gap?" requires replaying every -draw from an anchor (millions of draws per call for an epoch anchor) or -persisting RNG state in the cache. Persisted state breaks batch invariance and -catch-up determinism, and grows the cache contract. Renewal processes do not -admit O(1) random access; this disqualifies them here. The jittered grid keeps -the same [min, max] spacing bounds and mean, trading uniform spacing for a -triangular distribution inside the bounds. - -### Per-call probability draw - -Each scheduled call rolls "does an outage start now?". Rejected outright: gap -frequency then scales with trigger frequency rather than simulated time, and a -one-point-per-call configuration degenerates to all-or-nothing per cycle. -Chaos-testing tools that use per-request draws document exactly this coupling. - -### Stateful burst-loss model (Gilbert-Elliott) - -A two-state good/bad Markov chain produces realistic burst losses, but the -decision at each point depends on hidden state carried from the previous -point — sequential by construction, with the same random-access and -persistence problems as the renewal process, plus a distribution concept the -plugin does not otherwise expose. - -## Alternative Configuration Shapes Considered - -### Center Plus Jitter - -```text -gap_duration_seconds=20 -gap_duration_jitter_seconds=10 -gap_interval_seconds=180 -gap_interval_jitter_seconds=60 -``` - -This matches the timestamp-jitter vocabulary, but the interval arguments are -misleading under a jittered-grid schedule: `gap_interval_jitter_seconds` would -bound each gap's offset from its grid position, while the spacing between -consecutive starts varies by twice that amount (180±60 config actually yields -60–300s spacing). The non-overlap validation rule is also harder to state -(`interval - 2*jitter >= duration + duration_jitter`). Min/max bounds are -directly checkable guarantees, and their validation rule -(`min_interval >= max_duration`) is self-explanatory. - -### Probability Per Second - -```text -gap_start_probability_per_second=0.005 -gap_duration_seconds=20 -gap_duration_jitter_seconds=10 -``` - -This maps cleanly to a stochastic process, but users tend to think in "roughly -every few minutes" rather than per-second probabilities. - -### Average and Distribution - -```text -gap_avg_duration_seconds=20 -gap_avg_interval_seconds=180 -gap_distribution=exponential -``` - -This can produce realistic outage timing, with many short intervals and a few -long quiet periods. It is harder to explain, validate, and test, and it adds a -distribution concept that the plugin does not otherwise expose. - -### Fixed Plus Random Factor - -```text -gap_duration_seconds=20 -gap_duration_random_factor=0.5 -gap_interval_seconds=180 -gap_interval_random_factor=0.25 -``` - -This is compact, but less obvious than duration and jitter expressed directly in -seconds. - -## Chosen Design - -Use a jittered-grid schedule over absolute time — one gap per `pitch`-wide -stratum, with start offset and duration derived from a `blake2b` hash of -`(seed, gap_index)` — configured with explicit min/max bounds: - -```text -gap_min_duration_seconds -gap_max_duration_seconds -gap_min_interval_seconds -gap_max_interval_seconds -gap_seed -gap_enabled -``` - -The bounds are hard guarantees on outage duration and on spacing between -consecutive outage starts. All gap parameters stay optional while default gaps -remain visible, and the schedule is reproducible, stateless, O(1) to query, -and independent of how many points each scheduled call generates. From 12ce61b879bb7b8d38e092668263fa4c8afe491e Mon Sep 17 00:00:00 2001 From: caterryan Date: Fri, 10 Jul 2026 11:59:20 -0500 Subject: [PATCH 3/3] Delete influxdata/signal_generator/GAP_IMPLEMENTATION_PLAN.md --- .../GAP_IMPLEMENTATION_PLAN.md | 258 ------------------ 1 file changed, 258 deletions(-) delete mode 100644 influxdata/signal_generator/GAP_IMPLEMENTATION_PLAN.md diff --git a/influxdata/signal_generator/GAP_IMPLEMENTATION_PLAN.md b/influxdata/signal_generator/GAP_IMPLEMENTATION_PLAN.md deleted file mode 100644 index c3e9eb7..0000000 --- a/influxdata/signal_generator/GAP_IMPLEMENTATION_PLAN.md +++ /dev/null @@ -1,258 +0,0 @@ -# Data Gap Implementation Plan - -Implements `GAP_DESIGN.md` (jittered-grid gap schedule, min/max configuration -shape). Read that document first; this plan only sequences the work and maps -it onto files. Where this plan and the design doc disagree, the design doc -wins. - -## Files touched - -| File | Change | -|------|--------| -| `signal_generator.py` | Gap config parsing, schedule functions, filtering, entry-point wiring, docstring metadata | -| `test_signal_generator.py` | New gap tests; update existing entry-point tests that assert exact output (gaps are on by default) | -| `README.md` | Gap section, headline bullet rewrite, parameter table, troubleshooting entry | -| `manifest.toml` | Version bump `0.2.0` → `0.3.0` | -| `influxdata/library/plugin_library.json` | Description mentions simulated gaps; `last_update` | - -## Step 1: Gap config parsing (`signal_generator.py`) - -Add defaults next to the existing jitter constants: - -```python -DEFAULT_GAP_MIN_DURATION_SECONDS = 10.0 -DEFAULT_GAP_MAX_DURATION_SECONDS = 30.0 -DEFAULT_GAP_MIN_INTERVAL_SECONDS = 120.0 -DEFAULT_GAP_MAX_INTERVAL_SECONDS = 240.0 -``` - -Add `parse_gap_config(args)` following the `parse_jitter_config` contract — -returns `(config, None)` on success or `(None, error_message)` on failure, -error messages prefixed `"Invalid gap config: "`. Config is a tuple: - -```python -(enabled, min_duration, max_duration, min_interval, max_interval, pitch, offset, seed) -``` - -with `pitch = (min_interval + max_interval) / 2` and -`offset = (max_interval - min_interval) / 4` precomputed at parse time. -`seed` is the configured `gap_seed` or `None` (resolved later, Step 3). - -Rules (all from the design doc's Validation section): - -- `gap_enabled`: accept `"true"`/`"false"` case-insensitive; default `True`; - anything else is an error. No boolean parser exists yet — add a small - `_parse_bool` helper. -- All four numeric args parse via `float(raw)` and must be finite - (`math.isfinite`), matching the existing jitter/pps checks. -- `min_duration >= 0`, `max_duration >= min_duration`, `min_interval > 0`, - `max_interval >= min_interval`, and the load-bearing rule - `min_interval >= max_duration`. -- `gap_seed` parses with the same integer coercion used for `jitter_seed` - (reject bools and non-integral floats). -- Validate every provided argument even when `gap_enabled=false`; disabled - only skips filtering, not validation. - -## Step 2: Schedule and filtering functions (`signal_generator.py`) - -Place next to `_rng_for_timestamp` — same primitive, keyed by gap index, with -`"gap"` domain separation: - -```python -def _rng_for_gap_index(seed, n): - payload = f"{seed}:gap:{n}".encode("ascii") - digest = hashlib.blake2b(payload, digest_size=8).digest() - return random.Random(int.from_bytes(digest, "big")) -``` - -`gap_window(gap_config, n)` — draw order is a reproducibility contract: -start offset first, then duration: - -```python -rng = _rng_for_gap_index(seed, n) -start = n * pitch + rng.uniform(-offset, +offset) -duration = rng.uniform(min_duration, max_duration) -return (start, start + duration) -``` - -`in_gap(gap_config, t)` — check candidate indices -`n_hi = int((t + offset) // pitch)` and `n_hi - 1`, skipping `n < 0`; -half-open comparison `start <= t < end`. - -`apply_gaps(points, gap_config)` — runs after `apply_timestamp_jitter`, tests -each point's final (jittered) timestamp, returns -`(kept_points, removed_count, gap_window_count)` where `gap_window_count` is -the number of gap indices whose window intersects -`[min(point ts), max(point ts)]` (iterate the index range -`int((t_min - offset - max_duration) // pitch)` to `int((t_max + offset) // pitch)`; -the range is tiny relative to any realistic call window). Memoize -`gap_window` per index in a local dict for the duration of the call so a -window shared by thousands of points hashes once. - -## Step 3: Seed resolution and cache (`signal_generator.py`) - -```python -GAP_SEED_CACHE_KEY = "signal_gen:gap_seed" -``` - -`resolve_gap_seed(cache, configured_seed)`: - -- configured seed set → return it, never touch the cache; -- else cached value present → return it; -- else generate `random.getrandbits(63)`, `cache.put`, return it. - -Called on every scheduled call when gaps are enabled (including first-run -init, so the schedule exists before the first data write). Cache loss simply -regenerates — documented behavior. - -`signal_gen:last_time` semantics unchanged: always advances to `now`, even -when every generated point was removed by a gap. No backfill. - -## Step 4: Entry-point wiring (`process_scheduled_call`) - -Parse order: waveforms → output → pps → jitter → **gap** (new). A gap config -error logs `f"[{task_id}] {gap_error}"` and returns before touching the -cache, matching the invalid-jitter behavior exactly (existing test pattern: -cache stays `None`). - -First-run init block: additionally call `resolve_gap_seed` when gaps are -enabled, then return as today. - -Generation path, after `apply_timestamp_jitter`: - -```python -generated = len(points) -if gap_enabled: - points, removed, gap_windows = apply_gaps(points, gap_config) -``` - -Then write as today (an empty kept-list skips the write and still advances -the cache — `write_points` already returns 0 for no lines). Success log -becomes, when gaps removed anything: - -```text -Wrote {written} points to {m}.{f} ({generated} generated, {removed} removed by {gap_windows} gap windows) -``` - -and stays the current one-liner when `removed == 0`, keeping logs quiet in -the common case. - -## Step 5: Docstring metadata - -Add the six `gap_*` arguments to `scheduled_args_config` (name, example, -description, `"required": false` — same shape as the jitter entries). -Examples: `gap_enabled` → `"false"`, durations → `"10"`/`"30"`, intervals → -`"120"`/`"240"`, `gap_seed` → `"123"`. Descriptions state the min/max bound -guarantees and that gaps are on by default. - -## Step 6: Tests (`test_signal_generator.py`) - -### Fix existing tests first — gaps are on by default - -Any entry-point test asserting exact point counts or timestamps now runs with -an unseeded (random) gap schedule and becomes nondeterministic. Add -`"gap_enabled": "false"` to the args of: - -- `test_process_scheduled_call_second_run` (currently passes no args — give - it `{"gap_enabled": "false"}`) -- `test_process_scheduled_call_custom_pps` -- `test_process_scheduled_call_target_database` -- `test_process_scheduled_call_jitter_zero_keeps_regular_timestamps` -- `test_process_scheduled_call_seeded_jitter_offsets_timestamps` -- `test_process_scheduled_call_seeded_jitter_varies_across_one_point_calls` -- `test_process_scheduled_call_default_jitter_offsets_timestamps` - -`test_process_scheduled_call_first_run` and `..._updates_cache` assert only -cache/no-write behavior and can stay as-is; extend `first_run` to also assert -`signal_gen:gap_seed` is populated. - -### New parsing/validation tests (mirror the jitter test style) - -- defaults (no args → enabled, 10/30/120/240, derived pitch 180 / offset 30) -- explicit values round-trip; `gap_enabled` `"false"`/`"TRUE"`/garbage -- each rejection: non-numeric, `nan`, `inf`, negative min duration, - `max_duration < min_duration`, `min_interval <= 0`, - `max_interval < min_interval`, `min_interval < max_duration`, - non-integer `gap_seed` -- disabled-but-invalid still errors - -### New schedule-function tests (design doc Test Plan section) - -- **Bounded spacing + non-overlap**: iterate `gap_window` over a few thousand - indices at defaults and at the equality boundary - (`min_interval == max_duration`); assert spacing within `[min_i, max_i]`, - durations within `[min_d, max_d]`, windows disjoint. -- **Membership correctness**: `in_gap` vs brute-force scan over the covering - index range for a few thousand sampled timestamps. -- **Reproducibility**: same seed → identical windows; different seeds differ. -- **Determinism pin**: one exact-value test in the style of - `test_apply_timestamp_jitter_seeded_exact_offsets` — compute - `gap_window(seed=7, n=...)` once after implementation, pin the constants. - Guards the draw-order and payload-format contract against refactors. - -### New entry-point tests - -- **Batch invariance**: fixed range, seeded gaps and seeded jitter: one call - covering 60 s vs 60 calls of 1 s; identical written timestamps. (Choose a - `gap_seed` whose window lands inside the range so the test is meaningful — - find it by scanning indices near the chosen epoch time.) -- **Gap spanning call boundaries**: points removed in both adjacent calls. -- **Gap covering entire call window**: zero written, cache still advances, - info log contains the removed-by-gap count. -- **Catch-up**: init, then a single call far in the future; removed spans - match the seeded schedule positions. -- **Unseeded stability**: two consecutive calls share the cached auto-seed - (assert cache key written once and schedule consistent across the calls). -- **Disabled**: `gap_enabled=false` writes all generated points. - -## Step 7: README - -Per the design doc's Breaking Change section: - -- Rewrite the "Gap-filling" headline bullet; add a "Simulated gaps" bullet. -- Add the six parameters to the optional-parameters table. -- Add a "Simulated gaps" section: min/max bound guarantees, on-by-default, - `gap_enabled=false`, seed behavior, upgrade note (existing triggers pick up - gaps; deadman/no-data alert demos will fire). -- Troubleshooting entry: "missing spans in query results" → expected default - behavior, how to disable, how to distinguish from real failures via the - removed-by-gap log line. -- Output schema / timestamp note: mention missing spans. - -## Step 8: Versioning and library metadata - -- `manifest.toml`: `version = "0.3.0"` (behavior-changing default). -- `plugin_library.json` signal_generator entry: append simulated-gap mention - to description, update `last_update`. -- PR notes: call out the default-on behavior change. - -## Step 9: Verify - -1. `pytest influxdata/signal_generator/test_signal_generator.py` — all pass. -2. Manual smoke: `influxdb3 test` or Docker Compose flow per repo README — - default trigger for ~10 minutes, query for missing spans near the 3-minute - cadence; then `gap_enabled=false` run shows continuous data. -3. Grep logs for the removed-by-gap summary line during the smoke run. - -## Sequencing - -Steps 1–3 are pure additions (safe to land in one commit with their unit -tests). Step 4 flips the default and must land together with the Step 6 test -updates and Steps 5, 7, 8 in the same PR — never ship the wiring without the -README/metadata changes. - -## Risks / notes - -- **Performance**: 2 blake2b digests per point worst case; memoization in - `apply_gaps` reduces this to ~2 per gap index per call. Catch-up over days - of downtime at high `points_per_second` is the worst case and stays linear - in point count. -- **Float precision**: gap indices near current epoch are ~10^7; `n * pitch` - products are exact well past 2^53 — no precision concern. -- **Contract freeze**: payload format `f"{seed}:gap:{n}"`, digest size 8, and - draw order (offset, duration) are frozen once released — changing any of - them silently reshuffles every seeded schedule. The determinism-pin test - exists to catch this. -- **Do not** add gap logic to `generate_timestamps` or waveform evaluation; - filtering happens strictly on final jittered timestamps - (design doc Pipeline Semantics).