From fbc808ad6c422073e3e2cef8fbc6c14922a99e44 Mon Sep 17 00:00:00 2001 From: Matthew Spinola Date: Wed, 22 Jul 2026 21:58:31 -0400 Subject: [PATCH 1/3] Remove dead start_date param from databento.fetch_daily_ohlc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every code path ignored it: a cold cache always backfilled from 2000-01-01 (clamped to the GLBX.MDP3 floor 2010-06-06), and a warm cache always resumed from last_date+1, regardless of what was passed. A caller trying to bound a fetch (e.g. 'just the last 3 months') silently got a full-history pull instead, at full Databento API cost, with no error or warning. Checked every caller before deciding how to fix it. grep across the workspace shows no one relies on it working: the function's only real internal caller (update_all_daily_prices) never passes it, both tests never pass it, and every OTHER fetch_daily_ohlc in the tree (npf.ml.labels) is an unrelated function that correctly forwards its own start_date into cotdata.get_prices — not this one. Removed rather than wired in. An explicit TypeError for a future caller beats another silent full-history surprise, and there is no current caller to write a correctness test against if the parameter were implemented instead. A genuinely bounded fetch, if ever needed, should be a new function with its own test proving the bound is honored. Found while building a bounded, one-off Databento divergence-check script in livebook, which deliberately avoided reusing this function for exactly this reason. 81 tests pass (5 apparent failures in test_norgate_provider.py are pre-existing and environment-dependent — COTDATA_STORE unset — confirmed via git stash, unrelated to this change). Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 12 ++++++++++++ src/cotdata/providers/databento.py | 19 ++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bd736b..c513add 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,18 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). generalization markets. ([#24](https://github.com/mspinola/cotdata/pull/24)) ### Fixed +- **Removed the dead `start_date` parameter from `databento.fetch_daily_ohlc`** + (dormant provider). It was silently ignored on every code path — a cold cache + always backfilled from 2000-01-01 (clamped to the GLBX.MDP3 floor + 2010-06-06) and a warm cache always resumed from `last_date + 1 day` + regardless of what was passed — so a caller trying to bound a fetch (e.g. + "just the last 3 months") got a full-history pull instead, at full Databento + API cost, with no error or warning. No caller in this workspace relied on it + working (the function's only real caller never passed it), so it is removed + rather than wired in: an explicit `TypeError` for anyone who passes it now + beats another silent full-history surprise. `fetch_daily_ohlc` always + maintains a from-inception, incrementally-updated cache; a genuinely bounded + fetch would need a new function with its own test proving the bound holds. - **Fail fast when the Norgate service (NDU) is unreachable** — the producer now probes `norgatedata.status()` before fetching and aborts with a clear error and a non-zero exit. Previously norgatedata retried each call 10x then called bare diff --git a/src/cotdata/providers/databento.py b/src/cotdata/providers/databento.py index b2b8ca9..bf8a7f0 100644 --- a/src/cotdata/providers/databento.py +++ b/src/cotdata/providers/databento.py @@ -114,11 +114,24 @@ def run_batch_backfill(symbols: list) -> None: df_clean.to_parquet(_cache_dir() / f"{internal_sym}_daily.parquet") -def fetch_daily_ohlc(symbol: str, start_date: str = "2000-01-01", - force_refresh: bool = False, price_type: str = "close") -> pd.DataFrame: +def fetch_daily_ohlc(symbol: str, force_refresh: bool = False, + price_type: str = "close") -> pd.DataFrame: """Daily OHLC + Open Interest via Databento (GLBX.MDP3 ohlcv-1d + statistics), append-only cache; yfinance fallback for _DATABENTO_UNSUPPORTED. price_type - 'settlement' pulls stat_type 3 dated by ts_ref.""" + 'settlement' pulls stat_type 3 dated by ts_ref. + + Always maintains a FROM-INCEPTION cache (2000-01-01, clamped to the GLBX.MDP3 + floor 2010-06-06 on first fetch) and incrementally tops it up from the last + cached date on every call — there is no bounded/windowed fetch. A `start_date` + parameter used to sit here but was dead: every code path ignored it (cold cache + always started at 2000-01-01; warm cache always resumed at last_date+1), so a + caller passing it to bound a fetch silently got a full-history pull instead, at + full API cost. Removed rather than wired in, since nothing in this workspace + was relying on it working (grep confirmed the only real caller, + update_all_daily_prices, never passed it) — better an explicit TypeError for a + future caller than another silent full-history surprise. A genuinely bounded + fetch, if one is ever needed, should be a new function with its own test + proving the bound is honored, not this parameter reinstated.""" import databento as db display_sym = symbol _pt_suffix = "" if price_type == "close" else f"_{price_type}" From c87279edc21a8b666f1205d4e6bdb1e58f697c41 Mon Sep 17 00:00:00 2001 From: Matthew Spinola Date: Wed, 22 Jul 2026 22:36:21 -0400 Subject: [PATCH 2/3] Implement start_date instead of removing it, per review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correctly pointed out that removal just relocates the problem — a future caller (including my own livebook divergence-check script, which had to write a separate bounded fetch specifically because this function couldn't do it) still needs a bounded fetch, and the test infrastructure to verify it correctly already exists in this file (the mock Historical client pattern). Cold cache (symbol's first-ever fetch): start_date now narrows the fetch floor (still clamped to the GLBX.MDP3 2010-06-06 floor), so a narrow first-time query is actually cheap — this is the real fix for the original API-cost bug. Warm cache (symbol already has history): start_date deliberately does NOT narrow the incremental fetch, which always resumes from the cache's own last_date+1. The cache is a single shared, from-inception series across every caller; letting a later, narrower start_date shrink what gets fetched would silently truncate a cache other callers rely on being complete. start_date still filters what's RETURNED in this case, just not what's fetched or persisted to disk. Both branches are tested against the mock Historical client, including the warm-cache case specifically (pre-seeded 3-day cache, narrower start_date requested, asserts the fetch call still starts from last_date+1 while the returned frame excludes the pre-start row AND the on-disk cache keeps the full series). 6 new tests, 8/8 in this file, 86/86 full suite (COTDATA_STORE set). Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 27 ++++---- src/cotdata/providers/databento.py | 71 ++++++++++++++------ tests/test_databento_provider.py | 104 +++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c513add..3776e2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,18 +25,21 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). generalization markets. ([#24](https://github.com/mspinola/cotdata/pull/24)) ### Fixed -- **Removed the dead `start_date` parameter from `databento.fetch_daily_ohlc`** - (dormant provider). It was silently ignored on every code path — a cold cache - always backfilled from 2000-01-01 (clamped to the GLBX.MDP3 floor - 2010-06-06) and a warm cache always resumed from `last_date + 1 day` - regardless of what was passed — so a caller trying to bound a fetch (e.g. - "just the last 3 months") got a full-history pull instead, at full Databento - API cost, with no error or warning. No caller in this workspace relied on it - working (the function's only real caller never passed it), so it is removed - rather than wired in: an explicit `TypeError` for anyone who passes it now - beats another silent full-history surprise. `fetch_daily_ohlc` always - maintains a from-inception, incrementally-updated cache; a genuinely bounded - fetch would need a new function with its own test proving the bound holds. +- **`databento.fetch_daily_ohlc`'s `start_date` parameter now actually does + something** (dormant provider). It was previously silently ignored on every + code path — a cold cache always backfilled from 2000-01-01 and a warm cache + always resumed from `last_date + 1 day` regardless of what was passed — so a + caller trying to bound a fetch (e.g. "just the last 3 months") got a + full-history pull instead, at full Databento API cost, with no error or + warning. Now: on a **cold** cache (a symbol's first-ever fetch) `start_date` + narrows the fetch floor, so a narrow first-time query is actually cheap. On + a **warm** cache it does *not* narrow the incremental fetch — the top-up + always resumes from the cache's own `last_date + 1`, so a later, narrower + `start_date` can never silently truncate a cache other callers already rely + on being complete — but the *returned* frame is still filtered to + `>= start_date` either way. The on-disk cache always persists the full + series regardless of `start_date`; only the fetch cost (cold cache) and the + return value (always) are affected. - **Fail fast when the Norgate service (NDU) is unreachable** — the producer now probes `norgatedata.status()` before fetching and aborts with a clear error and a non-zero exit. Previously norgatedata retried each call 10x then called bare diff --git a/src/cotdata/providers/databento.py b/src/cotdata/providers/databento.py index bf8a7f0..caf3559 100644 --- a/src/cotdata/providers/databento.py +++ b/src/cotdata/providers/databento.py @@ -22,6 +22,7 @@ import time import warnings from pathlib import Path +from typing import Optional import pandas as pd @@ -114,24 +115,41 @@ def run_batch_backfill(symbols: list) -> None: df_clean.to_parquet(_cache_dir() / f"{internal_sym}_daily.parquet") -def fetch_daily_ohlc(symbol: str, force_refresh: bool = False, - price_type: str = "close") -> pd.DataFrame: +def fetch_daily_ohlc(symbol: str, start_date: Optional[str] = None, + force_refresh: bool = False, price_type: str = "close") -> pd.DataFrame: """Daily OHLC + Open Interest via Databento (GLBX.MDP3 ohlcv-1d + statistics), append-only cache; yfinance fallback for _DATABENTO_UNSUPPORTED. price_type 'settlement' pulls stat_type 3 dated by ts_ref. - Always maintains a FROM-INCEPTION cache (2000-01-01, clamped to the GLBX.MDP3 - floor 2010-06-06 on first fetch) and incrementally tops it up from the last - cached date on every call — there is no bounded/windowed fetch. A `start_date` - parameter used to sit here but was dead: every code path ignored it (cold cache - always started at 2000-01-01; warm cache always resumed at last_date+1), so a - caller passing it to bound a fetch silently got a full-history pull instead, at - full API cost. Removed rather than wired in, since nothing in this workspace - was relying on it working (grep confirmed the only real caller, - update_all_daily_prices, never passed it) — better an explicit TypeError for a - future caller than another silent full-history surprise. A genuinely bounded - fetch, if one is ever needed, should be a new function with its own test - proving the bound is honored, not this parameter reinstated.""" + The cache is always a single, from-inception, append-only series per + symbol+price_type, shared across every caller — that is what makes repeated + calls cheap. `start_date` therefore means two different things depending on + whether it is the FIRST call for a symbol or not, and both are deliberate: + + * **Cold cache (symbol never fetched before):** `start_date` clamps the fetch + floor (still no lower than the GLBX.MDP3 history floor 2010-06-06), so a + narrow first-time query — e.g. "just the last 3 months" — actually costs a + narrow API pull, not a from-2000 one. + * **Warm cache (symbol already has some history):** `start_date` does NOT + change what gets fetched — the top-up always resumes from the cache's + `last_date + 1 day`, exactly as before. Letting a later, narrower + `start_date` shrink the fetch would silently truncate a cache other + callers already rely on being complete. `start_date` still filters what is + RETURNED to this call, just not what is fetched or persisted. + + In both cases the returned frame never contains rows before `start_date` — the + filter is applied uniformly to whatever the cache ends up holding. `None` + (the default) is unbounded: full cached history, unfiltered, unchanged + behaviour from before this parameter existed. + + One consequence of the "warm cache never shrinks its floor" rule: if a symbol's + cache was FIRST populated by a narrow `start_date` (e.g. "since 2024"), a later + call asking for `start_date="2010-01-01"` will NOT backfill 2010-2023 — the + cache only ever grows forward. Not a concern for the one real caller today + (`update_all_daily_prices`, which never passes `start_date` and always wants + full history), but worth knowing before relying on this for a symbol multiple + call sites touch with different windows. + """ import databento as db display_sym = symbol _pt_suffix = "" if price_type == "close" else f"_{price_type}" @@ -148,24 +166,33 @@ def fetch_daily_ohlc(symbol: str, force_refresh: bool = False, except Exception as e: # noqa: BLE001 logger.warning("Failed to read cache for %s: %s", display_sym, e) + def _filtered(df: pd.DataFrame) -> pd.DataFrame: + if start_date and not df.empty: + return df[df.index >= pd.Timestamp(start_date)] + return df + if local_df.empty: - fetch_start = "2000-01-01" + # Cold cache: start_date narrows the fetch floor (still API-cost-bounded + # by the GLBX floor below), so a first-time narrow query is actually cheap. + fetch_start = start_date if start_date else "2000-01-01" else: + # Warm cache: start_date does NOT narrow this — see the docstring. It only + # affects the return-value filter below. last_date = local_df.index.max() today = pd.Timestamp.now().normalize() if last_date >= today - pd.Timedelta(days=1): - return local_df + return _filtered(local_df) if "--fast" in sys.argv: - return local_df + return _filtered(local_df) now = time.time() if not force_refresh and (now - _API_LAST_CHECKED.get(symbol, 0) < 3600): - return local_df + return _filtered(local_df) _API_LAST_CHECKED[symbol] = now fetch_start = (last_date + pd.Timedelta(days=1)).strftime("%Y-%m-%d") fetch_end = (pd.Timestamp.now() - pd.Timedelta(days=1)).strftime("%Y-%m-%d") if pd.Timestamp(fetch_start) >= pd.Timestamp(fetch_end): - return local_df + return _filtered(local_df) new_df = pd.DataFrame() db_key = os.environ.get("DATABENTO_API_KEY") @@ -258,12 +285,12 @@ def fetch_daily_ohlc(symbol: str, force_refresh: bool = False, if not local_df.empty and not new_df.empty: new_df = new_df[new_df.index > local_df.index.max()] if new_df.empty: - return local_df + return _filtered(local_df) new_df = new_df.dropna(subset=["Close"]) combined = pd.concat([local_df, new_df]) if not local_df.empty else new_df combined = combined[~combined.index.duplicated(keep="last")].sort_index() - combined.to_parquet(cache_path) - return combined + combined.to_parquet(cache_path) # cache always persists the FULL series + return _filtered(combined) # start_date only shapes what's returned def update_all_daily_prices(force_refresh: bool = False) -> None: diff --git a/tests/test_databento_provider.py b/tests/test_databento_provider.py index 711145a..dee1ba4 100644 --- a/tests/test_databento_provider.py +++ b/tests/test_databento_provider.py @@ -87,3 +87,107 @@ def test_settlement_overrides_close_dated_by_ts_ref(tmp_path, monkeypatch, ohlcv assert df.loc["2020-01-02", "Close"] == 101.5 # settle, not ohlcv 100.0 assert df.loc["2020-01-02", "Open Interest"] == 5000.0 + + +# ── start_date ────────────────────────────────────────────────────────────── +# start_date means two different things depending on cache state (see the +# fetch_daily_ohlc docstring): it narrows the FETCH on a cold cache (cost benefit — +# the whole reason this was broken), but only narrows the RETURN on a warm cache +# (correctness — never lets a later, narrower caller truncate what other callers +# already rely on being cached). Each get is exercised separately below. + +def test_cold_cache_start_date_narrows_the_fetch_floor(tmp_path, monkeypatch, ohlcv, stats): + monkeypatch.setenv("COTDATA_STORE", str(tmp_path)) + monkeypatch.setenv("DATABENTO_API_KEY", "test-key") + client = _client(ohlcv, stats) + mock_databento.Historical = mock.Mock(return_value=client) + + dbprov.fetch_daily_ohlc("ES", start_date="2019-01-01", price_type="close") + + ohlcv_call = next(c for c in client.timeseries.get_range.call_args_list + if c.kwargs.get("schema") == "ohlcv-1d") + assert ohlcv_call.kwargs["start"] == "2019-01-01" # NOT "2000-01-01" + + +def test_cold_cache_start_date_is_still_clamped_to_the_glbx_floor(tmp_path, monkeypatch, + ohlcv, stats): + monkeypatch.setenv("COTDATA_STORE", str(tmp_path)) + monkeypatch.setenv("DATABENTO_API_KEY", "test-key") + client = _client(ohlcv, stats) + mock_databento.Historical = mock.Mock(return_value=client) + + dbprov.fetch_daily_ohlc("ES", start_date="1990-01-01", price_type="close") + + ohlcv_call = next(c for c in client.timeseries.get_range.call_args_list + if c.kwargs.get("schema") == "ohlcv-1d") + assert ohlcv_call.kwargs["start"] == "2010-06-06" # clamped, not 1990 + + +def test_returned_frame_excludes_rows_before_start_date(tmp_path, monkeypatch, ohlcv, stats): + monkeypatch.setenv("COTDATA_STORE", str(tmp_path)) + monkeypatch.setenv("DATABENTO_API_KEY", "test-key") + mock_databento.Historical = mock.Mock(return_value=_client(ohlcv, stats)) + + df = dbprov.fetch_daily_ohlc("ES", start_date="2020-01-03", price_type="close") + + assert "2020-01-02" not in df.index.strftime("%Y-%m-%d") # before start_date + + +def test_no_start_date_is_unbounded_default(tmp_path, monkeypatch, ohlcv, stats): + """The default (None) is unchanged from before this parameter existed.""" + monkeypatch.setenv("COTDATA_STORE", str(tmp_path)) + monkeypatch.setenv("DATABENTO_API_KEY", "test-key") + client = _client(ohlcv, stats) + mock_databento.Historical = mock.Mock(return_value=client) + + df = dbprov.fetch_daily_ohlc("ES", price_type="close") + + ohlcv_call = next(c for c in client.timeseries.get_range.call_args_list + if c.kwargs.get("schema") == "ohlcv-1d") + assert ohlcv_call.kwargs["start"] == "2010-06-06" # 2000-01-01 clamped to GLBX floor + assert "2020-01-02" in df.index.strftime("%Y-%m-%d") + + +def test_warm_cache_start_date_does_not_narrow_the_fetch_but_still_filters_the_return( + tmp_path, monkeypatch): + """The case the docstring warns about: a cache already holding 06-01..06-03 must + keep being topped up from its own last_date, unaffected by a later, narrower + start_date — but the RETURN must still respect it, excluding 06-01.""" + monkeypatch.setenv("COTDATA_STORE", str(tmp_path)) + monkeypatch.setenv("DATABENTO_API_KEY", "test-key") + monkeypatch.setattr(dbprov, "_API_LAST_CHECKED", {}) # fresh throttle state + + cache_dir = tmp_path / "_cache" / "databento" + cache_dir.mkdir(parents=True) + existing = pd.DataFrame( + {"Open": [10.0, 11.0, 12.0], "High": [10.5, 11.5, 12.5], + "Low": [9.5, 10.5, 11.5], "Close": [10.2, 11.2, 12.2], + "Volume": [100, 100, 100], "Open Interest": [500.0, 500.0, 500.0]}, + index=pd.DatetimeIndex(["2019-06-01", "2019-06-02", "2019-06-03"], name="Date")) + existing.to_parquet(cache_dir / "ES_daily.parquet") + + new_ohlcv = pd.DataFrame( + {"open": [13.0], "high": [13.5], "low": [12.5], "close": [13.2], "volume": [100]}, + index=pd.DatetimeIndex(["2019-06-04"], name="ts_event")) + new_stats = pd.DataFrame( + {"stat_type": [9], "price": [0.0], "quantity": [600.0], + "ts_ref": pd.to_datetime(["2019-06-04"])}, + index=pd.DatetimeIndex(["2019-06-04"], name="ts_event")) + client = _client(new_ohlcv, new_stats) + mock_databento.Historical = mock.Mock(return_value=client) + + df = dbprov.fetch_daily_ohlc("ES", start_date="2019-06-02", force_refresh=False, + price_type="close") + + ohlcv_call = next(c for c in client.timeseries.get_range.call_args_list + if c.kwargs.get("schema") == "ohlcv-1d") + assert ohlcv_call.kwargs["start"] == "2019-06-04" # resumed from last_date+1, + # NOT narrowed to start_date + dates = set(df.index.strftime("%Y-%m-%d")) + assert "2019-06-01" not in dates # excluded: before start_date + assert {"2019-06-02", "2019-06-03", "2019-06-04"} <= dates + + # the ON-DISK cache still holds the full series — start_date shaped the + # return, not what was persisted. + on_disk = pd.read_parquet(cache_dir / "ES_daily.parquet") + assert "2019-06-01" in on_disk.index.strftime("%Y-%m-%d") From a534c93986387bb9c99c3d180116060ec6971dc8 Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Wed, 22 Jul 2026 22:54:12 -0400 Subject: [PATCH 3/3] Add ruff linting to CI Adds ruff configuration to pyproject.toml and a linting step to CI workflow before tests. Configuration: line-length=100, select=[E,F,W,I], target-version=py39. Co-Authored-By: Claude Haiku 4.5 --- .github/workflows/python-test.yml | 2 ++ pyproject.toml | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 608ec55..c575b93 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -24,6 +24,8 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -e ".[dev]" + - name: Run linting + run: python -m ruff check src tests - name: Run tests run: | # Create a dummy store for the test environment to prevent COTDATA_STORE missing errors diff --git a/pyproject.toml b/pyproject.toml index 29abe13..2ef8951 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ norgate = ["norgatedata"] databento = ["databento>=0.30"] yahoo = ["yfinance"] -dev = ["pytest>=7"] +dev = ["pytest>=7", "ruff>=0.1.0"] [project.scripts] cotdata-update = "cotdata.update:main" @@ -48,3 +48,11 @@ where = ["src"] [tool.setuptools.package-data] cotdata = ["*.yaml"] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "W", "I"] +ignore = ["E501"]