From 64289383c8aec2c317c156755fe03dcf707dfb26 Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Mon, 13 Jul 2026 23:30:22 -0400 Subject: [PATCH] feat: add Norgate metadata fetching for contract specifications --- README.md | 19 ++++++++ src/cotdata/config.py | 4 ++ src/cotdata/providers/norgate.py | 74 ++++++++++++++++++++++++++++++++ src/cotdata/store.py | 13 +++++- src/cotdata/update.py | 12 ++++-- 5 files changed, 118 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 33e32e7..468110b 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ The **store is the API boundary** — not Python imports. Producers write Parque `manifest.json`; consumers only read. Nobody touches a vendor SDK at app runtime. Swapping a vendor is a producer-only change. +- `metadata/contract_specs.parquet` — Norgate contract specifications (Tick Size, Point Value, Margin). - `prices/{symbol}_{adjustment}.parquet` — Open/High/Low/Close/Volume/Open Interest, tz-naive `Date` index. `adjustment` ∈ {`backadj`, `unadj`}. Close = exchange settlement. - `cot_legacy/{code}.parquet` — weekly CFTC Legacy positioning. @@ -92,6 +93,7 @@ Set `COTDATA_STORE` to the synced store directory. ``` COTDATA_STORE=/store cotdata-update --prices --symbols ES NQ # Norgate (Windows) +COTDATA_STORE=/store cotdata-update --metadata # Norgate Metadata (Windows) COTDATA_STORE=/store cotdata-update --cot-legacy # CFTC Legacy (cross-platform) COTDATA_STORE=/store cotdata-update --cot-disagg # CFTC Disaggregated (cross-platform) COTDATA_STORE=/store cotdata-update --cot-tff # CFTC Traders in Financial Futures (cross-platform) @@ -179,6 +181,23 @@ The primary source for price history (Norgate Data). Indexed by tz-naive `Date`. | `Open Interest` | float | Total open interest. | | `Delivery Month` | float | Expiration month of the active contract (e.g. `202609`). Used to detect contract rolls. | +### Contract Specifications (`metadata/contract_specs.parquet`) +The primary source for contract metadata (Norgate Data). Used for exact point-value risk sizing and transaction cost models. + +| Column | Type | Description | +|--------|------|-------------| +| `Symbol` | string | Internal ticker symbol (e.g., `ES`). | +| `Norgate_Symbol` | string | Raw Norgate symbol used to query the API (e.g., `&ES_CCB`). | +| `Name` | string | Full name of the contract. | +| `Exchange` | string | Name of the listing exchange. | +| `Group` | string | Norgate asset classification group. | +| `Contract Size` | float | Size multiplier (e.g., $50 for ES). Also called Point Value. | +| `Tick Size` | float | Minimum price fluctuation (e.g., 0.25 for ES). | +| `Tick Value` | float | Dollar value of one tick (`Tick Size` * `Contract Size`). | +| `Point Value` | float | Same as `Contract Size`. | +| `Currency` | string | Base currency of the contract. | +| `Margin` | float | Initial margin requirement (if provided by Norgate). | + ### COT Legacy Data (`cot_legacy/{code}.parquet`) The primary source for Legacy positioning data (CFTC Legacy Futures Report). **History starts in 1986.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`. diff --git a/src/cotdata/config.py b/src/cotdata/config.py index 24476d6..a84412b 100644 --- a/src/cotdata/config.py +++ b/src/cotdata/config.py @@ -19,6 +19,10 @@ def prices_dir() -> Path: return store_root() / "prices" +def metadata_dir() -> Path: + return store_root() / "metadata" + + def cot_legacy_dir() -> Path: return store_root() / "cot_legacy" diff --git a/src/cotdata/providers/norgate.py b/src/cotdata/providers/norgate.py index 7d11c14..62f36d1 100644 --- a/src/cotdata/providers/norgate.py +++ b/src/cotdata/providers/norgate.py @@ -76,3 +76,77 @@ def update(symbols=None) -> None: print(f"{sym:5s}: {len(out):6d} bars -> store") except Exception as e: # noqa: BLE001 print(f"{sym:5s}: FAILED — {e}") + + +def get_symbol_metadata(internal_symbol: str) -> dict | None: + """Fetch contract specifications for a single continuous futures symbol.""" + import norgatedata # imported lazily + ng_sym = REGISTRY[internal_symbol].norgate + CCB_SUFFIX + + data = {'Symbol': internal_symbol, 'Norgate_Symbol': ng_sym} + try: + data['Name'] = norgatedata.security_name(ng_sym) + except Exception: + data['Name'] = None + try: + data['Exchange'] = norgatedata.exchange_name(ng_sym) + except Exception: + data['Exchange'] = None + try: + data['Group'] = norgatedata.classification_at_level( + ng_sym, + schemename='NorgateDataFuturesClassification', + classificationresulttype='Name', + level=1, + ) + except Exception: + data['Group'] = None + try: + data['Contract Size'] = norgatedata.point_value(ng_sym) + except Exception: + data['Contract Size'] = None + try: + data['Tick Size'] = norgatedata.tick_size(ng_sym) + except Exception: + data['Tick Size'] = None + + ts = data['Tick Size'] + cs = data['Contract Size'] + data['Tick Value'] = (ts * cs) if (ts is not None and cs is not None) else None + data['Point Value'] = cs + + try: + data['Currency'] = norgatedata.currency(ng_sym) + except Exception: + data['Currency'] = None + try: + data['Margin'] = norgatedata.margin(ng_sym) + except Exception: + data['Margin'] = None + + return data + + +def update_metadata(symbols=None) -> None: + """Fetch and write contract specifications (metadata) to the store.""" + import concurrent.futures + syms = symbols or [s.internal for s in all_symbols()] + + print(f"Fetching metadata for {len(syms)} symbols...") + metadata_rows = [] + + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: + futs = {pool.submit(get_symbol_metadata, s): s for s in syms} + for f in concurrent.futures.as_completed(futs): + result = f.result() + if result: + metadata_rows.append(result) + + if metadata_rows: + df = pd.DataFrame(metadata_rows) + # Ensure consistent column ordering and sorting + df = df.sort_values("Symbol").reset_index(drop=True) + store.write_metadata(df, source="norgate") + print(f"Successfully wrote metadata for {len(df)} symbols -> store") + else: + print("No metadata fetched.") diff --git a/src/cotdata/store.py b/src/cotdata/store.py index 619ad20..ccbc73b 100644 --- a/src/cotdata/store.py +++ b/src/cotdata/store.py @@ -25,6 +25,17 @@ def _atomic_write_parquet(df: pd.DataFrame, path: Path) -> None: os.remove(tmp) +# ── Metadata ────────────────────────────────────────────────────────────── +def write_metadata(df: pd.DataFrame, source: str = "norgate") -> None: + _atomic_write_parquet(df, config.metadata_dir() / "contract_specs.parquet") + _touch_manifest("metadata", "contract_specs", df, source) + + +def read_metadata() -> pd.DataFrame: + p = config.metadata_dir() / "contract_specs.parquet" + return pd.read_parquet(p) if p.exists() else pd.DataFrame() + + # ── Prices ──────────────────────────────────────────────────────────────── def write_prices(symbol: str, adjustment: str, df: pd.DataFrame, source: str) -> None: _atomic_write_parquet(df, config.prices_dir() / f"{symbol}_{adjustment}.parquet") @@ -75,7 +86,7 @@ def load_manifest() -> dict: p = config.manifest_path() if p.exists(): return json.loads(p.read_text()) - return {"schema_version": config.SCHEMA_VERSION, "prices": {}, "cot_legacy": {}, "cot_disagg": {}, "cot_tff": {}} + return {"schema_version": config.SCHEMA_VERSION, "metadata": {}, "prices": {}, "cot_legacy": {}, "cot_disagg": {}, "cot_tff": {}} def _touch_manifest(kind: str, name: str, df: pd.DataFrame, source: str) -> None: diff --git a/src/cotdata/update.py b/src/cotdata/update.py index 53ac2a8..4073e68 100644 --- a/src/cotdata/update.py +++ b/src/cotdata/update.py @@ -10,6 +10,7 @@ def main() -> None: p = argparse.ArgumentParser(description="cotdata producer — fetch sources into the store.") p.add_argument("--prices", action="store_true", help="Update Norgate price bars (Windows).") + p.add_argument("--metadata", action="store_true", help="Update Norgate contract metadata (Windows).") p.add_argument("--cot-legacy", action="store_true", help="Update CFTC COT Legacy (cross-platform).") p.add_argument("--cot-disagg", action="store_true", help="Update CFTC COT Disaggregated Futures-Only (cross-platform).") p.add_argument("--cot-tff", action="store_true", help="Update Traders in Financial Futures (TFF) COT (cross-platform).") @@ -19,13 +20,18 @@ def main() -> None: config.store_root() # fail fast if COTDATA_STORE unset - if not (args.prices or args.cot_legacy or args.cot_disagg or args.cot_tff or args.cot_all): - p.error("nothing to do — pass --prices, --cot-legacy, --cot-disagg, --cot-tff, or --cot-all") + if not (args.prices or args.metadata or args.cot_legacy or args.cot_disagg or args.cot_tff or args.cot_all): + p.error("nothing to do — pass --prices, --metadata, --cot-legacy, --cot-disagg, --cot-tff, or --cot-all") - if args.prices: + if args.prices or args.metadata: from .providers import norgate + + if args.prices: norgate.update(symbols=args.symbols) + if args.metadata: + norgate.update_metadata(symbols=args.symbols) + if args.cot_legacy or args.cot_all: from .providers import cftc cftc.update()