feat: add Norgate metadata fetching for contract specifications

README.md

@@ -66,6 +66,7 @@ The **store is the API boundary** — not Python imports. Producers write Parque
 `manifest.json`; consumers only read. Nobody touches a vendor SDK at app runtime.
 Swapping a vendor is a producer-only change.
 
+- `metadata/contract_specs.parquet` — Norgate contract specifications (Tick Size, Point Value, Margin).
 - `prices/{symbol}_{adjustment}.parquet` — Open/High/Low/Close/Volume/Open Interest,
   tz-naive `Date` index. `adjustment` ∈ {`backadj`, `unadj`}. Close = exchange settlement.
 - `cot_legacy/{code}.parquet` — weekly CFTC Legacy positioning.
@@ -92,6 +93,7 @@ Set `COTDATA_STORE` to the synced store directory.
 
 ```
 COTDATA_STORE=/store  cotdata-update --prices --symbols ES NQ    # Norgate (Windows)
+COTDATA_STORE=/store  cotdata-update --metadata                  # Norgate Metadata (Windows)
 COTDATA_STORE=/store  cotdata-update --cot-legacy                # CFTC Legacy (cross-platform)
 COTDATA_STORE=/store  cotdata-update --cot-disagg                # CFTC Disaggregated (cross-platform)
 COTDATA_STORE=/store  cotdata-update --cot-tff                   # CFTC Traders in Financial Futures (cross-platform)
@@ -179,6 +181,23 @@ The primary source for price history (Norgate Data). Indexed by tz-naive `Date`.
 | `Open Interest` | float | Total open interest. |
 | `Delivery Month` | float | Expiration month of the active contract (e.g. `202609`). Used to detect contract rolls. |
 
+### Contract Specifications (`metadata/contract_specs.parquet`)
+The primary source for contract metadata (Norgate Data). Used for exact point-value risk sizing and transaction cost models.
+
+| Column | Type | Description |
+|--------|------|-------------|
+| `Symbol` | string | Internal ticker symbol (e.g., `ES`). |
+| `Norgate_Symbol` | string | Raw Norgate symbol used to query the API (e.g., `&ES_CCB`). |
+| `Name` | string | Full name of the contract. |
+| `Exchange` | string | Name of the listing exchange. |
+| `Group` | string | Norgate asset classification group. |
+| `Contract Size` | float | Size multiplier (e.g., $50 for ES). Also called Point Value. |
+| `Tick Size` | float | Minimum price fluctuation (e.g., 0.25 for ES). |
+| `Tick Value` | float | Dollar value of one tick (`Tick Size` * `Contract Size`). |
+| `Point Value` | float | Same as `Contract Size`. |
+| `Currency` | string | Base currency of the contract. |
+| `Margin` | float | Initial margin requirement (if provided by Norgate). |
+
 ### COT Legacy Data (`cot_legacy/{code}.parquet`)
 The primary source for Legacy positioning data (CFTC Legacy Futures Report). **History starts in 1986.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
 

src/cotdata/config.py

@@ -19,6 +19,10 @@ def prices_dir() -> Path:
     return store_root() / "prices"
 
 
+def metadata_dir() -> Path:
+    return store_root() / "metadata"
+
+
 def cot_legacy_dir() -> Path:
     return store_root() / "cot_legacy"
 

src/cotdata/providers/norgate.py

@@ -76,3 +76,77 @@ def update(symbols=None) -> None:
             print(f"{sym:5s}: {len(out):6d} bars -> store")
         except Exception as e:  # noqa: BLE001
             print(f"{sym:5s}: FAILED — {e}")
+
+
+def get_symbol_metadata(internal_symbol: str) -> dict | None:
+    """Fetch contract specifications for a single continuous futures symbol."""
+    import norgatedata  # imported lazily
+    ng_sym = REGISTRY[internal_symbol].norgate + CCB_SUFFIX
+
+    data = {'Symbol': internal_symbol, 'Norgate_Symbol': ng_sym}
+    try:
+        data['Name'] = norgatedata.security_name(ng_sym)
+    except Exception:
+        data['Name'] = None
+    try:
+        data['Exchange'] = norgatedata.exchange_name(ng_sym)
+    except Exception:
+        data['Exchange'] = None
+    try:
+        data['Group'] = norgatedata.classification_at_level(
+            ng_sym,
+            schemename='NorgateDataFuturesClassification',
+            classificationresulttype='Name',
+            level=1,
+        )
+    except Exception:
+        data['Group'] = None
+    try:
+        data['Contract Size'] = norgatedata.point_value(ng_sym)
+    except Exception:
+        data['Contract Size'] = None
+    try:
+        data['Tick Size'] = norgatedata.tick_size(ng_sym)
+    except Exception:
+        data['Tick Size'] = None
+
+    ts = data['Tick Size']
+    cs = data['Contract Size']
+    data['Tick Value'] = (ts * cs) if (ts is not None and cs is not None) else None
+    data['Point Value'] = cs
+
+    try:
+        data['Currency'] = norgatedata.currency(ng_sym)
+    except Exception:
+        data['Currency'] = None
+    try:
+        data['Margin'] = norgatedata.margin(ng_sym)
+    except Exception:
+        data['Margin'] = None
+
+    return data
+
+
+def update_metadata(symbols=None) -> None:
+    """Fetch and write contract specifications (metadata) to the store."""
+    import concurrent.futures
+    syms = symbols or [s.internal for s in all_symbols()]
+
+    print(f"Fetching metadata for {len(syms)} symbols...")
+    metadata_rows = []
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
+        futs = {pool.submit(get_symbol_metadata, s): s for s in syms}
+        for f in concurrent.futures.as_completed(futs):
+            result = f.result()
+            if result:
+                metadata_rows.append(result)
+
+    if metadata_rows:
+        df = pd.DataFrame(metadata_rows)
+        # Ensure consistent column ordering and sorting
+        df = df.sort_values("Symbol").reset_index(drop=True)
+        store.write_metadata(df, source="norgate")
+        print(f"Successfully wrote metadata for {len(df)} symbols -> store")
+    else:
+        print("No metadata fetched.")

src/cotdata/store.py

@@ -25,6 +25,17 @@ def _atomic_write_parquet(df: pd.DataFrame, path: Path) -> None:
             os.remove(tmp)
 
 
+# ── Metadata ──────────────────────────────────────────────────────────────
+def write_metadata(df: pd.DataFrame, source: str = "norgate") -> None:
+    _atomic_write_parquet(df, config.metadata_dir() / "contract_specs.parquet")
+    _touch_manifest("metadata", "contract_specs", df, source)
+
+
+def read_metadata() -> pd.DataFrame:
+    p = config.metadata_dir() / "contract_specs.parquet"
+    return pd.read_parquet(p) if p.exists() else pd.DataFrame()
+
+
 # ── Prices ────────────────────────────────────────────────────────────────
 def write_prices(symbol: str, adjustment: str, df: pd.DataFrame, source: str) -> None:
     _atomic_write_parquet(df, config.prices_dir() / f"{symbol}_{adjustment}.parquet")
@@ -75,7 +86,7 @@ def load_manifest() -> dict:
     p = config.manifest_path()
     if p.exists():
         return json.loads(p.read_text())
-    return {"schema_version": config.SCHEMA_VERSION, "prices": {}, "cot_legacy": {}, "cot_disagg": {}, "cot_tff": {}}
+    return {"schema_version": config.SCHEMA_VERSION, "metadata": {}, "prices": {}, "cot_legacy": {}, "cot_disagg": {}, "cot_tff": {}}
 
 
 def _touch_manifest(kind: str, name: str, df: pd.DataFrame, source: str) -> None:

src/cotdata/update.py

@@ -10,6 +10,7 @@
 def main() -> None:
     p = argparse.ArgumentParser(description="cotdata producer — fetch sources into the store.")
     p.add_argument("--prices", action="store_true", help="Update Norgate price bars (Windows).")
+    p.add_argument("--metadata", action="store_true", help="Update Norgate contract metadata (Windows).")
     p.add_argument("--cot-legacy", action="store_true", help="Update CFTC COT Legacy (cross-platform).")
     p.add_argument("--cot-disagg", action="store_true", help="Update CFTC COT Disaggregated Futures-Only (cross-platform).")
     p.add_argument("--cot-tff", action="store_true", help="Update Traders in Financial Futures (TFF) COT (cross-platform).")
@@ -19,13 +20,18 @@ def main() -> None:
 
     config.store_root()  # fail fast if COTDATA_STORE unset
 
-    if not (args.prices or args.cot_legacy or args.cot_disagg or args.cot_tff or args.cot_all):
-        p.error("nothing to do — pass --prices, --cot-legacy, --cot-disagg, --cot-tff, or --cot-all")
+    if not (args.prices or args.metadata or args.cot_legacy or args.cot_disagg or args.cot_tff or args.cot_all):
+        p.error("nothing to do — pass --prices, --metadata, --cot-legacy, --cot-disagg, --cot-tff, or --cot-all")
 
-    if args.prices:
+    if args.prices or args.metadata:
         from .providers import norgate
+
+    if args.prices:
         norgate.update(symbols=args.symbols)
 
+    if args.metadata:
+        norgate.update_metadata(symbols=args.symbols)
+
     if args.cot_legacy or args.cot_all:
         from .providers import cftc
         cftc.update()