diff --git a/src/cotdata/providers/norgate.py b/src/cotdata/providers/norgate.py index f8d1245..12b1636 100644 --- a/src/cotdata/providers/norgate.py +++ b/src/cotdata/providers/norgate.py @@ -115,19 +115,25 @@ def _reconstruct_volume(internal_symbol: str, continuous_df: pd.DataFrame, adjus all_indiv = pd.concat(frames, ignore_index=True) all_indiv['Date'] = pd.to_datetime(all_indiv['Date']).dt.tz_localize(None).dt.normalize() - # 5. Point-in-Time Contract Identification + # 5. Contract identification: First / Second = the two HIGHEST-VOLUME contracts + # trading that day, NOT the two nearest by expiry. Products with serial months + # around a bi-monthly liquid cycle (e.g. GC, SI) carry almost no volume in the + # nearest serial month, so an expiry-order pick would sum near-empty contracts + # and badly understate true volume. Rank by descending volume; ties break by + # nearest expiry (columns are pre-sorted by expiry and the sort is stable); + # NaN (contract not trading that day) sorts last. pivot = all_indiv.pivot(index="Date", columns="Symbol", values="Volume") - + def get_expiry(sym): m = pattern.match(sym) return pd.Timestamp(year=int(m.group(1)), month=MONTH_CODES[m.group(2)], day=1) - + sorted_cols = sorted(pivot.columns, key=get_expiry) pivot = pivot[sorted_cols] - + vol_array = pivot.values - sort_key = np.isnan(vol_array).astype(int) - order = np.argsort(sort_key, axis=1, kind='stable') + rank_key = np.where(np.isnan(vol_array), -np.inf, vol_array) + order = np.argsort(-rank_key, axis=1, kind='stable') compressed = np.take_along_axis(vol_array, order, axis=1) names_arr = np.array(pivot.columns) diff --git a/tests/test_norgate_provider.py b/tests/test_norgate_provider.py index 2c35ed1..6605173 100644 --- a/tests/test_norgate_provider.py +++ b/tests/test_norgate_provider.py @@ -127,6 +127,52 @@ def mock_ts_side_effect(sym, **kwargs): assert written_df["SecondContract"].iloc[0] == "ES-2026M" +@mock.patch("cotdata.providers.norgate.store.write_prices") +@mock.patch("cotdata.providers.norgate.store.read_prices") +@mock.patch("norgatedata.database_symbols", create=True) +@mock.patch("norgatedata.price_timeseries", create=True) +def test_reconstruction_picks_by_volume_not_expiry(mock_price_ts, mock_db_symbols, mock_read_prices, mock_write_prices): + """First/Second must be the two HIGHEST-VOLUME contracts, not the two nearest by + expiry. Models the GC/SI case: the nearest serial month is near-empty while a + later contract is dominant. An expiry-order pick would name the empty serial as + 'First' and understate volume; volume-rank must name the dominant contract.""" + mock_continuous = pd.DataFrame({ + "Open": [100.0], "High": [101.0], "Low": [99.0], "Close": [100.5], + "Volume": [1000], "Open Interest": [5000], "Delivery Month": [202606], + }, index=pd.DatetimeIndex(["2026-07-01"])) + + # ES-2026H (March) = nearest by expiry but near-empty; ES-2026M (June) = dominant. + near_empty = pd.DataFrame({"Date": [pd.Timestamp("2026-07-01")], "Volume": [50]}) + dominant = pd.DataFrame({"Date": [pd.Timestamp("2026-07-01")], "Volume": [900]}) + + def mock_ts_side_effect(sym, **kwargs): + if sym.endswith("CCB") or "-" not in sym: + return mock_continuous.copy() + if sym == "ES-2026H": + return near_empty.copy() + if sym == "ES-2026M": + return dominant.copy() + return pd.DataFrame() + + mock_price_ts.side_effect = mock_ts_side_effect + mock_db_symbols.return_value = ["ES-2026H", "ES-2026M"] + mock_read_prices.return_value = pd.DataFrame() + + mock_symbol = mock.Mock() + mock_symbol.internal = "ES" + with mock.patch("cotdata.providers.norgate.all_symbols", return_value=[mock_symbol]), \ + mock.patch.dict("cotdata.providers.norgate.REGISTRY", {"ES": mock.Mock(norgate="&ES")}): + norgate.update(symbols=["ES"]) + written_df = mock_write_prices.call_args_list[0][0][2] + + # Dominant (June, 900) is First even though March expires sooner. + assert written_df["FirstContract"].iloc[0] == "ES-2026M" + assert written_df["FirstVolume"].iloc[0] == 900 + assert written_df["SecondContract"].iloc[0] == "ES-2026H" + assert written_df["SecondVolume"].iloc[0] == 50 + assert written_df["Volume_Reconstructed"].iloc[0] == 950 + + @mock.patch("cotdata.providers.norgate.store.write_prices") @mock.patch("cotdata.providers.norgate.store.read_prices") @mock.patch("norgatedata.database_symbols", create=True)