From 51270a112856d522b7f520706f5f0a26f64018c2 Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Mon, 13 Jul 2026 22:45:29 -0400 Subject: [PATCH 1/2] feat: add Trader counts to Legacy dataset --- src/cotdata/providers/cftc.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/cotdata/providers/cftc.py b/src/cotdata/providers/cftc.py index df8b996..ba18006 100644 --- a/src/cotdata/providers/cftc.py +++ b/src/cotdata/providers/cftc.py @@ -34,9 +34,17 @@ NONREPT_LONG = "NonRept_Positions_Long_All" NONREPT_SHORT = "NonRept_Positions_Short_All" +TRADERS_TOT = "Traders_Tot_All" +TRADERS_COMM_LONG = "Traders_Comm_Long_All" +TRADERS_COMM_SHORT = "Traders_Comm_Short_All" +TRADERS_NONCOMM_LONG = "Traders_NonComm_Long_All" +TRADERS_NONCOMM_SHORT = "Traders_NonComm_Short_All" + TARGET_COLS = [MARKET_NAME, REPORT_DATE, CONTRACT_CODE, OPEN_INTEREST, COMM_LONG, COMM_SHORT, NONCOMM_LONG, NONCOMM_SHORT, - NONREPT_LONG, NONREPT_SHORT] + NONREPT_LONG, NONREPT_SHORT, + TRADERS_TOT, TRADERS_COMM_LONG, TRADERS_COMM_SHORT, + TRADERS_NONCOMM_LONG, TRADERS_NONCOMM_SHORT] def _cache_dir() -> Path: From 7e461b4def7cbafe0f7027d3c8b6cf7edf2ae0fc Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Mon, 13 Jul 2026 22:54:28 -0400 Subject: [PATCH 2/2] docs: update Legacy schema in README to include trader count columns --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 322189e..33e32e7 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,7 @@ The primary source for Legacy positioning data (CFTC Legacy Futures Report). **H > **Legacy Reports**: The Legacy reports are broken down by exchange. These reports have a futures only report and a combined futures and options report. Legacy reports break down the reportable open interest positions into two classifications: non-commercial and commercial traders. The `cotdata` pipeline strictly downloads the **Futures-only** reports (located at `https://www.cftc.gov/files/dea/history/dea_fut_xls_{YEAR}.zip`). > [!NOTE] -> **Column Subset**: While the raw CFTC `.xls` files contain [well over 100 columns](https://www.cftc.gov/MarketReports/CommitmentsofTraders/HistoricalViewable/cotvariableslegacy.html) (including spreading, concentration ratios, etc.), the producer pipeline explicitly discards them. The parquet files only maintain the exact 10-column subset listed below to keep the file sizes extremely small and strictly focused on what the downstream models require. To include additional data points from the raw reports, simply add the exact CFTC column name to the `TARGET_COLS` list inside `src/cotdata/providers/cftc.py`. +> **Column Subset**: While the raw CFTC `.xls` files contain [well over 100 columns](https://www.cftc.gov/MarketReports/CommitmentsofTraders/HistoricalViewable/cotvariableslegacy.html) (including spreading, concentration ratios, etc.), the producer pipeline explicitly discards them. The parquet files only maintain the exact 15-column subset listed below to keep the file sizes extremely small and strictly focused on what the downstream models require. To include additional data points from the raw reports, simply add the exact CFTC column name to the `TARGET_COLS` list inside `src/cotdata/providers/cftc.py`. | Column | Type | Description | |--------|------|-------------| @@ -200,6 +200,11 @@ The primary source for Legacy positioning data (CFTC Legacy Futures Report). **H | `NonComm_Positions_Short_All` | float | Non-Commercial (Large Speculator) Short positions. | | `NonRept_Positions_Long_All` | float | Non-Reportable (Small Speculator) Long positions. | | `NonRept_Positions_Short_All` | float | Non-Reportable (Small Speculator) Short positions. | +| `Traders_Tot_All` | float | Total number of reportable traders. | +| `Traders_Comm_Long_All` | float | Number of Commercial Long traders. | +| `Traders_Comm_Short_All` | float | Number of Commercial Short traders. | +| `Traders_NonComm_Long_All` | float | Number of Non-Commercial Long traders. | +| `Traders_NonComm_Short_All` | float | Number of Non-Commercial Short traders. | ### COT Disaggregated Data (`cot_disagg/{code}.parquet`) The primary source for entity-specific positioning and trader counts (CFTC Disaggregated Futures-Only Report). **History starts in 2006.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.