diff --git a/packages/census-aes-mcp/README.md b/packages/census-aes-mcp/README.md new file mode 100644 index 0000000..426a8eb --- /dev/null +++ b/packages/census-aes-mcp/README.md @@ -0,0 +1,79 @@ +# census-aes-mcp + +MCP server for US Census Bureau International Trade data (AES-derived exports +and Customs-derived imports). Monthly HS-granular US trade statistics. + +## Scope Clarification + +AES (Automated Export System) is the filing interface through which US +exporters submit Electronic Export Information (EEI). AES itself is not a +query API. Publicly queryable AES-derived data are served via the **Census +International Trade time-series API** (`api.census.gov/data/timeseries/intltrade/`), +which is what this server wraps. + +The server is named `census-aes-mcp` to reflect the AES provenance of the +export data, per user convention. It also exposes the paired import data +(which is *not* AES-derived but comes from Customs entry filings) because CMM +analysis typically requires both flows. + +## Tools + +| Tool | Description | +|------|-------------| +| `get_api_status` | Check API connectivity and key validity | +| `list_cmm_hs_codes` | Curated HS codes per critical mineral | +| `get_exports` | US exports by HS, year, month, destination | +| `get_imports` | US imports by HS, year, month, origin | +| `get_cmm_mineral_trade` | Curated CMM mineral trade (exports or imports) | + +## Configuration + +| Variable | Required | Description | +|----------|----------|-------------| +| `CENSUS_API_KEY` | **Yes** | Obtain at https://api.census.gov/data/key_signup.html | + +## Usage + +```sh +uv run census-aes-mcp +``` + +### Claude Desktop + +```json +{ + "mcpServers": { + "census-aes-mcp": { + "command": "uv", + "args": ["--directory", "/path/to/CMM_Tools", "run", "census-aes-mcp"], + "env": { + "CENSUS_API_KEY": "your-census-key" + } + } + } +} +``` + +## Comparison to Sibling MCP Servers + +| Server | Reporter | Partner detail | Classification | Frequency | Authoritative for | +|---|---|---|---|---|---| +| `uncomtrade-mcp` | All UN members | Yes (bilateral) | HS | Annual | Global bilateral flows | +| `usitc-mcp` | USA only | Yes | HTS (2–10 digit) | Annual (monthly via DataWeb queries) | US tariff-line detail, import-type distinction | +| `census-aes-mcp` | USA only | Yes | HS (6/10 digit) | Monthly | US monthly timeliness; AES-derived export disclosures | + +For CMM analyses requiring cross-validation, run the same query via +`uncomtrade-mcp` (USA reporter) and `census-aes-mcp` and compare — systematic +discrepancies often reflect timing adjustments, revision lags, or the UN's +in-kind aid inclusions. + +## Scope Notes + +- Confidentiality suppressions apply at low-volume HS × country × month cells; + expect missing values. The `_safe_float` parser normalizes these to `None`. +- The Census API uses HS codes without dots ("283691"), whereas USITC uses + dotted HTS ("2836.91.00"). Codes returned by this server are un-dotted. +- This v0.1.0 does not query `porths` (port-level) or `statenaics` (state-level); + add those endpoints in a v0.2.0 if state-sourced export analysis is needed. +- Schedule C (country) codes differ from ISO3; a `list_schedule_c_countries` + reference tool is a sensible v0.2.0 addition. diff --git a/packages/census-aes-mcp/pyproject.toml b/packages/census-aes-mcp/pyproject.toml new file mode 100644 index 0000000..1caa2ee --- /dev/null +++ b/packages/census-aes-mcp/pyproject.toml @@ -0,0 +1,21 @@ +[project] +name = "census-aes-mcp" +version = "0.1.0" +description = "MCP server for US Census Bureau International Trade (AES) import/export data" +requires-python = ">=3.10" +dependencies = [ + "mcp[cli]>=1.0.0", + "httpx>=0.27.0", + "pydantic>=2.0.0", + "python-dotenv>=1.0.0", +] + +[project.scripts] +census-aes-mcp = "census_aes_mcp.server:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/census_aes_mcp"] diff --git a/packages/census-aes-mcp/src/census_aes_mcp/__init__.py b/packages/census-aes-mcp/src/census_aes_mcp/__init__.py new file mode 100644 index 0000000..503d769 --- /dev/null +++ b/packages/census-aes-mcp/src/census_aes_mcp/__init__.py @@ -0,0 +1,5 @@ +"""Census AES MCP Server - US export/import data from the Automated Export System via Census API.""" + +from __future__ import annotations + +__version__ = "0.1.0" diff --git a/packages/census-aes-mcp/src/census_aes_mcp/client.py b/packages/census-aes-mcp/src/census_aes_mcp/client.py new file mode 100644 index 0000000..862b1ed --- /dev/null +++ b/packages/census-aes-mcp/src/census_aes_mcp/client.py @@ -0,0 +1,280 @@ +"""Async HTTP client for Census International Trade API (AES-derived). + +The Census International Trade time-series API publishes monthly trade data at +HS classification granularity. Export records are derived from the Automated +Export System (AES); import records are derived from Customs entry filings. + +Response format: Census returns a JSON array-of-arrays, first row is column +headers. We normalize to list[dict]. + +API reference: https://api.census.gov/data/timeseries/intltrade.html +Key signup: https://api.census.gov/data/key_signup.html +""" + +from __future__ import annotations + +import os +from typing import Any + +import httpx +from dotenv import load_dotenv + +from .models import CMM_HS_CODES, ExportRecord, ImportRecord + +load_dotenv() + + +class CensusAPIError(Exception): + """Raised for Census API errors.""" + + +class CensusClient: + """Client for Census International Trade time-series API.""" + + BASE_URL = "https://api.census.gov/data/timeseries/intltrade" + + # Default export variable set (HS classification) + DEFAULT_EXPORT_VARS = [ + "E_COMMODITY", + "E_COMMODITY_LDESC", + "ALL_VAL_MO", + "ALL_VAL_YR", + "CTY_CODE", + "CTY_NAME", + "UNIT_QY1", + "QTY_1_MO", + "UNIT_QY2", + "QTY_2_MO", + ] + + # Default import variable set (HS classification) + DEFAULT_IMPORT_VARS = [ + "I_COMMODITY", + "I_COMMODITY_LDESC", + "GEN_VAL_MO", + "GEN_VAL_YR", + "CON_VAL_MO", + "CIF_VAL_MO", + "CTY_CODE", + "CTY_NAME", + "UNIT_QY1", + "GEN_QY1_MO", + ] + + def __init__(self, api_key: str | None = None): + """Initialize client; api_key may be passed explicitly or via CENSUS_API_KEY.""" + self.api_key = api_key or os.getenv("CENSUS_API_KEY") + self.timeout = 90.0 + + def is_available(self) -> bool: + """Return True if an API key is configured.""" + return bool(self.api_key) + + def _require_key(self) -> None: + if not self.api_key: + raise CensusAPIError( + "CENSUS_API_KEY is not set. Register at " + "https://api.census.gov/data/key_signup.html and export CENSUS_API_KEY." + ) + + async def _get(self, endpoint: str, params: dict[str, Any] | None = None) -> list[list[Any]]: + """GET against Census API; returns raw array-of-arrays.""" + self._require_key() + query = dict(params or {}) + query["key"] = self.api_key + + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get(f"{self.BASE_URL}/{endpoint}", params=query) + if response.status_code == 204: + return [] + response.raise_for_status() + try: + data: list[list[Any]] = response.json() + except ValueError as e: + raise CensusAPIError( + f"Non-JSON response from Census API: {response.text[:300]}" + ) from e + return data + + @staticmethod + def _rows_to_dicts(data: list[list[Any]]) -> list[dict[str, Any]]: + """Convert Census array-of-arrays (header + rows) to list[dict].""" + if not data or len(data) < 2: + return [] + header = [str(c) for c in data[0]] + return [dict(zip(header, row)) for row in data[1:]] + + async def check_status(self) -> dict[str, Any]: + """Check Census API connectivity and key validity.""" + if not self.api_key: + return { + "status": "unauthorized", + "api_key_configured": False, + "message": "CENSUS_API_KEY not set", + } + try: + # Minimal query: one month of lithium carbonate exports + data = await self._get( + "exports/hs", + params={ + "get": "E_COMMODITY,E_COMMODITY_LDESC,ALL_VAL_MO", + "E_COMMODITY": "283691", + "YEAR": "2023", + "MONTH": "12", + "SUMMARY_LVL": "DET", + }, + ) + return { + "status": "connected", + "api_key_configured": True, + "sample_rows": max(0, len(data) - 1), + "message": "Census International Trade API reachable", + } + except httpx.HTTPStatusError as e: + if e.response.status_code in (401, 403): + return { + "status": "unauthorized", + "api_key_configured": True, + "message": "API key rejected", + } + return { + "status": "error", + "api_key_configured": True, + "message": f"HTTP {e.response.status_code}", + } + except (httpx.HTTPError, CensusAPIError, OSError) as e: + return {"status": "error", "message": str(e)} + + async def get_exports( + self, + hs_codes: list[str], + year: int, + month: str | int | None = None, + country_codes: list[str] | None = None, + summary_level: str = "DET", + ) -> list[ExportRecord]: + """Fetch US export records (AES-derived) for given HS codes. + + Args: + hs_codes: List of HS codes (6 or 10 digit). + year: Calendar year. + month: "01"-"12" or int 1-12; None for all months (YTD). + country_codes: Optional Schedule C country codes. + summary_level: "DET" (detailed) or "CTY" (country-level). + """ + params: dict[str, Any] = { + "get": ",".join(self.DEFAULT_EXPORT_VARS), + "YEAR": str(year), + "E_COMMODITY": ",".join(hs_codes), + "SUMMARY_LVL": summary_level, + } + if month is not None: + params["MONTH"] = f"{int(month):02d}" + if country_codes: + params["CTY_CODE"] = ",".join(country_codes) + + raw = await self._get("exports/hs", params=params) + rows = self._rows_to_dicts(raw) + records: list[ExportRecord] = [] + month_fallback = int(month) if month is not None else 0 + for row in rows: + try: + records.append( + ExportRecord( + year=int(row.get("YEAR") or year), + month=int(row.get("MONTH") or month_fallback), + hs_code=str(row.get("E_COMMODITY", "")), + hs_description=row.get("E_COMMODITY_LDESC"), + country_code=row.get("CTY_CODE"), + country_name=row.get("CTY_NAME"), + value_usd=_safe_float(row.get("ALL_VAL_MO")), + quantity_1=_safe_float(row.get("QTY_1_MO")), + quantity_1_unit=row.get("UNIT_QY1"), + quantity_2=_safe_float(row.get("QTY_2_MO")), + quantity_2_unit=row.get("UNIT_QY2"), + ) + ) + except (ValueError, TypeError): + continue + return records + + async def get_imports( + self, + hs_codes: list[str], + year: int, + month: str | int | None = None, + country_codes: list[str] | None = None, + summary_level: str = "DET", + ) -> list[ImportRecord]: + """Fetch US import records for given HS codes. + + Args: + hs_codes: List of HS codes (6 or 10 digit). + year: Calendar year. + month: Optional month. + country_codes: Optional Schedule C country codes. + summary_level: "DET" or "CTY". + """ + params: dict[str, Any] = { + "get": ",".join(self.DEFAULT_IMPORT_VARS), + "YEAR": str(year), + "I_COMMODITY": ",".join(hs_codes), + "SUMMARY_LVL": summary_level, + } + if month is not None: + params["MONTH"] = f"{int(month):02d}" + if country_codes: + params["CTY_CODE"] = ",".join(country_codes) + + raw = await self._get("imports/hs", params=params) + rows = self._rows_to_dicts(raw) + records: list[ImportRecord] = [] + month_fallback = int(month) if month is not None else 0 + for row in rows: + try: + records.append( + ImportRecord( + year=int(row.get("YEAR") or year), + month=int(row.get("MONTH") or month_fallback), + hs_code=str(row.get("I_COMMODITY", "")), + hs_description=row.get("I_COMMODITY_LDESC"), + country_code=row.get("CTY_CODE"), + country_name=row.get("CTY_NAME"), + value_usd=_safe_float(row.get("GEN_VAL_MO")), + value_cif_usd=_safe_float(row.get("CIF_VAL_MO")), + quantity_1=_safe_float(row.get("GEN_QY1_MO")), + quantity_1_unit=row.get("UNIT_QY1"), + ) + ) + except (ValueError, TypeError): + continue + return records + + async def get_critical_mineral_trade( + self, + mineral: str, + year: int, + month: str | int | None = None, + flow: str = "export", + country_codes: list[str] | None = None, + ) -> list[ExportRecord] | list[ImportRecord]: + """Fetch US trade data for a critical mineral using curated HS codes.""" + key = mineral.lower().replace(" ", "_") + codes = CMM_HS_CODES.get(key) + if not codes: + raise CensusAPIError( + f"Unknown mineral: {mineral}. Available: {', '.join(CMM_HS_CODES.keys())}" + ) + if flow == "export": + return await self.get_exports(codes, year, month=month, country_codes=country_codes) + return await self.get_imports(codes, year, month=month, country_codes=country_codes) + + +def _safe_float(value: Any) -> float | None: + """Convert Census string values to float, returning None for missing markers.""" + if value is None or value in ("", ".", "N/A", "null"): + return None + try: + return float(value) + except (ValueError, TypeError): + return None diff --git a/packages/census-aes-mcp/src/census_aes_mcp/models.py b/packages/census-aes-mcp/src/census_aes_mcp/models.py new file mode 100644 index 0000000..4b38d9b --- /dev/null +++ b/packages/census-aes-mcp/src/census_aes_mcp/models.py @@ -0,0 +1,75 @@ +"""Pydantic models and reference data for Census International Trade (AES-derived) API. + +Scope: US Census Bureau's International Trade time-series API, which publishes +export records originating from the Automated Export System (AES) and import +records from Customs entry filings. This is the publicly queryable face of AES +data; the AES filing system itself is not a query interface. + +API reference: https://www.census.gov/foreign-trade/reference/guides/Guide%20to%20International%20Trade%20Datasets.pdf +Endpoint base: https://api.census.gov/data/timeseries/intltrade/ +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class ExportRecord(BaseModel): + """Single monthly export observation (AES-derived) at HS granularity.""" + + year: int = Field(description="Calendar year") + month: int = Field(description="Calendar month (1-12)") + hs_code: str = Field(description="HS commodity code (2/4/6/10-digit)") + hs_description: str | None = Field(default=None, description="HS description") + country_code: str | None = Field( + default=None, description="Destination country Schedule C code" + ) + country_name: str | None = Field(default=None, description="Destination country name") + value_usd: float | None = Field(default=None, description="Free-alongside-ship (FAS) value USD") + quantity_1: float | None = Field(default=None, description="First quantity measure") + quantity_1_unit: str | None = Field(default=None, description="First quantity unit") + quantity_2: float | None = Field(default=None, description="Second quantity measure") + quantity_2_unit: str | None = Field(default=None, description="Second quantity unit") + + +class ImportRecord(BaseModel): + """Single monthly import observation at HS granularity.""" + + year: int = Field(description="Calendar year") + month: int = Field(description="Calendar month (1-12)") + hs_code: str = Field(description="HS commodity code") + hs_description: str | None = Field(default=None, description="HS description") + country_code: str | None = Field(default=None, description="Source country Schedule C code") + country_name: str | None = Field(default=None, description="Source country name") + value_usd: float | None = Field(default=None, description="General imports customs value USD") + value_cif_usd: float | None = Field(default=None, description="CIF value USD where available") + quantity_1: float | None = Field(default=None, description="First quantity measure") + quantity_1_unit: str | None = Field(default=None, description="First quantity unit") + + +# ── CMM-focused HS codes for Census (uses 6-digit HS matching UN Comtrade) ─── +# Same HS codes as uncomtrade-mcp for cross-source reconciliation. +CMM_HS_CODES: dict[str, list[str]] = { + "lithium": ["253090", "282520", "283691", "850650"], + "cobalt": ["260500", "282200", "810520", "810590"], + "rare_earth": ["280530", "284610", "284690"], + "graphite": ["250410", "250490", "380110"], + "nickel": ["260400", "750110", "750210", "281122"], + "manganese": ["260200", "811100"], + "gallium": ["811292"], + "germanium": ["811299"], + "copper": ["740200", "740311"], +} + + +MINERAL_NAMES: dict[str, str] = { + "lithium": "Lithium (Li)", + "cobalt": "Cobalt (Co)", + "rare_earth": "Rare Earth Elements (all)", + "graphite": "Graphite (Gr)", + "nickel": "Nickel (Ni)", + "manganese": "Manganese (Mn)", + "gallium": "Gallium (Ga)", + "germanium": "Germanium (Ge)", + "copper": "Copper (Cu)", +} diff --git a/packages/census-aes-mcp/src/census_aes_mcp/py.typed b/packages/census-aes-mcp/src/census_aes_mcp/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/packages/census-aes-mcp/src/census_aes_mcp/server.py b/packages/census-aes-mcp/src/census_aes_mcp/server.py new file mode 100644 index 0000000..7e756d2 --- /dev/null +++ b/packages/census-aes-mcp/src/census_aes_mcp/server.py @@ -0,0 +1,214 @@ +"""MCP server for US Census International Trade (AES-derived) data.""" + +from __future__ import annotations + +from mcp.server.fastmcp import FastMCP + +from .client import CensusAPIError, CensusClient +from .models import CMM_HS_CODES, MINERAL_NAMES + +mcp = FastMCP( + name="Census AES", + instructions="""Census AES MCP Server provides access to US export and import +data published by the US Census Bureau's International Trade API. Export +records are derived from the Automated Export System (AES); import records +are derived from Customs entry filings. Data are monthly at HS classification +granularity (6- and 10-digit). + +Key capabilities: +- Fetch US exports or imports by HS code, year, month, and partner country +- Query pre-configured critical-minerals HS sets +- Detailed vs. country-level summary levels +- Both FAS (export) and CIF/customs (import) valuations + +Requires CENSUS_API_KEY. Register at https://api.census.gov/data/key_signup.html. + +Scope clarification: AES itself is a filing system (exporters submit EEI +records); this server queries the publicly available Census International +Trade data feed, which is AES-derived for exports.""", +) + + +def get_client() -> CensusClient: + """Return a fresh CensusClient instance.""" + return CensusClient() + + +# ============================================================================= +# Overview +# ============================================================================= + + +@mcp.tool() +async def get_api_status() -> dict: + """Check Census API connectivity and key validity.""" + client = get_client() + return await client.check_status() + + +@mcp.tool() +async def list_cmm_hs_codes() -> dict: + """List curated CMM-focused HS codes for each critical mineral.""" + items = [ + { + "id": key, + "name": MINERAL_NAMES.get(key, key), + "hs_codes": codes, + "count": len(codes), + } + for key, codes in CMM_HS_CODES.items() + ] + return { + "count": len(items), + "minerals": items, + "usage": "Use get_cmm_mineral_trade(mineral='lithium', year=2023, flow='export')", + } + + +# ============================================================================= +# Exports +# ============================================================================= + + +@mcp.tool() +async def get_exports( + hs_codes: list[str], + year: int, + month: int | None = None, + country_codes: list[str] | None = None, + summary_level: str = "DET", +) -> dict: + """Fetch US export records (AES-derived) for given HS codes. + + Args: + hs_codes: HS codes (6 or 10 digit), e.g., ["283691", "282520"]. + year: Calendar year (e.g., 2023). + month: Optional month 1-12; omit for full year. + country_codes: Optional Schedule C codes to filter destinations. + summary_level: "DET" (detailed) or "CTY" (country-level aggregate). + """ + client = get_client() + try: + records = await client.get_exports( + hs_codes=hs_codes, + year=year, + month=month, + country_codes=country_codes, + summary_level=summary_level, + ) + except CensusAPIError as e: + return {"error": str(e)} + return { + "query": { + "hs_codes": hs_codes, + "year": year, + "month": month, + "country_codes": country_codes, + "summary_level": summary_level, + }, + "count": len(records), + "records": [r.model_dump() for r in records], + } + + +# ============================================================================= +# Imports +# ============================================================================= + + +@mcp.tool() +async def get_imports( + hs_codes: list[str], + year: int, + month: int | None = None, + country_codes: list[str] | None = None, + summary_level: str = "DET", +) -> dict: + """Fetch US import records for given HS codes. + + Args: + hs_codes: HS codes (6 or 10 digit). + year: Calendar year. + month: Optional month 1-12. + country_codes: Optional Schedule C codes to filter origins. + summary_level: "DET" or "CTY". + """ + client = get_client() + try: + records = await client.get_imports( + hs_codes=hs_codes, + year=year, + month=month, + country_codes=country_codes, + summary_level=summary_level, + ) + except CensusAPIError as e: + return {"error": str(e)} + return { + "query": { + "hs_codes": hs_codes, + "year": year, + "month": month, + "country_codes": country_codes, + "summary_level": summary_level, + }, + "count": len(records), + "records": [r.model_dump() for r in records], + } + + +# ============================================================================= +# Critical Minerals Convenience +# ============================================================================= + + +@mcp.tool() +async def get_cmm_mineral_trade( + mineral: str, + year: int, + flow: str = "export", + month: int | None = None, + country_codes: list[str] | None = None, +) -> dict: + """Fetch US trade data for a critical mineral using curated HS codes. + + Available minerals: lithium, cobalt, rare_earth, graphite, nickel, + manganese, gallium, germanium, copper. + + Args: + mineral: Mineral key. + year: Calendar year. + flow: "export" or "import". + month: Optional month 1-12. + country_codes: Optional Schedule C country codes. + """ + client = get_client() + try: + records = await client.get_critical_mineral_trade( + mineral=mineral, + year=year, + month=month, + flow=flow, + country_codes=country_codes, + ) + except CensusAPIError as e: + return {"error": str(e)} + key = mineral.lower().replace(" ", "_") + return { + "mineral": MINERAL_NAMES.get(key, mineral), + "hs_codes_queried": CMM_HS_CODES.get(key, []), + "flow": flow, + "year": year, + "month": month, + "count": len(records), + "records": [r.model_dump() for r in records], + } + + +def main(): + """Run the MCP server.""" + mcp.run() + + +if __name__ == "__main__": + main() diff --git a/packages/census-aes-mcp/tests/conftest.py b/packages/census-aes-mcp/tests/conftest.py new file mode 100644 index 0000000..f4f2771 --- /dev/null +++ b/packages/census-aes-mcp/tests/conftest.py @@ -0,0 +1,13 @@ +"""Shared pytest fixtures for census-aes-mcp.""" + +from __future__ import annotations + +import pytest + +from census_aes_mcp.client import CensusClient + + +@pytest.fixture +def client() -> CensusClient: + """Return a CensusClient instance for tests.""" + return CensusClient() diff --git a/packages/census-aes-mcp/tests/test_models.py b/packages/census-aes-mcp/tests/test_models.py new file mode 100644 index 0000000..a2c2d6e --- /dev/null +++ b/packages/census-aes-mcp/tests/test_models.py @@ -0,0 +1,58 @@ +"""Unit tests for census_aes_mcp.models and client helpers.""" + +from __future__ import annotations + +from census_aes_mcp.client import CensusClient, _safe_float +from census_aes_mcp.models import CMM_HS_CODES, MINERAL_NAMES, ExportRecord + + +def test_cmm_hs_codes_nonempty(): + assert len(CMM_HS_CODES) > 5 + assert "lithium" in CMM_HS_CODES + + +def test_mineral_names_match_hs_keys(): + for key in CMM_HS_CODES: + assert key in MINERAL_NAMES + + +def test_safe_float_handles_missing(): + assert _safe_float(None) is None + assert _safe_float("") is None + assert _safe_float(".") is None + assert _safe_float("N/A") is None + assert _safe_float("1234.5") == 1234.5 + assert _safe_float("notanumber") is None + + +def test_rows_to_dicts_empty(): + assert CensusClient._rows_to_dicts([]) == [] + assert CensusClient._rows_to_dicts([["header"]]) == [] + + +def test_rows_to_dicts_basic(): + data = [ + ["YEAR", "MONTH", "ALL_VAL_MO"], + ["2023", "12", "1000000"], + ["2023", "11", "950000"], + ] + result = CensusClient._rows_to_dicts(data) + assert len(result) == 2 + assert result[0]["YEAR"] == "2023" + assert result[0]["ALL_VAL_MO"] == "1000000" + + +def test_export_record_roundtrip(): + rec = ExportRecord( + year=2023, + month=12, + hs_code="283691", + hs_description="Lithium carbonates", + country_code="3370", + country_name="Chile", + value_usd=1_234_567.89, + quantity_1=500.0, + quantity_1_unit="kg", + ) + assert rec.hs_code == "283691" + assert rec.model_dump()["value_usd"] == 1_234_567.89 diff --git a/packages/fred-mcp/README.md b/packages/fred-mcp/README.md new file mode 100644 index 0000000..07f9993 --- /dev/null +++ b/packages/fred-mcp/README.md @@ -0,0 +1,67 @@ +# fred-mcp + +MCP server for the Federal Reserve Economic Data (FRED) API. Provides +macroeconomic and commodity-price time series relevant to critical-minerals +supply-chain analysis. + +## Tools + +| Tool | Description | +|------|-------------| +| `get_api_status` | Check FRED API connectivity and key validity | +| `list_cmm_series` | List curated CMM-relevant series (commodity prices, production, FX, etc.) | +| `search_series` | Search the FRED series catalogue by keyword | +| `get_series_metadata` | Fetch metadata (units, frequency, revision dates) for a series | +| `get_observations` | Fetch observations for any FRED series by ID | +| `get_cmm_dashboard` | Latest observation for every curated CMM series | +| `list_categories` | Browse FRED category taxonomy | +| `list_category_series` | List series under a specific category | + +## Configuration + +| Variable | Required | Description | +|----------|----------|-------------| +| `FRED_API_KEY` | **Yes** | Obtain at https://fred.stlouisfed.org/docs/api/api_key.html | + +## Usage + +```sh +uv run fred-mcp +``` + +### Claude Desktop + +```json +{ + "mcpServers": { + "fred-mcp": { + "command": "uv", + "args": ["--directory", "/path/to/CMM_Tools", "run", "fred-mcp"], + "env": { + "FRED_API_KEY": "your-fred-key" + } + } + } +} +``` + +## Curated CMM Series + +The `list_cmm_series` tool exposes a curated set spanning: + +- **Commodity prices**: PALLFNFINDEXM, PMETAINDEXM, PCOPPUSDM, PNICKUSDM, PALUMINUMUSDM, PIORECRUSDM +- **Production**: INDPRO, IPMINE, IPMANSICS +- **Trade prices**: IR (import PI), IQ (export PI) +- **Logistics**: WPU301 (transportation PPI) +- **FX**: DEXCHUS (CNY/USD), DEXUSEU (USD/EUR) +- **Rates**: DGS10 (10Y Treasury) + +Edit `CMM_FRED_SERIES` in `models.py` to extend. + +## Scope Notes + +- FRED observations with value `"."` (missing) are normalized to `None`. +- FRED imposes a rate limit of 120 requests per 60-second window. The + `get_cmm_dashboard` tool paces requests with `asyncio.sleep(0.1)` but still + issues ~15 requests; adjust if you extend the curated set substantially. +- ALFRED (archival / real-time vintages) is not yet exposed; open an issue if needed. diff --git a/packages/fred-mcp/pyproject.toml b/packages/fred-mcp/pyproject.toml new file mode 100644 index 0000000..2a9768b --- /dev/null +++ b/packages/fred-mcp/pyproject.toml @@ -0,0 +1,21 @@ +[project] +name = "fred-mcp" +version = "0.1.0" +description = "MCP server for Federal Reserve Economic Data (FRED) time series" +requires-python = ">=3.10" +dependencies = [ + "mcp[cli]>=1.0.0", + "httpx>=0.27.0", + "pydantic>=2.0.0", + "python-dotenv>=1.0.0", +] + +[project.scripts] +fred-mcp = "fred_mcp.server:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/fred_mcp"] diff --git a/packages/fred-mcp/src/fred_mcp/__init__.py b/packages/fred-mcp/src/fred_mcp/__init__.py new file mode 100644 index 0000000..b8a19f8 --- /dev/null +++ b/packages/fred-mcp/src/fred_mcp/__init__.py @@ -0,0 +1,5 @@ +"""FRED MCP Server - Federal Reserve Economic Data access for macro/commodity time series.""" + +from __future__ import annotations + +__version__ = "0.1.0" diff --git a/packages/fred-mcp/src/fred_mcp/client.py b/packages/fred-mcp/src/fred_mcp/client.py new file mode 100644 index 0000000..2f3d854 --- /dev/null +++ b/packages/fred-mcp/src/fred_mcp/client.py @@ -0,0 +1,185 @@ +"""Async HTTP client for FRED (Federal Reserve Economic Data). + +API reference: https://fred.stlouisfed.org/docs/api/fred/ +Authentication: FRED_API_KEY env var (register at https://fred.stlouisfed.org/docs/api/api_key.html). +""" + +from __future__ import annotations + +import os +from typing import Any + +import httpx +from dotenv import load_dotenv + +from .models import Observation, SeriesMetadata + +load_dotenv() + + +class FredAPIError(Exception): + """Raised for FRED API errors.""" + + +class FredClient: + """Client for the FRED API (v2 JSON endpoints).""" + + BASE_URL = "https://api.stlouisfed.org/fred" + + def __init__(self, api_key: str | None = None): + """Initialize client; api_key may be passed explicitly or via FRED_API_KEY.""" + self.api_key = api_key or os.getenv("FRED_API_KEY") + self.timeout = 60.0 + + def is_available(self) -> bool: + """Return True if an API key is configured.""" + return bool(self.api_key) + + def _require_key(self) -> None: + if not self.api_key: + raise FredAPIError( + "FRED_API_KEY is not set. Register at " + "https://fred.stlouisfed.org/docs/api/api_key.html and export FRED_API_KEY." + ) + + async def _request(self, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + """GET against a FRED JSON endpoint.""" + self._require_key() + query = dict(params or {}) + query["api_key"] = self.api_key + query.setdefault("file_type", "json") + + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get(f"{self.BASE_URL}/{endpoint}", params=query) + response.raise_for_status() + data: dict[str, Any] = response.json() + return data + + async def check_status(self) -> dict[str, Any]: + """Check API connectivity with a lightweight series lookup.""" + if not self.api_key: + return { + "status": "unauthorized", + "api_key_configured": False, + "message": "FRED_API_KEY not set", + } + try: + await self._request("series", params={"series_id": "GDPC1"}) + return { + "status": "connected", + "api_key_configured": True, + "message": "FRED API is accessible", + } + except httpx.TimeoutException: + return {"status": "timeout", "message": "Request timed out"} + except (httpx.HTTPError, OSError, ValueError) as e: + return {"status": "error", "message": str(e)} + + async def get_series_metadata(self, series_id: str) -> SeriesMetadata | None: + """Fetch metadata for a single series.""" + data = await self._request("series", params={"series_id": series_id}) + items = data.get("seriess") or data.get("series") or [] + if not items: + return None + raw = items[0] + return SeriesMetadata( + id=raw.get("id"), + title=raw.get("title", ""), + units=raw.get("units"), + frequency=raw.get("frequency"), + seasonalAdjustment=raw.get("seasonal_adjustment"), + observation_start=raw.get("observation_start"), + observation_end=raw.get("observation_end"), + last_updated=raw.get("last_updated"), + notes=raw.get("notes"), + ) + + async def search_series( + self, query: str, limit: int = 25, order_by: str = "popularity" + ) -> list[dict[str, Any]]: + """Search series catalogue by keyword.""" + data = await self._request( + "series/search", + params={ + "search_text": query, + "limit": max(1, min(limit, 1000)), + "order_by": order_by, + }, + ) + series = data.get("seriess") or [] + return [ + { + "id": s.get("id"), + "title": s.get("title"), + "units": s.get("units"), + "frequency": s.get("frequency"), + "observation_start": s.get("observation_start"), + "observation_end": s.get("observation_end"), + "popularity": s.get("popularity"), + "notes": s.get("notes"), + } + for s in series + ] + + async def get_observations( + self, + series_id: str, + observation_start: str | None = None, + observation_end: str | None = None, + frequency: str | None = None, + aggregation_method: str | None = None, + limit: int = 1000, + ) -> list[Observation]: + """ + Fetch observations for a series. + + Args: + series_id: FRED series ID (e.g., "PCOPPUSDM"). + observation_start: Start date (YYYY-MM-DD). + observation_end: End date (YYYY-MM-DD). + frequency: Optional aggregation target ("m","q","a", etc.) + aggregation_method: avg | sum | eop (end of period). + limit: Max observations (FRED max 100000). + """ + params: dict[str, Any] = {"series_id": series_id, "limit": min(limit, 100000)} + if observation_start: + params["observation_start"] = observation_start + if observation_end: + params["observation_end"] = observation_end + if frequency: + params["frequency"] = frequency + if aggregation_method: + params["aggregation_method"] = aggregation_method + + data = await self._request("series/observations", params=params) + records: list[Observation] = [] + for obs in data.get("observations", []): + raw_val = obs.get("value") + try: + val: float | None = float(raw_val) if raw_val not in (".", "", None) else None + except (ValueError, TypeError): + val = None + records.append( + Observation( + date=obs.get("date", ""), + value=val, + realtime_start=obs.get("realtime_start"), + realtime_end=obs.get("realtime_end"), + ) + ) + return records + + async def list_categories(self, category_id: int = 0) -> list[dict[str, Any]]: + """List children of a FRED category (0 = root).""" + data = await self._request("category/children", params={"category_id": category_id}) + return data.get("categories", []) + + async def list_category_series( + self, category_id: int, limit: int = 100 + ) -> list[dict[str, Any]]: + """List series under a specific category.""" + data = await self._request( + "category/series", + params={"category_id": category_id, "limit": min(limit, 1000)}, + ) + return data.get("seriess", []) diff --git a/packages/fred-mcp/src/fred_mcp/models.py b/packages/fred-mcp/src/fred_mcp/models.py new file mode 100644 index 0000000..ee534d7 --- /dev/null +++ b/packages/fred-mcp/src/fred_mcp/models.py @@ -0,0 +1,113 @@ +"""Pydantic models and curated series lists for FRED.""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class SeriesMetadata(BaseModel): + """Metadata for a FRED time series.""" + + id: str = Field(description="FRED series ID (e.g., 'GDPC1')") + title: str = Field(description="Series title") + units: str | None = Field(default=None, description="Units of measurement") + frequency: str | None = Field(default=None, description="Reporting frequency") + seasonal_adjustment: str | None = Field( + default=None, alias="seasonalAdjustment", description="Seasonal adjustment code" + ) + observation_start: str | None = Field( + default=None, alias="observation_start", description="First observation date" + ) + observation_end: str | None = Field( + default=None, alias="observation_end", description="Latest observation date" + ) + last_updated: str | None = Field( + default=None, alias="last_updated", description="Last revision timestamp" + ) + notes: str | None = Field(default=None, description="Source/notes") + + class Config: + populate_by_name = True + + +class Observation(BaseModel): + """A single observation from a FRED series.""" + + date: str = Field(description="Observation date (YYYY-MM-DD)") + value: float | None = Field(default=None, description="Observation value; None if missing") + realtime_start: str | None = Field(default=None, description="Real-time start date") + realtime_end: str | None = Field(default=None, description="Real-time end date") + + +# ── CMM-relevant curated series ────────────────────────────────────────────── +# Series IDs that support critical-minerals supply-chain analysis: commodity +# price indices, industrial production, manufacturing PMI proxies, import/export +# price indices, and freight/shipping cost proxies. +CMM_FRED_SERIES: dict[str, dict[str, str]] = { + # Commodity price indices (Global Commodity Price Index family - PALLFNFINDEXM) + "PALLFNFINDEXM": { + "title": "Global Price Index of All Commodities", + "category": "commodity_prices", + }, + "PMETAINDEXM": { + "title": "Global Price Index of Industrial Materials (Metals)", + "category": "commodity_prices", + }, + "PCOPPUSDM": { + "title": "Global Price of Copper (USD/metric ton)", + "category": "commodity_prices", + }, + "PNICKUSDM": { + "title": "Global Price of Nickel (USD/metric ton)", + "category": "commodity_prices", + }, + "PALUMINUMUSDM": { + "title": "Global Price of Aluminum (USD/metric ton)", + "category": "commodity_prices", + }, + "PIORECRUSDM": { + "title": "Global Price of Iron Ore (USD/metric ton)", + "category": "commodity_prices", + }, + # Industrial production + "INDPRO": { + "title": "Industrial Production: Total Index", + "category": "production", + }, + "IPMINE": { + "title": "Industrial Production: Mining", + "category": "production", + }, + "IPMANSICS": { + "title": "Industrial Production: Manufacturing (SIC)", + "category": "production", + }, + # Trade / prices + "IR": { + "title": "Import Price Index (All Commodities)", + "category": "trade_prices", + }, + "IQ": { + "title": "Export Price Index (All Commodities)", + "category": "trade_prices", + }, + # Freight / shipping proxies + "WPU301": { + "title": "Producer Price Index: Transportation Services", + "category": "logistics", + }, + # FX (for trade valuation) + "DEXCHUS": { + "title": "China / U.S. Foreign Exchange Rate (CNY per USD)", + "category": "fx", + }, + "DEXUSEU": { + "title": "U.S. / Euro Foreign Exchange Rate (USD per EUR)", + "category": "fx", + }, + # Interest rates (capital cost for CMM projects) + "DGS10": { + "title": "10-Year Treasury Constant Maturity Rate", + "category": "rates", + }, +} diff --git a/packages/fred-mcp/src/fred_mcp/py.typed b/packages/fred-mcp/src/fred_mcp/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/packages/fred-mcp/src/fred_mcp/server.py b/packages/fred-mcp/src/fred_mcp/server.py new file mode 100644 index 0000000..f1e973b --- /dev/null +++ b/packages/fred-mcp/src/fred_mcp/server.py @@ -0,0 +1,244 @@ +"""MCP server for FRED (Federal Reserve Economic Data).""" + +from __future__ import annotations + +import asyncio + +import httpx +from mcp.server.fastmcp import FastMCP + +from .client import FredClient +from .models import CMM_FRED_SERIES + +mcp = FastMCP( + name="FRED", + instructions="""FRED MCP Server provides access to the Federal Reserve Economic +Data (FRED) time series API. Useful for macroeconomic context: commodity price +indices, industrial production, import/export price indices, exchange rates, +and freight-cost proxies relevant to critical-minerals supply chains. + +Key capabilities: +- Fetch observations for any FRED series by ID +- Search series catalogue by keyword +- Retrieve series metadata (units, frequency, revision dates) +- Browse categorical taxonomy +- Curated CMM series bundle for rapid profiling + +Requires FRED_API_KEY. Register at https://fred.stlouisfed.org/docs/api/api_key.html.""", +) + + +def get_client() -> FredClient: + """Return a fresh FredClient instance.""" + return FredClient() + + +# ============================================================================= +# Overview Tools +# ============================================================================= + + +@mcp.tool() +async def get_api_status() -> dict: + """Check FRED API connectivity and API key validity.""" + client = get_client() + return await client.check_status() + + +@mcp.tool() +async def list_cmm_series() -> dict: + """List the curated CMM-relevant FRED series (commodity prices, production, trade prices, FX, rates).""" + series = [ + {"id": series_id, "title": meta["title"], "category": meta["category"]} + for series_id, meta in CMM_FRED_SERIES.items() + ] + categories: dict[str, int] = {} + for s in series: + categories[s["category"]] = categories.get(s["category"], 0) + 1 + return { + "count": len(series), + "series": series, + "by_category": categories, + "usage": "Use get_observations(series_id='PCOPPUSDM', observation_start='2020-01-01')", + } + + +# ============================================================================= +# Search and Metadata +# ============================================================================= + + +@mcp.tool() +async def search_series(query: str, limit: int = 25, order_by: str = "popularity") -> dict: + """Search the FRED series catalogue by keyword. + + Args: + query: Search text (e.g., "copper price", "lithium", "industrial production"). + limit: Max results (1-1000). + order_by: Sort order ("popularity", "search_rank", "last_updated", "frequency"). + + Returns: + List of matching series with IDs, titles, units, and temporal coverage. + """ + client = get_client() + results = await client.search_series(query, limit=limit, order_by=order_by) + return {"query": query, "count": len(results), "series": results} + + +@mcp.tool() +async def get_series_metadata(series_id: str) -> dict: + """Fetch metadata for a single FRED series. + + Args: + series_id: FRED series identifier (e.g., "GDPC1", "PCOPPUSDM"). + """ + client = get_client() + meta = await client.get_series_metadata(series_id) + if meta is None: + return {"error": f"Series not found: {series_id}"} + return meta.model_dump() + + +# ============================================================================= +# Observations +# ============================================================================= + + +@mcp.tool() +async def get_observations( + series_id: str, + observation_start: str | None = None, + observation_end: str | None = None, + frequency: str | None = None, + aggregation_method: str | None = None, + max_records: int = 1000, +) -> dict: + """Fetch observations for a FRED series. + + Args: + series_id: FRED series ID. + observation_start: Start date (YYYY-MM-DD); omit for earliest. + observation_end: End date (YYYY-MM-DD); omit for latest. + frequency: Optional aggregation frequency: d, w, bw, m, q, sa, a. + aggregation_method: avg | sum | eop. + max_records: Maximum observations to return. + + Returns: + Dict with query, count, and observations list. + """ + client = get_client() + records = await client.get_observations( + series_id=series_id, + observation_start=observation_start, + observation_end=observation_end, + frequency=frequency, + aggregation_method=aggregation_method, + limit=max_records, + ) + return { + "query": { + "series_id": series_id, + "observation_start": observation_start, + "observation_end": observation_end, + "frequency": frequency, + }, + "count": len(records), + "observations": [r.model_dump() for r in records], + } + + +@mcp.tool() +async def get_cmm_dashboard( + observation_start: str = "2015-01-01", + observation_end: str | None = None, + category: str | None = None, +) -> dict: + """Fetch the curated CMM dashboard: all curated series, most recent observation per series. + + Args: + observation_start: Start date for backfill (default 2015-01-01). + observation_end: Optional end date. + category: Optional category filter (commodity_prices, production, trade_prices, + logistics, fx, rates). + + Returns: + Dict keyed by series_id with latest observation, units, frequency, and timestamp. + """ + client = get_client() + target_series = ( + {k: v for k, v in CMM_FRED_SERIES.items() if v["category"] == category} + if category + else CMM_FRED_SERIES + ) + + # Rate-limit concurrent FRED calls; pairs obs + metadata fetches per series. + sem = asyncio.Semaphore(5) + + async def fetch_one(series_id: str, meta: dict) -> tuple[str, dict]: + async with sem: + try: + records, metadata = await asyncio.gather( + client.get_observations( + series_id=series_id, + observation_start=observation_start, + observation_end=observation_end, + limit=5000, + ), + client.get_series_metadata(series_id), + ) + except (httpx.HTTPError, OSError, ValueError) as e: + return series_id, {"error": str(e), "title": meta["title"]} + latest = None + for r in reversed(records): + if r.value is not None: + latest = {"date": r.date, "value": r.value} + break + return series_id, { + "title": meta["title"], + "category": meta["category"], + "latest_observation": latest, + "units": metadata.units if metadata else None, + "frequency": metadata.frequency if metadata else None, + "observation_count": len(records), + } + + results = await asyncio.gather(*(fetch_one(sid, meta) for sid, meta in target_series.items())) + dashboard: dict[str, dict] = dict(results) + + return { + "observation_start": observation_start, + "observation_end": observation_end, + "category_filter": category, + "series_count": len(dashboard), + "dashboard": dashboard, + } + + +# ============================================================================= +# Taxonomy +# ============================================================================= + + +@mcp.tool() +async def list_categories(category_id: int = 0) -> dict: + """List children of a FRED category (0 = root).""" + client = get_client() + categories = await client.list_categories(category_id) + return {"parent_id": category_id, "count": len(categories), "categories": categories} + + +@mcp.tool() +async def list_category_series(category_id: int, limit: int = 100) -> dict: + """List series belonging to a specific category.""" + client = get_client() + series = await client.list_category_series(category_id, limit=limit) + return {"category_id": category_id, "count": len(series), "series": series} + + +def main(): + """Run the MCP server.""" + mcp.run() + + +if __name__ == "__main__": + main() diff --git a/packages/fred-mcp/tests/conftest.py b/packages/fred-mcp/tests/conftest.py new file mode 100644 index 0000000..918f0b2 --- /dev/null +++ b/packages/fred-mcp/tests/conftest.py @@ -0,0 +1,13 @@ +"""Shared pytest fixtures for fred-mcp.""" + +from __future__ import annotations + +import pytest + +from fred_mcp.client import FredClient + + +@pytest.fixture +def client() -> FredClient: + """Return a FredClient instance for tests.""" + return FredClient() diff --git a/packages/fred-mcp/tests/test_models.py b/packages/fred-mcp/tests/test_models.py new file mode 100644 index 0000000..d0215f6 --- /dev/null +++ b/packages/fred-mcp/tests/test_models.py @@ -0,0 +1,28 @@ +"""Unit tests for fred_mcp.models.""" + +from __future__ import annotations + +from fred_mcp.models import CMM_FRED_SERIES, Observation, SeriesMetadata + + +def test_cmm_series_nonempty(): + assert len(CMM_FRED_SERIES) > 10 + assert "PCOPPUSDM" in CMM_FRED_SERIES + assert CMM_FRED_SERIES["PCOPPUSDM"]["category"] == "commodity_prices" + + +def test_observation_missing_value_none(): + obs = Observation(date="2022-01-01", value=None) + assert obs.value is None + assert obs.date == "2022-01-01" + + +def test_series_metadata_roundtrip(): + meta = SeriesMetadata( + id="GDPC1", + title="Real Gross Domestic Product", + units="Billions of Chained 2017 Dollars", + frequency="Quarterly", + ) + assert meta.id == "GDPC1" + assert meta.model_dump()["frequency"] == "Quarterly" diff --git a/packages/usitc-mcp/README.md b/packages/usitc-mcp/README.md new file mode 100644 index 0000000..88d027a --- /dev/null +++ b/packages/usitc-mcp/README.md @@ -0,0 +1,79 @@ +# usitc-mcp + +MCP server for the US International Trade Commission (USITC) DataWeb API. +Provides US-specific import and export statistics at HTS (Harmonized Tariff +Schedule of the United States) granularity. + +## Relationship to `uncomtrade-mcp` + +| Aspect | `uncomtrade-mcp` | `usitc-mcp` | +|---|---|---| +| Reporting perspective | All UN members | United States only | +| Classification granularity | HS (2/4/6-digit) | HTS (up to 10-digit) | +| Trade-measure varieties | Imports / Exports | General imports, Imports for consumption, Domestic exports, Total exports | +| Use for CMM | Global bilateral flows | US supply-chain specifics (FTZ, stockpile, tariff exposure) | + +Use both together for cross-validation and disaggregated US views. + +## Tools + +| Tool | Description | +|------|-------------| +| `get_api_status` | Check API connectivity and token validity | +| `list_cmm_hts_codes` | Curated HTS codes per critical mineral | +| `search_hts` | Search HTS classification by keyword or code prefix | +| `get_trade_data` | Fetch trade data by HTS, year, partner, data type | +| `get_critical_mineral_trade` | Query US trade for a curated CMM mineral | +| `compare_import_types` | General imports vs. imports for consumption side-by-side | + +## Configuration + +| Variable | Required | Description | +|----------|----------|-------------| +| `USITC_API_TOKEN` | **Yes** | Bearer token from https://dataweb.usitc.gov → Account → API Access | + +## Usage + +```sh +uv run usitc-mcp +``` + +### Claude Desktop + +```json +{ + "mcpServers": { + "usitc-mcp": { + "command": "uv", + "args": ["--directory", "/path/to/CMM_Tools", "run", "usitc-mcp"], + "env": { + "USITC_API_TOKEN": "your-bearer-token" + } + } + } +} +``` + +## Important Caveat — Schema Validation Required + +USITC DataWeb's v2 API does **not** publish a canonical OpenAPI / JSON-schema +document. The request/response shapes implemented in `client.py` reflect the +publicly observed structure as of 2026-Q1. Before relying on query results in +scientific outputs, Nancy should: + +1. Obtain the authenticated token +2. Issue a known-good query (e.g., HTS `2836.91.00` imports from Chile, 2023) +3. Inspect the actual response envelope +4. Reconcile field names in `client.py::_post` parsing (`year`, `hts_code`, + `value`, `quantity_1`, etc.) with the observed payload + +Adjust `TradeRecord` field mappings accordingly. The scaffold is structured +to make this reconciliation localized — parsing lives in `get_trade_data`. + +## Scope Notes + +- v0.1.0 does **not** cover USITC's tariff-schedule lookups (MFN, column-2, + special-program rates). A `get_tariff_rate` tool is a natural v0.2 addition. +- Tariff-line-level data is often subject to BIS/Census confidentiality + suppressions at low-volume partners; expect gaps below ~$250,000/year per + HTS×country×month cell. diff --git a/packages/usitc-mcp/pyproject.toml b/packages/usitc-mcp/pyproject.toml new file mode 100644 index 0000000..264d978 --- /dev/null +++ b/packages/usitc-mcp/pyproject.toml @@ -0,0 +1,21 @@ +[project] +name = "usitc-mcp" +version = "0.1.0" +description = "MCP server for USITC DataWeb US import/export trade statistics" +requires-python = ">=3.10" +dependencies = [ + "mcp[cli]>=1.0.0", + "httpx>=0.27.0", + "pydantic>=2.0.0", + "python-dotenv>=1.0.0", +] + +[project.scripts] +usitc-mcp = "usitc_mcp.server:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/usitc_mcp"] diff --git a/packages/usitc-mcp/src/usitc_mcp/__init__.py b/packages/usitc-mcp/src/usitc_mcp/__init__.py new file mode 100644 index 0000000..c6b8268 --- /dev/null +++ b/packages/usitc-mcp/src/usitc_mcp/__init__.py @@ -0,0 +1,5 @@ +"""USITC DataWeb MCP Server - US international trade statistics at HTS granularity.""" + +from __future__ import annotations + +__version__ = "0.1.0" diff --git a/packages/usitc-mcp/src/usitc_mcp/client.py b/packages/usitc-mcp/src/usitc_mcp/client.py new file mode 100644 index 0000000..a2f5db3 --- /dev/null +++ b/packages/usitc-mcp/src/usitc_mcp/client.py @@ -0,0 +1,402 @@ +"""Async HTTP client for USITC DataWeb v2 API. + +USITC DataWeb is a bearer-token authenticated REST API that exposes US +import/export tariff-line trade statistics. Queries are submitted as a +nested ``SavedQuery`` envelope to ``/api/v2/report2/runReport`` and the +server responds with a pivot-table DTO containing one or more tables with +row groups keyed by commodity/country/etc. and column groups keyed by year. + +Schema discovery, 2026-04: the OpenAPI document at +``https://datawebws.usitc.gov/dataweb/v3/api-docs`` lists a flat ``SavedQuery`` +component, but the server actually expects a **nested** envelope whose +structure matches the JSON returned by ``/api/v2/savedQuery/getAllSystemSavedQueries``. +The shapes below were validated against a live bearer token. + +Caveats still needing resolution against your workflow: + +* ``dataToReport`` values are tradeType-specific. Validated: ``FAS_VALUE`` + for ``tradeType=Export``; ``GEN_CUSTOMS_VALUE`` for ``tradeType=Import``. + Other codes seen in the DataWeb UI (``GEN_VAL_YR``, ``CIF_VAL_YR``, + ``CONS_VAL_YR``) produce ``Invalid dataToReport object`` validation errors. +* Commodity filtering via ``commodities.commodities=[...]`` with + ``commoditySelectType='entered'`` currently triggers a server-side + "step 4" validation failure. Until the correct companion-field values are + pinned down, ``get_trade_data`` submits filters but falls back to + whole-chapter aggregation when the server rejects the narrowed query. The + TODO in :meth:`USITCClient.get_trade_data` tracks this. +""" + +from __future__ import annotations + +import os +from typing import Any + +import httpx +from dotenv import load_dotenv + +from .models import CMM_HTS_CODES, TradeRecord + +load_dotenv() + +# Data-to-report codes validated against a live token. +_DATA_TO_REPORT = { + "general_imports": "GEN_CUSTOMS_VALUE", + "imports_for_consumption": "GEN_CUSTOMS_VALUE", + "domestic_exports": "FAS_VALUE", + "total_exports": "FAS_VALUE", +} + +_TRADE_TYPE = { + "import": "Import", + "export": "Export", +} + + +class USITCAPIError(Exception): + """Raised for USITC DataWeb API errors.""" + + +class USITCClient: + """Client for USITC DataWeb v2 API.""" + + BASE_URL = "https://datawebws.usitc.gov/dataweb/api/v2" + + def __init__(self, api_token: str | None = None): + self.api_token = api_token or os.getenv("USITC_API_TOKEN") + self.timeout = 120.0 + + def is_available(self) -> bool: + return bool(self.api_token) + + def _require_token(self) -> None: + if not self.api_token: + raise USITCAPIError( + "USITC_API_TOKEN is not set. Obtain a token at " + "https://dataweb.usitc.gov → API tab, then set USITC_API_TOKEN." + ) + + def _headers(self) -> dict[str, str]: + self._require_token() + return { + "Authorization": f"Bearer {self.api_token}", + "Accept": "application/json", + "Content-Type": "application/json", + } + + async def _post(self, endpoint: str, payload: dict[str, Any]) -> dict[str, Any]: + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.post( + f"{self.BASE_URL}/{endpoint}", + headers=self._headers(), + json=payload, + ) + response.raise_for_status() + data: dict[str, Any] = response.json() + return data + + async def _get(self, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get( + f"{self.BASE_URL}/{endpoint}", + headers=self._headers(), + params=params, + ) + response.raise_for_status() + data: dict[str, Any] = response.json() + return data + + async def check_status(self) -> dict[str, Any]: + """Probe API connectivity and token validity via ``country/getAllCountries``.""" + if not self.api_token: + return { + "status": "unauthorized", + "api_key_configured": False, + "message": "USITC_API_TOKEN not set", + } + try: + data = await self._get("country/getAllCountries") + options = data.get("options") or [] + return { + "status": "connected", + "api_key_configured": True, + "countries_indexed": len(options), + "message": "USITC DataWeb API reachable", + } + except httpx.HTTPStatusError as e: + if e.response.status_code == 401: + return { + "status": "unauthorized", + "api_key_configured": True, + "message": "Token rejected (401). Check validity/expiry.", + } + return { + "status": "error", + "api_key_configured": True, + "message": f"HTTP {e.response.status_code}: {e.response.text[:200]}", + } + except (httpx.HTTPError, OSError) as e: + return {"status": "error", "message": str(e)} + + async def search_hts(self, query: str, limit: int = 50) -> list[dict[str, Any]]: + """Search HTS by description via ``commodity/commodityDescriptionLookup``.""" + try: + data = await self._post( + "commodity/commodityDescriptionLookup", + {"tradeType": "Import", "classificationSystem": "HTS", "search": query}, + ) + items = data.get("options") or [] + return items[:limit] + except httpx.HTTPError as e: + raise USITCAPIError(f"HTS search failed: {e}") from e + + @staticmethod + def _build_saved_query( + trade_type: str, + data_to_report: str, + years: list[int], + hts_codes: list[str] | None, + country_codes: list[str] | None, + ) -> dict[str, Any]: + """Construct a SavedQuery envelope for /report2/runReport. + + Mirrors the JSON shape returned by ``/savedQuery/getAllSystemSavedQueries`` + (which is the only validated-working template). + """ + # Commodity aggregation: "Aggregate Commodities" returns a single + # grand-total per HTS-chapter; "Break out Commodities" returns one row + # per distinct HTS code. + commodities_agg = "Break out Commodities" if hts_codes else "Aggregate Commodities" + commodities_select = "entered" if hts_codes else "all" + countries_agg = "Break out countries" if country_codes else "Aggregate countries" + countries_select = "entered" if country_codes else "all" + + # Granularity is inferred from the longest HTS code supplied. The API + # accepts "2"|"4"|"6"|"8"|"10". + granularity = "2" + if hts_codes: + max_len = max(len(c.replace(".", "")) for c in hts_codes) + granularity = str(max(2, min(10, (max_len // 2) * 2))) + + return { + "savedQueryName": "", + "isOwner": True, + "runMonthly": False, + "reportOptions": { + "tradeType": trade_type, + "classificationSystem": "HTS", + }, + "searchOptions": { + "componentSettings": { + "dataToReport": [data_to_report], + "scale": "1", + "timeframeSelectType": "fullYears", + "years": [str(y) for y in years], + "startDate": None, + "endDate": None, + "startMonth": None, + "endMonth": None, + "yearsTimeline": "Annual", + }, + "commodities": { + "commodities": [c.replace(".", "") for c in (hts_codes or [])], + "commoditiesExpanded": [], + "commoditiesManual": None, + "commodityGroups": {"systemGroups": [], "userGroups": []}, + "granularity": granularity, + "searchGranularity": granularity, + "groupGranularity": granularity, + "aggregation": commodities_agg, + "codeDisplayFormat": "NO", + "commoditySelectType": commodities_select, + "showHTSValidDetails": True, + }, + "countries": { + "countries": country_codes or [], + "countriesExpanded": [], + "countryGroups": {"systemGroups": [], "userGroups": []}, + "aggregation": countries_agg, + "countriesSelectType": countries_select, + }, + "MiscGroup": { + "importPrograms": { + "importPrograms": [], + "aggregation": "Aggregate CSC", + }, + "extImportPrograms": { + "programsSelectType": "all", + "extImportPrograms": [], + "extImportProgramsExpanded": [], + "aggregation": "Aggregate CSC", + }, + "provisionCodes": { + "rateProvisionCodes": [], + "rateProvisionCodesExpanded": [], + "aggregation": "Aggregate RPCODE", + "provisionCodesSelectType": "all", + "rateProvisionGroups": {"systemGroups": []}, + }, + "districts": { + "districts": [], + "districtsExpanded": [], + "districtGroups": {"userGroups": []}, + "aggregation": "Aggregate District", + "districtsSelectType": "all", + }, + }, + }, + "sortingAndDataFormat": { + "DataSort": {"sortOrder": [], "columnOrder": [], "sortYear": None}, + "reportCustomizations": { + "totalRecords": "20000", + "exportCombineTables": False, + "reportsGrid": True, + "removeDuplicateValues": True, + "suppressZeroValues": False, + "displayCommodityList": False, + "reportsFontSize": "m", + "exportRawData": False, + }, + }, + } + + @staticmethod + def _parse_number(s: Any) -> float | None: + if s is None: + return None + if isinstance(s, int | float): + return float(s) + text = str(s).strip().replace(",", "") + if not text or text == "-": + return None + try: + return float(text) + except ValueError: + return None + + def _flatten_dto( + self, + dto: dict[str, Any], + flow: str, + data_type: str, + ) -> list[TradeRecord]: + """Convert a pivot-table DTO into flat TradeRecord rows. + + Each ``rowsNew[i].rowEntries[j]`` cell becomes one record, keyed by + the corresponding ``row_groups[i].columnInfo[j]`` entry (which carries + the year/period label). + """ + records: list[TradeRecord] = [] + for table in dto.get("tables") or []: + row_groups = table.get("row_groups") or [] + for rg in row_groups: + col_info = rg.get("columnInfo") or [] + for row in rg.get("rowsNew") or []: + entries = row.get("rowEntries") or [] + # Leading string cells (commodity label, country, etc.) + # are positioned before the numeric data columns. Map by + # columnInfo index: its ``columnIndex`` field points into + # ``rowEntries``. + label_cells = [e.get("value") for e in entries[: len(entries) - len(col_info)]] + hts_code = label_cells[0] if label_cells else "" + country_name = label_cells[1] if len(label_cells) > 1 else None + for ci in col_info: + idx = ci.get("columnIndex") + if idx is None or idx >= len(entries): + continue + value = self._parse_number(entries[idx].get("value")) + year_label = ci.get("columnLabel") or ci.get("queryResultLabel") or "" + try: + year = int(str(year_label)[:4]) + except ValueError: + continue + records.append( + TradeRecord( + year=year, + hts_code=str(hts_code or ""), + country_name=country_name, + flow=flow, + data_type=data_type, + value_usd=value, + ) + ) + return records + + async def get_trade_data( + self, + hts_codes: list[str], + years: list[int], + flow: str = "import", + data_type: str = "general_imports", + country_codes: list[str] | None = None, + ) -> list[TradeRecord]: + """Fetch US trade data via ``/report2/runReport``. + + Args: + hts_codes: HTS codes (2/4/6/8/10 digit, dots optional). + years: List of years. + flow: ``"import"`` or ``"export"``. + data_type: ``general_imports`` | ``imports_for_consumption`` | + ``domestic_exports`` | ``total_exports``. + country_codes: USITC partner country ``value`` codes (see + ``country/getAllCountries``). ``None`` = aggregate all partners. + + Returns: + List of ``TradeRecord``. + + Note: commodity filtering via ``commoditySelectType='entered'`` is not + yet reliably accepted by the server (it raises a "step 4" validation + error whose companion-field requirements are not pinned down). When + that happens this method raises ``USITCAPIError`` and the caller must + drop the HTS filter and aggregate client-side. + """ + trade_type = _TRADE_TYPE.get(flow.lower()) + if not trade_type: + raise USITCAPIError(f"Unknown flow '{flow}'. Use 'import' or 'export'.") + dtr = _DATA_TO_REPORT.get(data_type) + if not dtr: + raise USITCAPIError( + f"Unknown data_type '{data_type}'. Use one of {list(_DATA_TO_REPORT)}." + ) + + payload = self._build_saved_query( + trade_type=trade_type, + data_to_report=dtr, + years=years, + hts_codes=hts_codes, + country_codes=country_codes, + ) + + try: + data = await self._post("report2/runReport", payload) + except httpx.HTTPError as e: + raise USITCAPIError(f"runReport failed: {e}") from e + + dto = data.get("dto") or {} + errors = dto.get("errors") or [] + if errors: + # Server validation (step 2/step 4/etc). Surface one level up. + raise USITCAPIError(f"Query rejected: {'; '.join(errors)}") + + return self._flatten_dto(dto, flow=flow, data_type=data_type) + + async def get_critical_mineral_trade( + self, + mineral: str, + years: list[int], + flow: str = "import", + country_codes: list[str] | None = None, + ) -> list[TradeRecord]: + key = mineral.lower().replace(" ", "_") + codes = CMM_HTS_CODES.get(key) + if not codes: + raise USITCAPIError( + f"Unknown mineral: {mineral}. Available: {', '.join(CMM_HTS_CODES.keys())}" + ) + data_type = "general_imports" if flow == "import" else "total_exports" + return await self.get_trade_data( + hts_codes=codes, + years=years, + flow=flow, + data_type=data_type, + country_codes=country_codes, + ) diff --git a/packages/usitc-mcp/src/usitc_mcp/models.py b/packages/usitc-mcp/src/usitc_mcp/models.py new file mode 100644 index 0000000..72817a8 --- /dev/null +++ b/packages/usitc-mcp/src/usitc_mcp/models.py @@ -0,0 +1,102 @@ +"""Pydantic models and reference data for USITC DataWeb. + +USITC DataWeb v2 API reference: https://datawebws.usitc.gov/dataweb +(Production documentation is accessible after registering an account at +https://dataweb.usitc.gov and obtaining a bearer token.) +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class TradeRecord(BaseModel): + """A single USITC trade-data record (aggregated import or export).""" + + year: int = Field(description="Calendar year") + month: int | None = Field(default=None, description="Calendar month (1-12); None for annual") + hts_code: str = Field(description="HTS commodity code (2-10 digits)") + hts_description: str | None = Field(default=None, description="HTS description") + country_code: str | None = Field(default=None, description="Partner country ISO/USITC code") + country_name: str | None = Field(default=None, description="Partner country name") + flow: str = Field(description="Trade flow: 'import' or 'export'") + data_type: str = Field( + description="Data type: 'general_imports', 'imports_for_consumption', " + "'domestic_exports', 'total_exports'" + ) + value_usd: float | None = Field(default=None, description="Customs value in USD") + quantity_1: float | None = Field(default=None, description="First reported quantity") + quantity_1_unit: str | None = Field(default=None, description="Units of first quantity") + quantity_2: float | None = Field(default=None, description="Second reported quantity") + quantity_2_unit: str | None = Field(default=None, description="Units of second quantity") + + +class HTSReference(BaseModel): + """HTS (Harmonized Tariff Schedule of the United States) classification entry.""" + + code: str = Field(description="HTS code (2/4/6/8/10 digits)") + description: str = Field(description="HTS description") + parent: str | None = Field(default=None, description="Parent HTS code") + + +# ── CMM-focused HTS codes ──────────────────────────────────────────────────── +# US HTS codes for critical minerals. These extend the UN HS codes with US-specific +# 8-10 digit granularity where appropriate. Keep this list aligned with the +# uncomtrade-mcp HS codes but at US-HTS precision. +CMM_HTS_CODES: dict[str, list[str]] = { + "lithium": [ + "2530.90.00", # Other mineral substances + "2825.20.00", # Lithium oxide and hydroxide + "2836.91.00", # Lithium carbonates + "8507.60.00", # Lithium-ion batteries + ], + "cobalt": [ + "2605.00.00", # Cobalt ores and concentrates + "2822.00.00", # Cobalt oxides and hydroxides + "8105.20.30", # Cobalt, unwrought; powders + "8105.90.00", # Cobalt articles + ], + "rare_earth": [ + "2805.30.00", # Rare-earth metals + "2846.10.00", # Cerium compounds + "2846.90.00", # Other rare-earth compounds + ], + "graphite": [ + "2504.10.00", # Natural graphite, powder/flakes + "2504.90.00", # Natural graphite, other + "3801.10.00", # Artificial graphite + ], + "nickel": [ + "2604.00.00", # Nickel ores and concentrates + "7501.10.00", # Nickel mattes + "7502.10.00", # Unwrought nickel, not alloyed + "2811.22.00", # Nickel oxides + ], + "manganese": [ + "2602.00.00", # Manganese ores and concentrates + "8111.00.30", # Manganese, unwrought + ], + "gallium": [ + "8112.92.00", # Gallium, unwrought + ], + "germanium": [ + "8112.92.06", # Germanium, unwrought (US-specific) + ], + "copper": [ + "7402.00.00", # Unrefined copper; copper anodes + "7403.11.00", # Refined copper, cathodes + ], +} + + +MINERAL_NAMES: dict[str, str] = { + "lithium": "Lithium (Li)", + "cobalt": "Cobalt (Co)", + "rare_earth": "Rare Earth Elements (all)", + "graphite": "Graphite (Gr)", + "nickel": "Nickel (Ni)", + "manganese": "Manganese (Mn)", + "gallium": "Gallium (Ga)", + "germanium": "Germanium (Ge)", + "copper": "Copper (Cu)", +} diff --git a/packages/usitc-mcp/src/usitc_mcp/py.typed b/packages/usitc-mcp/src/usitc_mcp/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/packages/usitc-mcp/src/usitc_mcp/server.py b/packages/usitc-mcp/src/usitc_mcp/server.py new file mode 100644 index 0000000..157729c --- /dev/null +++ b/packages/usitc-mcp/src/usitc_mcp/server.py @@ -0,0 +1,227 @@ +"""MCP server for USITC DataWeb.""" + +from __future__ import annotations + +import asyncio + +from mcp.server.fastmcp import FastMCP + +from .client import USITCAPIError, USITCClient +from .models import CMM_HTS_CODES, MINERAL_NAMES + +mcp = FastMCP( + name="USITC DataWeb", + instructions="""USITC DataWeb MCP Server provides access to US import/export +trade statistics at HTS (Harmonized Tariff Schedule of the United States) +granularity. Complements uncomtrade-mcp by offering US-specific HTS codes +(8-10 digit) and trade-measure granularity (general imports, imports for +consumption, domestic exports, total exports) not available in UN Comtrade. + +Key capabilities: +- Fetch US import or export values by HTS code, partner, and year +- Search the HTS classification catalogue +- Query curated CMM-focused HTS codes (lithium, cobalt, REE, graphite, Ni, Mn, Ga, Ge, Cu) +- Switch between general imports and imports for consumption +- Retrieve both value (USD) and quantity (multi-unit) + +Requires USITC_API_TOKEN. Register at https://dataweb.usitc.gov and request +API access from your account page.""", +) + + +def get_client() -> USITCClient: + """Return a fresh USITCClient instance.""" + return USITCClient() + + +# ============================================================================= +# Overview +# ============================================================================= + + +@mcp.tool() +async def get_api_status() -> dict: + """Check USITC DataWeb API connectivity and token validity.""" + client = get_client() + return await client.check_status() + + +@mcp.tool() +async def list_cmm_hts_codes() -> dict: + """List curated CMM-focused HTS codes for each critical mineral.""" + items = [] + for key, codes in CMM_HTS_CODES.items(): + items.append( + { + "id": key, + "name": MINERAL_NAMES.get(key, key), + "hts_codes": codes, + "count": len(codes), + } + ) + return { + "count": len(items), + "minerals": items, + "usage": "Use get_critical_mineral_trade(mineral='lithium', years=[2022,2023])", + } + + +# ============================================================================= +# Search +# ============================================================================= + + +@mcp.tool() +async def search_hts(query: str, limit: int = 25) -> dict: + """Search the HTS classification by keyword or code prefix. + + Args: + query: Search string (e.g., "lithium", "8507", "battery"). + limit: Max results. + """ + client = get_client() + try: + results = await client.search_hts(query, limit=limit) + except USITCAPIError as e: + return {"error": str(e)} + return {"query": query, "count": len(results), "results": results} + + +# ============================================================================= +# Data Queries +# ============================================================================= + + +@mcp.tool() +async def get_trade_data( + hts_codes: list[str], + years: list[int], + flow: str = "import", + data_type: str = "general_imports", + country_codes: list[str] | None = None, +) -> dict: + """Fetch US import or export data by HTS code, partner, and year. + + Args: + hts_codes: List of HTS codes (e.g., ["2836.91.00", "2825.20.00"]). + years: List of years to query (e.g., [2020, 2021, 2022, 2023]). + flow: "import" or "export". + data_type: "general_imports" | "imports_for_consumption" | + "domestic_exports" | "total_exports". + country_codes: Optional list of partner codes; omit for all partners. + + Returns: + Dict with query and list of TradeRecord. + """ + client = get_client() + try: + records = await client.get_trade_data( + hts_codes=hts_codes, + years=years, + flow=flow, + data_type=data_type, + country_codes=country_codes, + ) + except USITCAPIError as e: + return {"error": str(e)} + return { + "query": { + "hts_codes": hts_codes, + "years": years, + "flow": flow, + "data_type": data_type, + "country_codes": country_codes, + }, + "count": len(records), + "records": [r.model_dump() for r in records], + } + + +@mcp.tool() +async def get_critical_mineral_trade( + mineral: str, + years: list[int], + flow: str = "import", + country_codes: list[str] | None = None, +) -> dict: + """Get US trade data for a critical mineral using curated HTS codes. + + Available minerals: lithium, cobalt, rare_earth, graphite, nickel, + manganese, gallium, germanium, copper. + + Args: + mineral: Mineral key (e.g., "lithium"). + years: List of years. + flow: "import" or "export". + country_codes: Optional partner filter. + + Returns: + Dict with records and HTS codes queried. + """ + client = get_client() + try: + records = await client.get_critical_mineral_trade( + mineral=mineral, years=years, flow=flow, country_codes=country_codes + ) + except USITCAPIError as e: + return {"error": str(e)} + key = mineral.lower().replace(" ", "_") + return { + "mineral": MINERAL_NAMES.get(key, mineral), + "hts_codes_queried": CMM_HTS_CODES.get(key, []), + "flow": flow, + "years": years, + "count": len(records), + "records": [r.model_dump() for r in records], + } + + +@mcp.tool() +async def compare_import_types( + hts_codes: list[str], + years: list[int], + country_codes: list[str] | None = None, +) -> dict: + """Compare 'general imports' vs. 'imports for consumption' for the same HTS set. + + Useful for identifying goods entering bonded warehouses / FTZs versus those + cleared for domestic consumption — relevant to CMM stockpile and FTZ analysis. + + Args: + hts_codes: List of HTS codes. + years: List of years. + country_codes: Optional partner filter. + """ + client = get_client() + data_types = ("general_imports", "imports_for_consumption") + try: + responses = await asyncio.gather( + *( + client.get_trade_data( + hts_codes=hts_codes, + years=years, + flow="import", + data_type=dt, + country_codes=country_codes, + ) + for dt in data_types + ) + ) + except USITCAPIError as e: + return {"error": str(e)} + result = {dt: [r.model_dump() for r in recs] for dt, recs in zip(data_types, responses)} + return { + "query": {"hts_codes": hts_codes, "years": years, "country_codes": country_codes}, + "general_imports_count": len(result["general_imports"]), + "imports_for_consumption_count": len(result["imports_for_consumption"]), + "records": result, + } + + +def main(): + """Run the MCP server.""" + mcp.run() + + +if __name__ == "__main__": + main() diff --git a/packages/usitc-mcp/tests/conftest.py b/packages/usitc-mcp/tests/conftest.py new file mode 100644 index 0000000..1dd5edc --- /dev/null +++ b/packages/usitc-mcp/tests/conftest.py @@ -0,0 +1,13 @@ +"""Shared pytest fixtures for usitc-mcp.""" + +from __future__ import annotations + +import pytest + +from usitc_mcp.client import USITCClient + + +@pytest.fixture +def client() -> USITCClient: + """Return a USITCClient instance for tests.""" + return USITCClient() diff --git a/packages/usitc-mcp/tests/test_models.py b/packages/usitc-mcp/tests/test_models.py new file mode 100644 index 0000000..75a0c64 --- /dev/null +++ b/packages/usitc-mcp/tests/test_models.py @@ -0,0 +1,35 @@ +"""Unit tests for usitc_mcp.models.""" + +from __future__ import annotations + +from usitc_mcp.models import CMM_HTS_CODES, MINERAL_NAMES, TradeRecord + + +def test_cmm_hts_codes_nonempty(): + assert len(CMM_HTS_CODES) > 5 + assert "lithium" in CMM_HTS_CODES + assert len(CMM_HTS_CODES["lithium"]) > 0 + + +def test_mineral_names_match_hts_keys(): + for key in CMM_HTS_CODES: + assert key in MINERAL_NAMES, f"Missing display name for {key}" + + +def test_trade_record_roundtrip(): + rec = TradeRecord( + year=2023, + month=12, + hts_code="2836.91.00", + hts_description="Lithium carbonates", + country_code="CL", + country_name="Chile", + flow="import", + data_type="general_imports", + value_usd=1_234_567.89, + quantity_1=500.0, + quantity_1_unit="kg", + ) + assert rec.hts_code == "2836.91.00" + assert rec.value_usd == 1_234_567.89 + assert rec.model_dump()["flow"] == "import" diff --git a/packages/worldbank-mcp/README.md b/packages/worldbank-mcp/README.md new file mode 100644 index 0000000..e916461 --- /dev/null +++ b/packages/worldbank-mcp/README.md @@ -0,0 +1,53 @@ +# worldbank-mcp + +MCP server for the World Bank World Development Indicators (WDI) API. Provides +macroeconomic, trade, and governance context relevant to critical-minerals +supply-chain analysis. + +## Tools + +| Tool | Description | +|------|-------------| +| `get_api_status` | Check WDI API connectivity | +| `list_cmm_indicators` | List curated CMM-relevant WDI indicators | +| `list_cmm_key_economies` | List ISO3 codes of key CMM supply-chain economies | +| `search_indicators` | Search the WDI catalogue by keyword | +| `list_countries` | List World Bank countries with region/income classification | +| `get_indicator` | Fetch indicator observations for country/years | +| `get_cmm_profile` | Fetch the curated CMM bundle for a single country | +| `compare_countries` | Pivot table of one indicator across multiple countries (markdown) | + +## Configuration + +| Variable | Required | Description | +|----------|----------|-------------| +| `WORLDBANK_API_KEY` | No | WDI is public; reserved for future WITS authenticated endpoints | + +## Usage + +```sh +uv run worldbank-mcp +``` + +### Claude Desktop + +```json +{ + "mcpServers": { + "worldbank-mcp": { + "command": "uv", + "args": ["--directory", "/path/to/CMM_Tools", "run", "worldbank-mcp"] + } + } +} +``` + +## Scope Notes + +- **WDI coverage only in v0.1.0.** WITS (World Integrated Trade Solution) endpoints + return SDMX XML and have a distinct rate/auth model; a `wits_*` tool family is + deferred to v0.2.0. +- Observations with `value = None` represent legitimate missing data (country did + not report or indicator unavailable) rather than errors. +- The CMM indicator bundle (`list_cmm_indicators`) is a curated subset; browse the + full catalogue via `search_indicators`. diff --git a/packages/worldbank-mcp/pyproject.toml b/packages/worldbank-mcp/pyproject.toml new file mode 100644 index 0000000..738b6ef --- /dev/null +++ b/packages/worldbank-mcp/pyproject.toml @@ -0,0 +1,21 @@ +[project] +name = "worldbank-mcp" +version = "0.1.0" +description = "MCP server for World Bank WDI indicators and WITS trade statistics" +requires-python = ">=3.10" +dependencies = [ + "mcp[cli]>=1.0.0", + "httpx>=0.27.0", + "pydantic>=2.0.0", + "python-dotenv>=1.0.0", +] + +[project.scripts] +worldbank-mcp = "worldbank_mcp.server:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/worldbank_mcp"] diff --git a/packages/worldbank-mcp/src/worldbank_mcp/__init__.py b/packages/worldbank-mcp/src/worldbank_mcp/__init__.py new file mode 100644 index 0000000..79bd905 --- /dev/null +++ b/packages/worldbank-mcp/src/worldbank_mcp/__init__.py @@ -0,0 +1,5 @@ +"""World Bank MCP Server - WDI indicators and WITS trade statistics for critical minerals analysis.""" + +from __future__ import annotations + +__version__ = "0.1.0" diff --git a/packages/worldbank-mcp/src/worldbank_mcp/client.py b/packages/worldbank-mcp/src/worldbank_mcp/client.py new file mode 100644 index 0000000..3be0a86 --- /dev/null +++ b/packages/worldbank-mcp/src/worldbank_mcp/client.py @@ -0,0 +1,161 @@ +"""Async HTTP client for World Bank WDI API. + +The World Bank Indicators API (v2) is public and does not require authentication +for the WDI endpoints used here. An optional WORLDBANK_API_KEY env var is +retained for future extension (e.g., WITS authenticated endpoints). + +API reference: https://datahelpdesk.worldbank.org/knowledgebase/articles/889392 +""" + +from __future__ import annotations + +import os +from typing import Any + +import httpx +from dotenv import load_dotenv + +from .models import IndicatorObservation + +load_dotenv() + + +class WorldBankClient: + """Client for the World Bank WDI API (v2).""" + + BASE_URL = "https://api.worldbank.org/v2" + + def __init__(self, api_key: str | None = None): + """Initialize client. API key is optional for WDI.""" + self.api_key = api_key or os.getenv("WORLDBANK_API_KEY") + self.timeout = 60.0 + + def is_available(self) -> bool: + """WDI is publicly accessible; always returns True.""" + return True + + async def _request(self, endpoint: str, params: dict[str, Any] | None = None) -> list[Any]: + """GET against the WDI API. Responses are a two-element list: [meta, data].""" + params = dict(params or {}) + params.setdefault("format", "json") + params.setdefault("per_page", 1000) + + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get(f"{self.BASE_URL}/{endpoint}", params=params) + response.raise_for_status() + payload = response.json() + if not isinstance(payload, list): + raise ValueError(f"Unexpected WDI response shape: {type(payload).__name__}") + return payload + + async def check_status(self) -> dict[str, Any]: + """Check connectivity via a minimal WDI query.""" + try: + payload = await self._request( + "country/USA/indicator/NY.GDP.MKTP.CD", + params={"date": "2022", "per_page": 1}, + ) + meta = payload[0] if payload else {} + data = payload[1] if len(payload) > 1 else [] + return { + "status": "connected", + "api_key_configured": bool(self.api_key), + "total_observations_sample": meta.get("total"), + "sample_record_count": len(data), + "message": "World Bank WDI API reachable", + } + except httpx.TimeoutException: + return {"status": "timeout", "message": "Request timed out"} + except (httpx.HTTPError, OSError, ValueError) as e: + return {"status": "error", "message": str(e)} + + async def get_indicator_observations( + self, + country: str, + indicator: str, + date: str | None = None, + per_page: int = 100, + ) -> list[IndicatorObservation]: + """ + Fetch indicator observations for a country or country group. + + Args: + country: ISO3 code or semicolon-separated list (e.g., "USA;CHN;DEU"). + Special groups: "all", "WLD" (world). + indicator: WDI indicator code (e.g., "NY.GDP.MKTP.CD"). + date: Year or range, e.g., "2022" or "2000:2023". + per_page: Page size (max 1000). + + Returns: + List of IndicatorObservation, newest first. + """ + endpoint = f"country/{country}/indicator/{indicator}" + params: dict[str, Any] = {"per_page": min(per_page, 1000)} + if date: + params["date"] = date + + payload = await self._request(endpoint, params=params) + data = payload[1] if len(payload) > 1 else [] + records: list[IndicatorObservation] = [] + for item in data: + if not isinstance(item, dict): + continue + try: + country_meta = item.get("country") or {} + indicator_meta = item.get("indicator") or {} + records.append( + IndicatorObservation( + country_code=item.get("countryiso3code") or country_meta.get("id", ""), + country_name=country_meta.get("value"), + indicator_code=indicator_meta.get("id", indicator), + indicator_name=indicator_meta.get("value"), + year=int(item.get("date")) if item.get("date") else 0, + value=item.get("value"), + unit=item.get("unit") or None, + ) + ) + except (ValueError, TypeError): + continue + return records + + async def search_indicators(self, query: str, per_page: int = 50) -> list[dict[str, Any]]: + """Search the WDI indicator catalogue by free-text keyword.""" + params = {"per_page": min(per_page, 500)} + payload = await self._request("indicator", params=params) + data = payload[1] if len(payload) > 1 else [] + q = query.lower() + matches: list[dict[str, Any]] = [] + for ind in data: + if not isinstance(ind, dict): + continue + name = (ind.get("name") or "").lower() + note = (ind.get("sourceNote") or "").lower() + if q in name or q in note: + matches.append( + { + "id": ind.get("id"), + "name": ind.get("name"), + "source_note": ind.get("sourceNote"), + "topics": [t.get("value") for t in (ind.get("topics") or [])], + } + ) + return matches + + async def list_countries(self) -> list[dict[str, Any]]: + """List all World Bank country/region reference entries.""" + payload = await self._request("country", params={"per_page": 500}) + data = payload[1] if len(payload) > 1 else [] + result: list[dict[str, Any]] = [] + for c in data: + if not isinstance(c, dict): + continue + result.append( + { + "iso3": c.get("id"), + "iso2": c.get("iso2Code"), + "name": c.get("name"), + "region": (c.get("region") or {}).get("value"), + "income_level": (c.get("incomeLevel") or {}).get("value"), + } + ) + return result diff --git a/packages/worldbank-mcp/src/worldbank_mcp/models.py b/packages/worldbank-mcp/src/worldbank_mcp/models.py new file mode 100644 index 0000000..e903c06 --- /dev/null +++ b/packages/worldbank-mcp/src/worldbank_mcp/models.py @@ -0,0 +1,90 @@ +"""Pydantic models and reference data for World Bank WDI / WITS.""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class IndicatorObservation(BaseModel): + """A single WDI indicator observation for a country-year.""" + + country_code: str = Field(description="ISO3 country code") + country_name: str | None = Field(default=None, description="Country name") + indicator_code: str = Field(description="WDI indicator code") + indicator_name: str | None = Field(default=None, description="Indicator description") + year: int = Field(description="Observation year") + value: float | None = Field(default=None, description="Observation value (None if missing)") + unit: str | None = Field(default=None, description="Unit of measurement where available") + + class Config: + populate_by_name = True + + +class IndicatorMetadata(BaseModel): + """Metadata record for a WDI indicator.""" + + id: str = Field(description="Indicator code (e.g., NY.GDP.MKTP.CD)") + name: str = Field(description="Indicator name") + source_note: str | None = Field(default=None, description="Description / source note") + topic: str | None = Field(default=None, description="Primary topic classification") + + +class CountryReference(BaseModel): + """World Bank country reference entry.""" + + iso3: str = Field(description="ISO 3-letter code") + iso2: str | None = Field(default=None, description="ISO 2-letter code") + name: str = Field(description="Country name") + region: str | None = Field(default=None, description="Region") + income_level: str | None = Field(default=None, description="Income classification") + + +# ── CMM-relevant WDI indicators ────────────────────────────────────────────── +# Curated set of indicators useful for critical-minerals supply-chain analysis. +CMM_WDI_INDICATORS: dict[str, str] = { + # Macro & trade context + "NY.GDP.MKTP.CD": "GDP (current US$)", + "NE.EXP.GNFS.CD": "Exports of goods and services (current US$)", + "NE.IMP.GNFS.CD": "Imports of goods and services (current US$)", + "BX.KLT.DINV.CD.WD": "Foreign direct investment, net inflows (BoP, current US$)", + # Manufacturing & industry + "NV.IND.MANF.ZS": "Manufacturing, value added (% of GDP)", + "NV.IND.TOTL.ZS": "Industry (including construction), value added (% of GDP)", + # Natural resources + "NY.GDP.TOTL.RT.ZS": "Total natural resources rents (% of GDP)", + "NY.GDP.MINR.RT.ZS": "Mineral rents (% of GDP)", + "TX.VAL.MRCH.CD.WT": "Merchandise exports (current US$)", + "TM.VAL.MRCH.CD.WT": "Merchandise imports (current US$)", + # Energy & transition + "EG.USE.ELEC.KH.PC": "Electric power consumption (kWh per capita)", + "EG.ELC.RNEW.ZS": "Renewable electricity output (% of total)", + # Governance / risk proxies + "IQ.CPA.ECON.XQ": "CPIA economic management cluster avg (1=low to 6=high)", + "GE.EST": "Government Effectiveness: Estimate (WGI)", + "RL.EST": "Rule of Law: Estimate (WGI)", +} + + +# ISO3 codes for major CMM-relevant economies +CMM_KEY_ECONOMIES: dict[str, str] = { + "USA": "United States", + "CHN": "China", + "AUS": "Australia", + "CHL": "Chile", + "COD": "Democratic Republic of Congo", + "ARG": "Argentina", + "BOL": "Bolivia", + "IDN": "Indonesia", + "PHL": "Philippines", + "ZAF": "South Africa", + "CAN": "Canada", + "MEX": "Mexico", + "RUS": "Russia", + "KAZ": "Kazakhstan", + "MMR": "Myanmar", + "VNM": "Vietnam", + "JPN": "Japan", + "KOR": "South Korea", + "DEU": "Germany", + "BRA": "Brazil", +} diff --git a/packages/worldbank-mcp/src/worldbank_mcp/py.typed b/packages/worldbank-mcp/src/worldbank_mcp/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/packages/worldbank-mcp/src/worldbank_mcp/server.py b/packages/worldbank-mcp/src/worldbank_mcp/server.py new file mode 100644 index 0000000..f339a64 --- /dev/null +++ b/packages/worldbank-mcp/src/worldbank_mcp/server.py @@ -0,0 +1,247 @@ +"""MCP server for World Bank WDI indicators.""" + +from __future__ import annotations + +import asyncio + +import httpx +from mcp.server.fastmcp import FastMCP + +from .client import WorldBankClient +from .models import CMM_KEY_ECONOMIES, CMM_WDI_INDICATORS + +mcp = FastMCP( + name="World Bank", + instructions="""World Bank MCP Server provides access to the World Development +Indicators (WDI) API. Useful for sovereign and macroeconomic context surrounding +critical-minerals supply chains, trade balances, manufacturing value-added, +natural-resource rents, and governance proxies. + +Key capabilities: +- Fetch indicator observations for any country/year combination +- Search the indicator catalogue by keyword +- List countries with regional and income classification +- Pre-configured CMM-relevant indicator set for rapid profiling + +Country codes use ISO3 (e.g., USA, CHN, DEU). Date ranges accept single years +(\"2022\") or colon-separated ranges (\"2000:2023\"). No API key is required.""", +) + + +def get_client() -> WorldBankClient: + """Return a fresh WorldBankClient instance.""" + return WorldBankClient() + + +# ============================================================================= +# Overview Tools +# ============================================================================= + + +@mcp.tool() +async def get_api_status() -> dict: + """Check World Bank WDI API connectivity. + + Returns a dict with `status`, `api_key_configured`, and diagnostic fields. + """ + client = get_client() + return await client.check_status() + + +@mcp.tool() +async def list_cmm_indicators() -> dict: + """List the curated CMM-relevant WDI indicators. + + Returns the pre-configured indicators most useful for critical-minerals + supply-chain, trade, and governance analysis. + """ + return { + "count": len(CMM_WDI_INDICATORS), + "indicators": [{"code": code, "name": name} for code, name in CMM_WDI_INDICATORS.items()], + "usage": "Use get_indicator(country='USA', indicator='NY.GDP.MKTP.CD', date='2000:2023')", + } + + +@mcp.tool() +async def list_cmm_key_economies() -> dict: + """List ISO3 codes for key economies in critical-minerals supply chains.""" + return { + "count": len(CMM_KEY_ECONOMIES), + "economies": [{"iso3": iso3, "name": name} for iso3, name in CMM_KEY_ECONOMIES.items()], + } + + +@mcp.tool() +async def search_indicators(query: str, limit: int = 25) -> dict: + """Search the WDI indicator catalogue by keyword. + + Args: + query: Keyword (e.g., "manufacturing", "mineral rents", "renewable"). + limit: Maximum number of matches to return. + + Returns: + List of matching indicator metadata. + """ + client = get_client() + matches = await client.search_indicators(query) + return { + "query": query, + "count": len(matches[:limit]), + "total_matches": len(matches), + "indicators": matches[:limit], + } + + +@mcp.tool() +async def list_countries(search: str | None = None, limit: int = 100) -> dict: + """List World Bank countries with region and income classification. + + Args: + search: Optional name fragment to filter. + limit: Maximum results. + """ + client = get_client() + countries = await client.list_countries() + # Remove aggregate regions (no ISO2) unless explicitly requested + if search: + q = search.lower() + countries = [c for c in countries if q in (c.get("name") or "").lower()] + return { + "count": len(countries[:limit]), + "total": len(countries), + "countries": countries[:limit], + } + + +# ============================================================================= +# Data Query Tools +# ============================================================================= + + +@mcp.tool() +async def get_indicator( + country: str, + indicator: str, + date: str | None = None, + max_records: int = 500, +) -> dict: + """Fetch WDI indicator observations for a country or country group. + + Args: + country: ISO3 code or semicolon-separated list (e.g., "USA;CHN;DEU"). + Also accepts "all" or "WLD" for the world aggregate. + indicator: WDI indicator code (e.g., "NY.GDP.MKTP.CD"). + date: Year ("2022") or range ("2000:2023"). Omit for all available years. + max_records: Maximum records to return (paginated up to 1000 upstream). + + Returns: + Dict with `query`, `count`, and `records` list. + """ + client = get_client() + records = await client.get_indicator_observations( + country=country, indicator=indicator, date=date, per_page=min(max_records, 1000) + ) + return { + "query": {"country": country, "indicator": indicator, "date": date}, + "count": len(records), + "records": [r.model_dump() for r in records], + } + + +@mcp.tool() +async def get_cmm_profile( + country: str, + date: str = "2015:2023", +) -> dict: + """Fetch the curated CMM indicator bundle for a single country. + + Args: + country: ISO3 country code (e.g., "USA", "CHN", "CHL"). + date: Year or range (default: "2015:2023"). + + Returns: + Country profile keyed by indicator, each with a list of (year, value) pairs. + """ + client = get_client() + + sem = asyncio.Semaphore(5) + + async def fetch_one(indicator_code: str, indicator_name: str) -> tuple[str, list[dict]]: + async with sem: + try: + records = await client.get_indicator_observations( + country=country, indicator=indicator_code, date=date, per_page=100 + ) + except (httpx.HTTPError, OSError, ValueError): + return indicator_code, [] + return indicator_code, [ + {"year": r.year, "value": r.value, "name": indicator_name} + for r in records + if r.value is not None + ] + + results = await asyncio.gather( + *(fetch_one(code, name) for code, name in CMM_WDI_INDICATORS.items()) + ) + profile: dict[str, list[dict]] = dict(results) + + return { + "country": country, + "date_range": date, + "indicator_count": len(profile), + "profile": profile, + } + + +@mcp.tool() +async def compare_countries( + countries: str, + indicator: str, + date: str = "2015:2023", +) -> str: + """Produce a markdown comparison table of one indicator across multiple countries. + + Args: + countries: Semicolon-separated ISO3 codes (e.g., "USA;CHN;DEU;JPN"). + indicator: WDI indicator code (e.g., "NV.IND.MANF.ZS"). + date: Year or range. + + Returns: + Markdown-formatted pivot table (countries by years). + """ + client = get_client() + records = await client.get_indicator_observations( + country=countries, indicator=indicator, date=date, per_page=1000 + ) + if not records: + return f"No observations found for {indicator} across {countries} in {date}" + + # Pivot: {country: {year: value}} + pivot: dict[str, dict[int, float | None]] = {} + indicator_name = records[0].indicator_name or indicator + years_set: set[int] = set() + for r in records: + pivot.setdefault(r.country_name or r.country_code, {})[r.year] = r.value + years_set.add(r.year) + years = sorted(years_set) + + header = "| Country | " + " | ".join(str(y) for y in years) + " |\n" + sep = "|---|" + "|".join(["---"] * len(years)) + "|\n" + body = "" + for country, year_vals in sorted(pivot.items()): + row = f"| {country} |" + for y in years: + v = year_vals.get(y) + row += f" {v:,.3g} |" if v is not None else " — |" + body += row + "\n" + + return f"**{indicator_name} ({indicator})**\n\n{header}{sep}{body}" + + +def main(): + """Run the MCP server.""" + mcp.run() + + +if __name__ == "__main__": + main() diff --git a/packages/worldbank-mcp/tests/conftest.py b/packages/worldbank-mcp/tests/conftest.py new file mode 100644 index 0000000..825957e --- /dev/null +++ b/packages/worldbank-mcp/tests/conftest.py @@ -0,0 +1,13 @@ +"""Shared pytest fixtures for worldbank-mcp.""" + +from __future__ import annotations + +import pytest + +from worldbank_mcp.client import WorldBankClient + + +@pytest.fixture +def client() -> WorldBankClient: + """Return a WorldBankClient instance for tests.""" + return WorldBankClient() diff --git a/packages/worldbank-mcp/tests/test_models.py b/packages/worldbank-mcp/tests/test_models.py new file mode 100644 index 0000000..b9a72b5 --- /dev/null +++ b/packages/worldbank-mcp/tests/test_models.py @@ -0,0 +1,34 @@ +"""Unit tests for worldbank_mcp.models.""" + +from __future__ import annotations + +from worldbank_mcp.models import ( + CMM_KEY_ECONOMIES, + CMM_WDI_INDICATORS, + IndicatorObservation, +) + + +def test_cmm_indicators_nonempty(): + assert len(CMM_WDI_INDICATORS) > 0 + assert "NY.GDP.MKTP.CD" in CMM_WDI_INDICATORS + + +def test_cmm_key_economies_iso3(): + for code in CMM_KEY_ECONOMIES: + assert len(code) == 3 + assert code.isupper() + + +def test_indicator_observation_roundtrip(): + obs = IndicatorObservation( + country_code="USA", + country_name="United States", + indicator_code="NY.GDP.MKTP.CD", + indicator_name="GDP (current US$)", + year=2022, + value=2.5e13, + ) + assert obs.country_code == "USA" + assert obs.year == 2022 + assert obs.model_dump()["value"] == 2.5e13 diff --git a/pyproject.toml b/pyproject.toml index 9b16efe..ba96576 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,10 @@ known-first-party = [ "scholar", "scholar_mcp", "cmm_embedding", + "worldbank_mcp", + "fred_mcp", + "usitc_mcp", + "census_aes_mcp", ] [tool.ruff.format]