diff --git a/README.md b/README.md index ccdb6882b9..dbdae65012 100755 --- a/README.md +++ b/README.md @@ -270,6 +270,8 @@ OHLCV: open, high, low, and close prices; volume. adjusted_close: adjusted close Technical indicators: 'macd', 'boll_ub', 'boll_lb', 'rsi_30', 'dx_30', 'close_30_sma', 'close_60_sma'. Users also can add new features. +Optional structured market sentiment features can be added on top of existing OHLCV datasets through `finrl.meta.preprocessor.adanos_sentiment`. See `examples/FinRL_StockTrading_2026_1_data_with_adanos.py` for a recent-window example using lagged Adanos sentiment features. The helper defaults to `lag=1`, producing t-1 columns such as `adanos_buzz_mean_lag1` to avoid same-day sentiment leakage; users can set `lag=2` or higher for a more conservative delay. + ## Installation + [Install description for all operating systems (MAC OS, Ubuntu, Windows 10)](./docs/source/start/installation.rst) diff --git a/docs/source/faq.rst b/docs/source/faq.rst index a4bc32a7d8..cfb6c3efdb 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -66,11 +66,11 @@ Outline - :raw-html:`Can a sentiment feature be added to improve the model's performance? ` - *Yes, you can add it. Remember to check on the code that this additional feature is being fed to the model (state)* + *Yes, you can add it. Remember to check on the code that this additional feature is being fed to the model (state). See the optional `finrl.meta.preprocessor.adanos_sentiment` helper for one structured market sentiment workflow.* - :raw-html:`Is there a good free source for market sentiment to use as a feature? ` - *No, you'll have to use a paid service or library/code to scrape news and obtain the sentiment from them (normally, using deep learning and NLP)* + *Usually no. In practice you'll often use a paid service or your own scraping/NLP pipeline. FinRL now includes an optional Adanos-based example showing how a structured paid sentiment feed can be merged into the state as lagged features.* .. _Section-2: diff --git a/docs/source/finrl_meta/Data_layer.rst b/docs/source/finrl_meta/Data_layer.rst index b7e4e16dfa..26cbd7a6d2 100644 --- a/docs/source/finrl_meta/Data_layer.rst +++ b/docs/source/finrl_meta/Data_layer.rst @@ -40,3 +40,5 @@ Feature Engineering Feature engineering is the last part of the data layer. We automate the calculation of technical indicators by connecting the Stockstats or TAlib library in our data processor. Common technical indicators including Moving Average Convergence Divergence (MACD), Relative Strength Index (RSI), Average Directional Index (ADX), and Commodity Channel Index (CCI), and so on, are supported. Users can also quickly add indicators from other libraries, or add the user-defined features directly. Users can add their features by two ways: 1) Write user-defined feature extraction functions directly. The returned features will be added to a feature array. 2) Store the features in a file, and move it to a specified folder. Then, these features will be obtained by reading from the specified file. + +For example, optional structured market sentiment features can be merged into the standard OHLCV pipeline through ``finrl.meta.preprocessor.adanos_sentiment`` and then appended to the environment ``tech_indicator_list`` as lagged state features. The helper defaults to ``lag=1`` and creates t-1 columns such as ``adanos_buzz_mean_lag1`` to avoid same-day sentiment leakage; ``lag=2`` or higher can be used when a more conservative delay is desired. diff --git a/examples/FinRL_StockTrading_2026_1_data_with_adanos.py b/examples/FinRL_StockTrading_2026_1_data_with_adanos.py new file mode 100644 index 0000000000..96a6a53c8f --- /dev/null +++ b/examples/FinRL_StockTrading_2026_1_data_with_adanos.py @@ -0,0 +1,79 @@ +# Stock Trading 2026 Part 1 (optional Adanos sentiment features). +# +# This example keeps the classic FinRL Yahoo + technical-indicator pipeline and +# optionally augments it with lagged market sentiment features when +# ADANOS_API_KEY is available. The default lag=1 creates t-1 sentiment +# columns to avoid same-day sentiment leakage. +from __future__ import annotations + +import itertools +import os + +import pandas as pd + +from finrl.config import INDICATORS +from finrl.meta.preprocessor.adanos_sentiment import ADANOS_SENTIMENT_FEATURES +from finrl.meta.preprocessor.adanos_sentiment import add_adanos_market_sentiment +from finrl.meta.preprocessor.preprocessors import data_split +from finrl.meta.preprocessor.preprocessors import FeatureEngineer +from finrl.meta.preprocessor.yahoodownloader import YahooDownloader + +TRAIN_START_DATE = "2025-12-01" +TRAIN_END_DATE = "2026-02-15" +TRADE_START_DATE = "2026-02-15" +TRADE_END_DATE = "2026-03-20" +TICKERS = ["AAPL", "MSFT", "NVDA", "TSLA"] + +api_key = os.getenv("ADANOS_API_KEY") + +df_raw = YahooDownloader( + start_date=TRAIN_START_DATE, + end_date=TRADE_END_DATE, + ticker_list=TICKERS, +).fetch_data() + +feature_engineer = FeatureEngineer( + use_technical_indicator=True, + tech_indicator_list=INDICATORS, + use_vix=True, + use_turbulence=False, + user_defined_feature=False, +) + +processed = feature_engineer.preprocess_data(df_raw) + +list_ticker = processed["tic"].unique().tolist() +list_date = list( + pd.date_range(processed["date"].min(), processed["date"].max()).astype(str) +) +combination = list(itertools.product(list_date, list_ticker)) + +processed_full = pd.DataFrame(combination, columns=["date", "tic"]).merge( + processed, on=["date", "tic"], how="left" +) +processed_full = processed_full[processed_full["date"].isin(processed["date"])] +processed_full = processed_full.sort_values(["date", "tic"]).fillna(0) + +processed_with_sentiment = add_adanos_market_sentiment( + processed_full, + api_key=api_key, + days=90, +) + +if api_key: + feature_columns = INDICATORS + ADANOS_SENTIMENT_FEATURES + print("Added Adanos sentiment features:") + print(ADANOS_SENTIMENT_FEATURES) +else: + feature_columns = INDICATORS + print("ADANOS_API_KEY is not set. Proceeding without optional sentiment features.") + +train = data_split(processed_with_sentiment, TRAIN_START_DATE, TRAIN_END_DATE) +trade = data_split(processed_with_sentiment, TRADE_START_DATE, TRADE_END_DATE) + +train.to_csv("train_data_with_adanos.csv") +trade.to_csv("trade_data_with_adanos.csv") + +print("Feature columns to feed into the environment:") +print(feature_columns) +print("Saved train_data_with_adanos.csv and trade_data_with_adanos.csv") diff --git a/examples/README.md b/examples/README.md index 7a6834bf89..1be6ef4e27 100644 --- a/examples/README.md +++ b/examples/README.md @@ -30,6 +30,14 @@ python examples/FinRL_StockTrading_2026_1_data.py This script downloads DOW 30 stock data from Yahoo Finance, adds technical indicators (MACD, RSI, etc.), VIX, and turbulence index, then splits the data into training set (2014–2025) and trading set (2026-01-01 to 2026-03-20), saving them as `train_data.csv` and `trade_data.csv`. +**Optional: Add structured market sentiment features** + +```bash +ADANOS_API_KEY=your_key_here python examples/FinRL_StockTrading_2026_1_data_with_adanos.py +``` + +This companion example keeps the classic Yahoo Finance preprocessing flow and optionally augments the state with lagged retail sentiment features from Adanos. Without `ADANOS_API_KEY`, the script runs as a no-op enrichment step and keeps the original feature set. The default `lag=1` creates t-1 feature columns such as `adanos_buzz_mean_lag1` so same-day sentiment cannot leak into the current state; pass `lag=2` or higher for a more conservative delay. + **2. Train DRL Agents** ```bash diff --git a/finrl/meta/preprocessor/__init__.py b/finrl/meta/preprocessor/__init__.py index e69de29bb2..1d6bb55dbf 100644 --- a/finrl/meta/preprocessor/__init__.py +++ b/finrl/meta/preprocessor/__init__.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from finrl.meta.preprocessor.adanos_sentiment import ADANOS_SENTIMENT_FEATURES +from finrl.meta.preprocessor.adanos_sentiment import ADANOS_SENTIMENT_SOURCES +from finrl.meta.preprocessor.adanos_sentiment import add_adanos_market_sentiment + +__all__ = [ + "ADANOS_SENTIMENT_FEATURES", + "ADANOS_SENTIMENT_SOURCES", + "add_adanos_market_sentiment", +] diff --git a/finrl/meta/preprocessor/adanos_sentiment.py b/finrl/meta/preprocessor/adanos_sentiment.py new file mode 100644 index 0000000000..4bdc7e8c97 --- /dev/null +++ b/finrl/meta/preprocessor/adanos_sentiment.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Iterable +from typing import Mapping +from urllib.parse import quote + +import pandas as pd +import requests + +ADANOS_SENTIMENT_SOURCES = ("reddit", "x", "news", "polymarket") + +ADANOS_SENTIMENT_FEATURES = [ + "adanos_buzz_mean_lag1", + "adanos_sentiment_mean_lag1", + "adanos_source_coverage_lag1", + "adanos_reddit_buzz_lag1", + "adanos_reddit_sentiment_lag1", + "adanos_x_buzz_lag1", + "adanos_x_sentiment_lag1", + "adanos_news_buzz_lag1", + "adanos_news_sentiment_lag1", + "adanos_polymarket_buzz_lag1", + "adanos_polymarket_sentiment_lag1", +] + + +@dataclass(frozen=True) +class _DailyFeatureKey: + ticker: str + date: str + + +def _safe_float(value) -> float | None: + if value is None: + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _normalize_date_column(series: pd.Series) -> pd.Series: + return pd.to_datetime(series).dt.strftime("%Y-%m-%d") + + +def _resolve_days(df: pd.DataFrame, date_column: str, lag: int, max_days: int) -> int: + dates = pd.to_datetime(df[date_column]) + span_days = max(int((dates.max() - dates.min()).days) + 1 + lag, 7) + return min(span_days, max_days) + + +def _build_feature_frame( + ticker_history: Mapping[_DailyFeatureKey, dict[str, float | None]], + sources: Iterable[str], +) -> pd.DataFrame: + source_buzz_columns = [f"adanos_{source}_buzz" for source in sources] + source_sentiment_columns = [f"adanos_{source}_sentiment" for source in sources] + + frame = pd.DataFrame(ticker_history.values()) + for column in source_buzz_columns + source_sentiment_columns: + if column not in frame.columns: + frame[column] = pd.NA + + frame["adanos_buzz_mean"] = frame[source_buzz_columns].mean(axis=1, skipna=True) + frame["adanos_sentiment_mean"] = frame[source_sentiment_columns].mean( + axis=1, skipna=True + ) + frame["adanos_source_coverage"] = ( + frame[source_buzz_columns].notna().sum(axis=1).astype(float) + ) + return frame + + +def add_adanos_market_sentiment( + df: pd.DataFrame, + *, + api_key: str | None = None, + base_url: str = "https://api.adanos.org", + days: int | None = None, + sources: Iterable[str] = ADANOS_SENTIMENT_SOURCES, + lag: int = 1, + ticker_column: str = "tic", + date_column: str = "date", + timeout: int = 15, + fillna: bool = True, + session: requests.Session | None = None, +) -> pd.DataFrame: + """ + Optionally enrich a FinRL dataframe with lagged Adanos market sentiment. + + The function is intentionally fail-open: importing this module never reads + environment variables or calls the external API. Without an explicit + ``api_key``, or when remote requests fail, a copy of the original dataframe + is returned unchanged. + + Sentiment features are shifted per ticker before merging back into the + state. The default ``lag=1`` creates t-1 columns such as + ``adanos_buzz_mean_lag1`` to avoid same-day sentiment leakage. Use + ``lag=2`` or higher for a more conservative delay. + """ + + enriched = df.copy() + if enriched.empty or not api_key: + return enriched + + sources = tuple(source for source in sources if source in ADANOS_SENTIMENT_SOURCES) + if not sources: + return enriched + + normalized_dates = _normalize_date_column(enriched[date_column]) + enriched[date_column] = normalized_dates + + effective_days = days or _resolve_days(enriched, date_column, lag, max_days=365) + client = session or requests.Session() + headers = {"X-API-Key": api_key, "Accept": "application/json"} + + ticker_history: dict[_DailyFeatureKey, dict[str, float | None]] = {} + + for ticker in sorted(enriched[ticker_column].dropna().astype(str).unique()): + for source in sources: + endpoint = ( + f"{base_url.rstrip('/')}/{source}/stocks/v1/stock/{quote(ticker)}" + ) + try: + response = client.get( + endpoint, + params={"days": effective_days}, + headers=headers, + timeout=timeout, + ) + if response.status_code == 404: + continue + response.raise_for_status() + payload = response.json() + except requests.RequestException: + continue + + for item in payload.get("daily_trend") or []: + date_value = item.get("date") + if not date_value: + continue + + key = _DailyFeatureKey(ticker=ticker, date=str(date_value)) + row = ticker_history.setdefault( + key, + { + ticker_column: ticker, + date_column: str(date_value), + }, + ) + row[f"adanos_{source}_buzz"] = _safe_float(item.get("buzz_score")) + row[f"adanos_{source}_sentiment"] = _safe_float( + item.get("sentiment_score") + ) + + if not ticker_history: + return enriched + + feature_frame = _build_feature_frame(ticker_history, sources) + base_frame = ( + enriched[[ticker_column, date_column]] + .drop_duplicates() + .sort_values([ticker_column, date_column]) + .reset_index(drop=True) + ) + feature_frame = base_frame.merge( + feature_frame, + on=[ticker_column, date_column], + how="left", + ) + lag_base_columns = [ + "adanos_buzz_mean", + "adanos_sentiment_mean", + "adanos_source_coverage", + *[f"adanos_{source}_buzz" for source in sources], + *[f"adanos_{source}_sentiment" for source in sources], + ] + + feature_frame = feature_frame.sort_values([ticker_column, date_column]) + shifted = feature_frame.groupby(ticker_column)[lag_base_columns].shift(lag) + shifted.columns = [f"{column}_lag{lag}" for column in lag_base_columns] + feature_frame = pd.concat( + [feature_frame[[ticker_column, date_column]].reset_index(drop=True), shifted], + axis=1, + ) + + enriched = enriched.merge( + feature_frame, on=[ticker_column, date_column], how="left" + ) + + if fillna: + lag_columns = [f"{column}_lag{lag}" for column in lag_base_columns] + enriched[lag_columns] = enriched[lag_columns].apply( + pd.to_numeric, errors="coerce" + ) + enriched[lag_columns] = enriched[lag_columns].fillna(0.0) + + return enriched diff --git a/unit_tests/preprocessors/test_adanos_sentiment.py b/unit_tests/preprocessors/test_adanos_sentiment.py new file mode 100644 index 0000000000..4e377dad43 --- /dev/null +++ b/unit_tests/preprocessors/test_adanos_sentiment.py @@ -0,0 +1,217 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pandas as pd +import pytest + + +MODULE_PATH = ( + Path(__file__).resolve().parents[2] + / "finrl" + / "meta" + / "preprocessor" + / "adanos_sentiment.py" +) +SPEC = importlib.util.spec_from_file_location("adanos_sentiment", MODULE_PATH) +ADANOS_SENTIMENT = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +sys.modules[SPEC.name] = ADANOS_SENTIMENT +SPEC.loader.exec_module(ADANOS_SENTIMENT) + +add_adanos_market_sentiment = ADANOS_SENTIMENT.add_adanos_market_sentiment + + +class DummyResponse: + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + + def json(self): + return self._payload + + def raise_for_status(self): + if self.status_code >= 400: + raise RuntimeError(f"bad status: {self.status_code}") + + +class DummySession: + def __init__(self, responses): + self.responses = responses + self.calls = [] + + def get(self, url, params, headers, timeout): + self.calls.append((url, params, headers, timeout)) + return self.responses[url] + + +class FailingSession: + def get(self, url, params, headers, timeout): + raise AssertionError("API should not be called without an explicit api_key") + + +def _simple_ohlcv_frame(ticker="AAPL", dates=None): + if dates is None: + dates = ["2026-03-01", "2026-03-02"] + return pd.DataFrame( + { + "date": dates, + "tic": [ticker] * len(dates), + "close": [100.0 + idx for idx in range(len(dates))], + } + ) + + +def test_add_adanos_market_sentiment_is_noop_without_api_key(): + df = _simple_ohlcv_frame() + + result = add_adanos_market_sentiment(df, api_key=None) + + assert list(result.columns) == ["date", "tic", "close"] + pd.testing.assert_frame_equal(result, df) + assert result is not df + + +def test_add_adanos_market_sentiment_default_no_key_skips_network_call(): + df = _simple_ohlcv_frame() + + result = add_adanos_market_sentiment(df, session=FailingSession()) + + pd.testing.assert_frame_equal(result, df) + + +def test_add_adanos_market_sentiment_merges_lagged_daily_features(): + df = _simple_ohlcv_frame() + + responses = { + "https://api.adanos.org/reddit/stocks/v1/stock/AAPL": DummyResponse( + { + "daily_trend": [ + { + "date": "2026-03-01", + "buzz_score": 40.0, + "sentiment_score": 0.10, + }, + { + "date": "2026-03-02", + "buzz_score": 60.0, + "sentiment_score": 0.30, + }, + ] + } + ), + "https://api.adanos.org/news/stocks/v1/stock/AAPL": DummyResponse( + { + "daily_trend": [ + { + "date": "2026-03-01", + "buzz_score": 50.0, + "sentiment_score": 0.20, + }, + { + "date": "2026-03-02", + "buzz_score": 70.0, + "sentiment_score": 0.40, + }, + ] + } + ), + } + + session = DummySession(responses) + result = add_adanos_market_sentiment( + df, + api_key="test-key", + sources=("reddit", "news"), + session=session, + ) + + assert result.loc[0, "adanos_buzz_mean_lag1"] == 0.0 + assert result.loc[0, "adanos_source_coverage_lag1"] == 0.0 + assert result.loc[1, "adanos_reddit_buzz_lag1"] == 40.0 + assert result.loc[1, "adanos_news_buzz_lag1"] == 50.0 + assert result.loc[1, "adanos_buzz_mean_lag1"] == 45.0 + assert result.loc[1, "adanos_sentiment_mean_lag1"] == pytest.approx(0.15) + assert result.loc[1, "adanos_source_coverage_lag1"] == 2.0 + assert len(session.calls) == 2 + + +def test_add_adanos_market_sentiment_ignores_missing_sources(): + df = _simple_ohlcv_frame(ticker="TSLA") + + responses = { + "https://api.adanos.org/reddit/stocks/v1/stock/TSLA": DummyResponse( + {"daily_trend": []}, + status_code=404, + ), + "https://api.adanos.org/x/stocks/v1/stock/TSLA": DummyResponse( + { + "daily_trend": [ + { + "date": "2026-03-01", + "buzz_score": 55.0, + "sentiment_score": 0.25, + } + ] + } + ), + } + + session = DummySession(responses) + result = add_adanos_market_sentiment( + df, + api_key="test-key", + sources=("reddit", "x"), + session=session, + ) + + assert "adanos_x_buzz_lag1" in result.columns + assert result.loc[1, "adanos_x_buzz_lag1"] == 55.0 + assert result.loc[1, "adanos_source_coverage_lag1"] == 1.0 + + +def test_add_adanos_market_sentiment_supports_conservative_lag(): + df = _simple_ohlcv_frame( + dates=["2026-03-01", "2026-03-02", "2026-03-03"], + ) + + responses = { + "https://api.adanos.org/reddit/stocks/v1/stock/AAPL": DummyResponse( + { + "daily_trend": [ + { + "date": "2026-03-01", + "buzz_score": 40.0, + "sentiment_score": 0.10, + }, + { + "date": "2026-03-02", + "buzz_score": 60.0, + "sentiment_score": 0.30, + }, + { + "date": "2026-03-03", + "buzz_score": 80.0, + "sentiment_score": 0.50, + }, + ] + } + ) + } + + result = add_adanos_market_sentiment( + df, + api_key="test-key", + sources=("reddit",), + lag=2, + session=DummySession(responses), + ) + + assert "adanos_buzz_mean_lag2" in result.columns + assert "adanos_buzz_mean_lag1" not in result.columns + assert result.loc[0, "adanos_buzz_mean_lag2"] == 0.0 + assert result.loc[1, "adanos_buzz_mean_lag2"] == 0.0 + assert result.loc[2, "adanos_reddit_buzz_lag2"] == 40.0 + assert result.loc[2, "adanos_sentiment_mean_lag2"] == pytest.approx(0.10)