From 8875bec93ac9a3284cebad083ad7b82c6aca0d17 Mon Sep 17 00:00:00 2001 From: Evan Low Date: Fri, 26 Jun 2026 20:27:58 +0800 Subject: [PATCH 1/2] Expand smoke coverage and docs --- README.md | 15 +- docs/README.md | 3 + docs/TESTING.md | 20 + docs/TEST_COVERAGE_ANALYSIS.md | 508 +++---------------- docs/WEB_API_REFERENCE.md | 136 ++++- pytest.ini | 4 +- tests/conftest.py | 12 + tests/run_all_smoke.py | 65 ++- tests/smoke_manifest.py | 70 +++ tests/test_assisted_live_stop_requirement.py | 2 + tests/test_autonomous_engine.py | 3 + tests/test_autonomous_engine_basket.py | 1 + tests/test_order_lifecycle.py | 42 +- tests/test_recovery_manager.py | 42 +- 14 files changed, 452 insertions(+), 471 deletions(-) create mode 100644 tests/smoke_manifest.py diff --git a/README.md b/README.md index 0e22dcf..122c459 100644 --- a/README.md +++ b/README.md @@ -320,6 +320,19 @@ pytest test_backtest_engine.py -v - [Emergency Procedures](docs/runbooks/emergency-procedures.md) - [Architecture Documentation](docs/architecture/overview.md) +### Smoke Tests + +```bash +# Run safety-critical smoke coverage +python tests/run_all_smoke.py + +# Equivalent direct pytest command +pytest -m smoke --no-cov +``` + +The smoke suite covers the safety-critical autonomous, execution, emergency, +auth, broker-state, recovery, evidence-learning, and risk-lifecycle paths. + --- ## ๐Ÿ“š Documentation Index @@ -955,4 +968,4 @@ We love feature requests! Open an issue with the "enhancement" label. --- -**Happy Trading! ๐Ÿ“ˆ** \ No newline at end of file +**Happy Trading! ๐Ÿ“ˆ** diff --git a/docs/README.md b/docs/README.md index c4cbac9..01a1c03 100644 --- a/docs/README.md +++ b/docs/README.md @@ -18,6 +18,9 @@ python scripts/run_web.py # Run tests (Prime Directive: 100% pass rate) pytest -v +# Run safety-critical smoke tests +python tests/run_all_smoke.py + # Check coverage pytest --cov ``` diff --git a/docs/TESTING.md b/docs/TESTING.md index 91d129a..2bd952c 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -254,6 +254,26 @@ pytest tests/test_strategy_registry.py::TestStrategyRegistry::test_initializatio pytest -k "test_signal" ``` +### Smoke Suite + +```bash +# Run safety-critical smoke tests +python tests/run_all_smoke.py + +# Equivalent direct pytest command +pytest -m smoke --no-cov + +# Narrow the smoke suite while keeping the smoke marker +python tests/run_all_smoke.py -k emergency -x +``` + +The smoke inventory is maintained in `tests/smoke_manifest.py` and applied as +the `smoke` marker during pytest collection. It covers emergency stops, auth, +config security, order execution, TWS bridge behavior, autonomous API/runner +paths, live dry-run gates, order lifecycle, idempotency, recovery, replay, +market-data health, evidence learning, risk lifecycle, and capital-promotion +advisory behavior. + ### Coverage Analysis ```bash diff --git a/docs/TEST_COVERAGE_ANALYSIS.md b/docs/TEST_COVERAGE_ANALYSIS.md index 35914f2..5ae17f9 100644 --- a/docs/TEST_COVERAGE_ANALYSIS.md +++ b/docs/TEST_COVERAGE_ANALYSIS.md @@ -1,442 +1,102 @@ -# Test Coverage Analysis - Profile Comparison System +# Test Coverage Analysis -**Date:** January 22, 2026 -**Prime Directive Compliance Review** +**Date:** June 26, 2026 ---- +This document summarizes the current coverage posture after the autonomous and +evidence-learning work moved TWS Robot's safety-critical surface beyond the +older backtest/profile-comparison focus. -## โœ… Current Test Status +## Current Status -### Tests Passing (120 total) -- **test_profiles.py**: 36 tests โœ… (All passing, 0 warnings) -- **test_profile_comparison.py**: 20 tests โœ… (All passing, 0 warnings) -- **test_strategy_templates.py**: 20 tests โœ… (All passing, 0 warnings) -- **test_backtest_engine.py**: 44 tests โœ… (All passing, 0 warnings) +The historical ProfileComparator integration gap has been addressed: -**Result:** โœ… **100% pass rate, 0 warnings** (Prime Directive ยง1 โœ“) +- `backtest/profile_comparison.py` now builds a `StrategyConfig` from the risk + profile and passes that config to real strategy classes. +- `tests/test_profile_comparison.py` includes integration coverage for + `MomentumStrategy`, `MovingAverageCrossStrategy`, and + `MeanReversionStrategy`. ---- +The remaining coverage priorities are broader and safety-focused. -## โŒ Critical Gap Found: Integration Testing +## Critical Coverage Requirements -### Issue Discovered -When running `example_profile_comparison.py`, the script fails immediately: +The default pytest coverage configuration must include the current core +packages: -``` -โŒ Error: MomentumStrategy.__init__() got an unexpected keyword argument 'profile' -``` - -### Root Cause Analysis - -**File:** [backtest/profile_comparison.py](backtest/profile_comparison.py#L193-194) -```python -# Lines 193-194 -strategy_params_with_profile = strategy_params or {} -strategy_params_with_profile['profile'] = profile # โŒ WRONG! - -strategy = strategy_class(**strategy_params_with_profile) # โŒ FAILS HERE -``` - -**Problem:** `ProfileComparator` passes `profile` parameter to strategy constructors, but: - -**Actual Strategy Signatures:** -```python -# backtest/strategy_templates.py -class MomentumStrategy(Strategy): - def __init__(self, config: StrategyConfig, mom_config: Optional[MomentumConfig] = None): - # โŒ Does NOT accept 'profile' parameter! -``` - -```python -# backtest/strategy.py (base class) -class Strategy(ABC): - def __init__(self, config: StrategyConfig): - # โŒ Base class also doesn't accept 'profile' -``` - -### Why Tests Didn't Catch This - -**Analysis of test_profile_comparison.py:** - -1. **Uses Mock Strategies** - ```python - # test_profile_comparison.py doesn't test with REAL strategies - # It creates minimal test fixtures instead - ``` - -2. **No End-to-End Integration Tests** - - Tests verify comparison logic (rankings, statistics, etc.) - - Tests do NOT verify strategy instantiation with profiles - - Tests do NOT use actual strategy classes (MomentumStrategy, etc.) - -3. **Missing Test Cases:** - - โŒ Test `ProfileComparator.compare_profiles()` with `MomentumStrategy` - - โŒ Test `ProfileComparator.compare_profiles()` with `MovingAverageCrossStrategy` - - โŒ Test `ProfileComparator.compare_profiles()` with `MeanReversionStrategy` - - โŒ Test that profiles properly configure strategy behavior - - โŒ Test `example_profile_comparison.py` examples can run - ---- - -## ๐Ÿšจ Prime Directive Violations - -### Violation 1: Missing Integration Tests (ยง1) -**Prime Directive ยง1:** "100% Test Pass Rate + Zero Warnings - Non-Negotiable" -- **Substatus:** "Tests Must Exist Before Code Changes" -- **Violation:** ProfileComparator was created without end-to-end integration tests - -### Violation 2: API Verification Not Performed (ยง2) -**Prime Directive ยง2:** "Verify First, Code Second" -- **Quote:** "Never assume how existing code works. Always verify before implementing." -- **Violation:** ProfileComparator assumed strategies accept `profile` parameter without verifying - -### Violation 3: Missing Pre-Implementation Research (ยง3) -**Prime Directive - Pre-Implementation Checklist:** -- โŒ **Method signatures NOT checked** (Strategy.__init__ signature) -- โŒ **Usage examples NOT reviewed** (how strategies are instantiated) -- โŒ **Existing tests NOT consulted** (test_strategy_templates.py shows correct usage) - -**Quote from Prime Directive:** -> "For Integration Tests of Existing Code (NEW): -> โš ๏ธ CRITICAL: Research APIs BEFORE writing integration tests -> -> 3. Read Existing Tests (API Documentation) -> - Read the test file completely -> - Note ALL constructor signatures, method calls, return values" - ---- - -## ๐Ÿ“‹ Required Actions (Prime Directive Compliance) - -### Immediate (Critical Path) - -#### 1. Fix ProfileComparator API Usage โŒ BROKEN -**File:** `backtest/profile_comparison.py` lines 189-197 - -**Current (Wrong):** -```python -strategy_params_with_profile = strategy_params or {} -strategy_params_with_profile['profile'] = profile # โŒ Wrong! -strategy = strategy_class(**strategy_params_with_profile) -``` - -**Should Be (Option A - Pass via StrategyConfig):** -```python -# Create StrategyConfig with profile-based risk parameters -config = StrategyConfig( - name=profile.name, - symbols=symbols, - initial_capital=initial_capital, - max_position_size=profile.max_position_size_pct, - max_total_exposure=profile.max_total_exposure_pct, - # Add other profile parameters -) - -# Strategy-specific config (if needed) -strategy_config_param = strategy_params.get('strategy_config') if strategy_params else None - -# Instantiate strategy with correct signature -strategy = strategy_class(config, strategy_config_param) -``` - -**Should Be (Option B - Separate Strategy Factory):** -```python -# Create a factory that knows how to instantiate each strategy type -def create_strategy_from_profile( - strategy_class: type, - profile: RiskProfile, - symbols: List[str], - initial_capital: float, - strategy_params: Optional[Dict] = None -) -> Strategy: - """Create strategy instance configured with profile parameters""" - config = StrategyConfig( - name=profile.name, - symbols=symbols, - initial_capital=initial_capital, - max_position_size=profile.max_position_size_pct, - max_total_exposure=profile.max_total_exposure_pct, - ) - - # Handle different strategy types - if strategy_class == MomentumStrategy: - mom_config = strategy_params.get('mom_config') if strategy_params else None - return strategy_class(config, mom_config) - elif strategy_class == MovingAverageCrossStrategy: - ma_config = strategy_params.get('ma_config') if strategy_params else None - return strategy_class(config, ma_config) - # ... etc - else: - # Default to single-parameter constructor - return strategy_class(config) -``` - -#### 2. Add Integration Tests โŒ MISSING -**File:** `test_profile_comparison.py` (add new test class) - -```python -class TestProfileComparisonIntegrationWithRealStrategies: - """Integration tests using actual strategy implementations""" - - def test_compare_profiles_with_momentum_strategy(self): - """Test ProfileComparator with MomentumStrategy""" - comparator = ProfileComparator() - - # This should NOT raise TypeError about 'profile' parameter - result = comparator.compare_profiles( - strategy_class=MomentumStrategy, - profile_names=['conservative', 'moderate', 'aggressive'], - start_date='2023-01-01', - end_date='2023-03-31', - symbols=['AAPL'], - initial_capital=100000.0 - ) - - # Verify result structure - assert len(result.profile_results) == 3 - assert 'conservative' in result.profile_results - assert result.sharpe_ranking is not None - - def test_compare_profiles_with_ma_cross_strategy(self): - """Test ProfileComparator with MovingAverageCrossStrategy""" - # Similar test for MA cross strategy - pass - - def test_compare_profiles_with_mean_reversion_strategy(self): - """Test ProfileComparator with MeanReversionStrategy""" - # Similar test for mean reversion strategy - pass - - def test_profile_affects_strategy_behavior(self): - """Verify that different profiles actually change strategy behavior""" - # Conservative profile should result in smaller positions - # Aggressive profile should result in larger positions - # This tests that profiles are being applied correctly - pass -``` - -#### 3. Add Example Script Test โŒ MISSING -**File:** `test_examples.py` (new file) - -```python -""" -Tests for example scripts to ensure they run without errors -""" - -import pytest -from unittest.mock import patch, MagicMock -import example_profile_comparison - -class TestExampleScripts: - """Test that example scripts can be imported and run""" - - def test_example_profile_comparison_imports(self): - """Verify example_profile_comparison can be imported""" - # Already passes - we verified this - pass - - @patch('example_profile_comparison.input', return_value='') - @patch('example_profile_comparison.ProfileComparator.compare_profiles') - def test_example_1_basic_comparison_runs(self, mock_compare, mock_input): - """Test example 1 can run without errors""" - mock_compare.return_value = MagicMock() # Mock result - - # Should not raise exception - example_profile_comparison.example_1_basic_comparison() - - # Add tests for other examples... -``` +- `core` +- `data` +- `ai` +- `strategies` +- `backtest` +- `risk` +- `execution` +- `monitoring` +- `strategy` +- `autonomous` +- `web` -### Short-term (Required for Robustness) +`autonomous` is safety-critical because it contains scan/rank/plan/execute +logic, assisted-live gates, evidence learning, recovery, replay, broker +protection, idempotency, and capital-promotion advisory logic. -#### 4. Add API Documentation Comments โš ๏ธ IMPROVEMENT -**File:** `backtest/profile_comparison.py` +## Smoke Coverage Requirements -Add comprehensive docstring explaining correct usage: - -```python -class ProfileComparator: - """ - Compare different risk profiles by running backtests - - Usage Pattern: - ------------- - The comparator creates strategy instances using the following approach: - 1. Converts RiskProfile to StrategyConfig parameters - 2. Passes StrategyConfig to strategy constructor - 3. Strategy-specific config passed via strategy_params - - Example: - -------- - >>> comparator = ProfileComparator() - >>> result = comparator.compare_profiles( - ... strategy_class=MomentumStrategy, - ... profile_names=['conservative', 'aggressive'], - ... start_date='2023-01-01', - ... end_date='2023-12-31', - ... symbols=['AAPL'], - ... strategy_params={'mom_config': MomentumConfig(lookback_period=20)} - ... ) - - Strategy Requirements: - --------------------- - Strategy classes must accept: - - config: StrategyConfig (required) - - strategy_config: Optional[StrategySpecificConfig] (optional) - - โŒ Strategies do NOT receive RiskProfile directly - โœ… Risk parameters are passed via StrategyConfig - """ -``` - -#### 5. Document Integration Patterns ๐Ÿ“š DOCUMENTATION -**File:** `docs/INTEGRATION_TESTING.md` (new file) - -Document how to properly test integrations between ProfileComparator and strategies: -- API verification steps -- Constructor signature checking -- Parameter passing patterns -- Common pitfalls to avoid - -### Long-term (Best Practices) - -#### 6. Create Strategy Factory Pattern ๐Ÿ—๏ธ REFACTOR -**Rationale:** Centralize strategy instantiation logic -- Single place to handle different strategy constructors -- Easier to add new strategy types -- Better separation of concerns - -#### 7. Add Type Hints and Runtime Validation ๐Ÿ”’ SAFETY -```python -from typing import Protocol - -class StrategyProtocol(Protocol): - """Protocol for strategy classes that can be compared""" - def __init__(self, config: StrategyConfig, **kwargs): ... -``` - ---- - -## ๐Ÿ“Š Test Coverage Metrics - -### Current Coverage -| Component | Unit Tests | Integration Tests | E2E Tests | Coverage | -|-----------|------------|-------------------|-----------|----------| -| RiskProfile | โœ… 11 tests | โœ… 3 tests | โš ๏ธ None | 95% | -| ProfileManager | โœ… 15 tests | โœ… 2 tests | โš ๏ธ None | 90% | -| ProfileComparator | โœ… 14 tests | โš ๏ธ **0 tests** | โŒ **0 tests** | **40%** โš ๏ธ | -| Strategy Templates | โœ… 20 tests | โš ๏ธ None | โš ๏ธ None | 80% | -| Example Scripts | โŒ **0 tests** | โŒ **0 tests** | โŒ **0 tests** | **0%** โŒ | - -### Target Coverage (Prime Directive Compliant) -| Component | Unit Tests | Integration Tests | E2E Tests | Coverage | -|-----------|------------|-------------------|-----------|----------| -| RiskProfile | โœ… 11 tests | โœ… 3 tests | โœ… Added | 95% | -| ProfileManager | โœ… 15 tests | โœ… 2 tests | โœ… Added | 90% | -| ProfileComparator | โœ… 14 tests | โœ… **+5 tests** | โœ… **+3 tests** | **90%** | -| Strategy Templates | โœ… 20 tests | โœ… **+3 tests** | โœ… **+2 tests** | 90% | -| Example Scripts | โœ… **+6 tests** | โœ… **+6 tests** | โœ… **+6 tests** | **80%** | - -**New Tests Required:** ~31 additional tests - ---- - -## ๐ŸŽฏ Implementation Priority - -### Priority 1 (Blocking - Must Fix Immediately) -- [ ] Fix ProfileComparator strategy instantiation bug -- [ ] Add integration test for ProfileComparator + MomentumStrategy -- [ ] Verify fix with all three strategy types -- [ ] Run `example_profile_comparison.py` to confirm it works - -**Time Estimate:** 2-3 hours -**Blocker:** Cannot use profile comparison feature until fixed - -### Priority 2 (Critical - Required for Release) -- [ ] Add integration tests for all strategy types -- [ ] Add example script tests -- [ ] Add API verification to ProfileComparator -- [ ] Document correct usage patterns - -**Time Estimate:** 4-6 hours -**Impact:** Prevents future regressions - -### Priority 3 (Important - Technical Debt) -- [ ] Create strategy factory pattern -- [ ] Add comprehensive integration test suite -- [ ] Document integration testing guidelines -- [ ] Add type hints and protocols - -**Time Estimate:** 6-8 hours -**Impact:** Improves maintainability - ---- - -## ๐Ÿ“ Lessons Learned (Prime Directive Updates) - -### New Rule Proposal: "Integration Tests Required for Public APIs" - -**Add to Prime Directive ยง1:** -> **Integration Test Requirements:** -> - Any public API that orchestrates multiple components MUST have integration tests -> - Integration tests MUST use real implementations, not mocks -> - Integration tests MUST verify end-to-end workflows -> - Example scripts MUST have automated tests -> - Unit tests alone are insufficient for complex integrations - -### Enhanced Pre-Implementation Checklist - -**Add to Prime Directive - Pre-Implementation Checklist:** -> **โ˜‘๏ธ Before Writing Integration Code:** -> - [ ] Read constructor signatures of ALL components you'll integrate -> - [ ] Check existing tests for usage patterns -> - [ ] Verify parameter passing approaches (StrategyConfig vs direct params) -> - [ ] Document expected API in test docstrings BEFORE implementing -> - [ ] Write integration test skeleton BEFORE implementation -> - [ ] Verify integration test passes with real components - ---- - -## โœ… Success Criteria - -ProfileComparator system is fully tested when: - -1. โœ… All unit tests pass (120/120) โœ“ **Currently passing** -2. โŒ Integration tests with real strategies pass (0/5) โŒ **MISSING** -3. โŒ Example scripts can run without errors (0/6) โŒ **FAILING** -4. โœ… Zero warnings in test output โœ“ **Currently achieved** -5. โŒ Test coverage โ‰ฅ 85% for ProfileComparator (40%) โŒ **BELOW TARGET** -6. โŒ Documentation includes correct usage examples โš ๏ธ **INCOMPLETE** - -**Current Status:** ๐Ÿ”ด **2/6 criteria met** - Critical gaps remain - ---- - -## ๐Ÿš€ Immediate Next Steps +The smoke suite is marker-based: ```bash -# Step 1: Fix the bug -# Edit backtest/profile_comparison.py lines 189-197 - -# Step 2: Add integration test -# Edit test_profile_comparison.py - add new test class - -# Step 3: Verify fix -python -m pytest test_profile_comparison.py::TestProfileComparisonIntegrationWithRealStrategies -v - -# Step 4: Test example script -python example_profile_comparison.py -# (Press Ctrl+C after first example to avoid waiting through all 6) - -# Step 5: Run full test suite -python -m pytest test_profiles.py test_profile_comparison.py -v --tb=short - -# Step 6: Verify zero warnings -python -m pytest test_profiles.py test_profile_comparison.py -W error::Warning +python tests/run_all_smoke.py +pytest -m smoke ``` ---- +The maintained smoke inventory lives in: -**Remember:** Prime Directive ยง1 - "Tests Must Exist Before Code Changes" +```text +tests/smoke_manifest.py +``` -The ProfileComparator was implemented without adequate integration testing. This analysis ensures we don't repeat this mistake and provides a roadmap to achieve 100% Prime Directive compliance. +Smoke coverage should include: + +- emergency-stop and order-blocking regressions; +- authentication and config-security gates; +- core order execution and TWS bridge safety paths; +- autonomous API, dashboard, engine, runner, and live-runner paths; +- paper/live mode separation and dry-run guards; +- basket risk allocation; +- order lifecycle, broker fill ingestion, idempotency, recovery, and replay; +- quote freshness and market-data health guards; +- trade planning and sizing caps; +- evidence learning, setup eligibility, risk lifecycle, and capital promotion; +- operator-facing portfolio and FX research smoke coverage. + +## Documentation Coverage Requirements + +Documentation checks should cover more than file presence. For safety-critical +features, docs should state: + +- what changed; +- which order paths are affected; +- whether live behavior changed; +- whether defaults remain safe; +- which tests and smoke tests cover the behavior; +- known limitations and required manual checks. + +The following docs are canonical for autonomous/evidence-learning behavior: + +- `docs/AUTONOMOUS_TRADING_SYSTEM_SPEC.md` +- `docs/AUTONOMOUS_IMPLEMENTATION_TRACKER.md` +- `docs/AUTONOMOUS_EVIDENCE_LEARNING_SPEC.md` +- `docs/AUTONOMOUS_EVIDENCE_LEARNING_TRACKER.md` +- `docs/WEB_API_REFERENCE.md` +- `docs/TESTING.md` + +## Known Limitations + +- Example scripts are syntax-checked but are not comprehensively executed as + smoke tests because several examples expect market data or longer-running + backtest inputs. +- Some live-trading readiness checks remain intentionally manual because IBKR + account permissions, TWS/Gateway settings, and market-data subscriptions must + be verified by an operator. +- The smoke suite is intentionally broader than a minimal "app starts" check; + if runtime becomes too high, split it by marker expressions rather than + removing safety-critical modules from the manifest. diff --git a/docs/WEB_API_REFERENCE.md b/docs/WEB_API_REFERENCE.md index 00f1daf..76903af 100644 --- a/docs/WEB_API_REFERENCE.md +++ b/docs/WEB_API_REFERENCE.md @@ -24,14 +24,15 @@ 7. [Strategies API](#strategies-api) 8. [Backtest API](#backtest-api) 9. [Emergency Controls API](#emergency-controls-api) -10. [Events & Monitoring API](#events--monitoring-api) -11. [System API](#system-api) -12. [Account Intelligence API](#account-intelligence-api) -13. [AI Assistant APIs](#ai-assistant-apis) -14. [S&P 500 Screener API](#sp500-screener-api) -15. [STI Screener API](#sti-screener-api) -16. [HSI Screener API](#hsi-screener-api) -17. [Stock Analysis API](#stock-analysis-api) +10. [Autonomous Trading API](#autonomous-trading-api) +11. [Events & Monitoring API](#events--monitoring-api) +12. [System API](#system-api) +13. [Account Intelligence API](#account-intelligence-api) +14. [AI Assistant APIs](#ai-assistant-apis) +15. [S&P 500 Screener API](#sp500-screener-api) +16. [STI Screener API](#sti-screener-api) +17. [HSI Screener API](#hsi-screener-api) +18. [Stock Analysis API](#stock-analysis-api) --- @@ -2023,6 +2024,125 @@ Get current emergency and risk state. --- +## Autonomous Trading API + +Autonomous endpoints expose the safety-gated scan, paper runner, live-readiness, evidence, and operator control-tower surfaces. State-changing endpoints require the same authentication and CSRF protections as the rest of the web API. + +### `GET /api/autonomous/status` + +Return current autonomous configuration, mode state, and emergency-stop status. + +Safety notes: +- Read-only. +- Does not submit, cancel, or flatten orders. + +### `POST /api/autonomous/scan` + +Run the autonomous candidate scan and return ranked candidates and rejection reasons. + +Safety notes: +- Recommend-only scan path. +- Does not place orders. + +### `POST /api/autonomous/propose` + +Build a proposed autonomous trade plan after scanner, ranker, planner, and engine gates run. + +Safety notes: +- Returns a plan for review. +- Does not place live orders. + +### `POST /api/autonomous/execute-paper` + +Submit an approved autonomous plan through the paper-execution path. + +Safety notes: +- Paper only. +- Requires paper-mode readiness gates to pass. + +### `GET /api/autonomous/control-tower` + +Return the consolidated passive operator snapshot for autonomous readiness. + +The response includes mode state, connection verification, supervisor heartbeat, paper/live readiness, market-data health, cash/deployable-cash diagnostics, open autonomous trades, broker-visible orders, order-lifecycle state, basket-risk diagnostics, broker-protection diagnostics, recovery/risk lifecycle status, recent evidence, evidence-learning summary, and emergency-stop status. + +Safety notes: +- Read-only/passive. +- Does not call live-runner reconciliation. +- Does not submit orders, cancel orders, flatten positions, or advance lifecycle state. + +### `GET /api/autonomous/evidence` + +Return recent autonomous evidence records. + +Query parameters: +- `limit` (optional, default `100`) + +### `GET /api/autonomous/evidence/learning-status` + +Return the full read-only evidence-learning dashboard payload. + +Query parameters: +- `limit` (optional, default `1000`, max `1000`) +- `setup_limit` (optional, default `50`, max `250`) +- `current_level` (optional, default `0`, range `0-6`) +- `recent_trades` (optional, default `10`) +- `min_trades` (optional, default `3`) +- `expected_r_delta` (optional, default `0.25`) + +### `GET /api/autonomous/evidence/setup-performance` + +Return setup-level realized performance, eligibility, and evidence diagnostics. + +### `GET /api/autonomous/evidence/promotion-report` + +Return the advisory capital promotion report. + +Safety notes: +- Advisory only. +- Does not apply approvals, change capital caps, or enable live trading. + +### `GET /api/autonomous/evidence/weak-setups` + +Return setups that should remain paper-only, reduced, weak, or retired based on realized evidence. + +### `GET /api/autonomous/evidence/drift-report` + +Return setup families whose recent realized evidence diverges from historical evidence. + +### `GET /api/autonomous/live/status` + +Return live-runner status and readiness gates. + +Safety notes: +- May perform live-runner readiness/reconciliation work. +- Does not submit a new live order by itself. + +### `POST /api/autonomous/emergency-stop` + +Create the autonomous emergency-stop marker, turn autonomous paper/live modes off, stop lifecycle workers, pause the live continuous supervisor, and write an audit event. + +Optional body fields: +- `cancel_pending_entries` (boolean): request cancellation for pending live autonomous entry order IDs only. + +Safety notes: +- Does not flatten positions. +- Preserves protective exit orders unless a separate explicit flatten control is used. + +### `POST /api/autonomous/emergency-reset` + +Reset an autonomous emergency-stop marker created by `/api/autonomous/emergency-stop`. + +Required body fields: +- `confirm: true` + +Safety notes: +- Keeps autonomous modes off. +- Does not clear global/manual emergency markers from `/api/emergency/halt`. +- Does not reactivate live trading. + +--- + ## Events & Monitoring API ### `GET /api/events/stream` diff --git a/pytest.ini b/pytest.ini index 82e293d..46eaf2d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -20,6 +20,7 @@ addopts = --cov=execution --cov=monitoring --cov=strategy + --cov=autonomous --cov=web --cov-report=term-missing --cov-report=html:htmlcov @@ -31,6 +32,7 @@ testpaths = tests markers = unit: Unit tests (fast, no external dependencies) integration: Integration tests (may require database or API) + smoke: Smoke tests for safety-critical trading, autonomous, auth, API, and execution paths slow: Slow running tests requires_tws: Tests that require TWS connection @@ -50,7 +52,7 @@ log_file_level = DEBUG # Coverage options [coverage:run] -source = core,data,strategies,backtest,risk,execution,monitoring,strategy,web +source = core,data,strategies,backtest,risk,execution,monitoring,strategy,autonomous,web omit = */tests/* */venv/* diff --git a/tests/conftest.py b/tests/conftest.py index c9e0769..d4ddf63 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,11 @@ """Shared pytest fixtures for the tws_robot test suite.""" +from pathlib import Path + import pytest +from tests.smoke_manifest import SMOKE_TEST_MODULES + @pytest.fixture(autouse=True) def _isolate_stop_file(tmp_path, monkeypatch): @@ -10,3 +14,11 @@ def _isolate_stop_file(tmp_path, monkeypatch): monkeypatch.setattr(api_emergency, "EMERGENCY_STOP_FILE", tmp_path / "EMERGENCY_STOP") monkeypatch.setattr(api_autonomous, "EMERGENCY_STOP_FILE", tmp_path / "EMERGENCY_STOP") + + +def pytest_collection_modifyitems(config, items): + """Apply the smoke marker from the maintained smoke inventory.""" + + for item in items: + if Path(str(item.path)).name in SMOKE_TEST_MODULES: + item.add_marker(pytest.mark.smoke) diff --git a/tests/run_all_smoke.py b/tests/run_all_smoke.py index 40d06c5..711cad5 100644 --- a/tests/run_all_smoke.py +++ b/tests/run_all_smoke.py @@ -5,57 +5,52 @@ Scripts/python.exe tests/run_all_smoke.py -k emergency Scripts/python.exe tests/run_all_smoke.py -- -x -By default, this script runs smoke-focused test modules that cover critical -safety, auth, API, and execution paths. +By default, this script runs tests marked ``smoke``. The smoke inventory lives +in ``tests/smoke_manifest.py`` and is applied during pytest collection. """ from __future__ import annotations -import argparse +from pathlib import Path import sys import pytest +PROJECT_ROOT = Path(__file__).resolve().parents[1] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) -DEFAULT_SMOKE_TARGETS = [ - "tests/test_safety_regression.py", - "tests/test_web_api.py", - "tests/test_portfolio_analysis.py", - "tests/test_auth.py", - "tests/test_config_security.py", - "tests/test_order_executor.py", - "tests/test_tws_bridge.py", - "tests/test_fx_research.py", -] - - -def parse_args() -> tuple[argparse.Namespace, list[str]]: - parser = argparse.ArgumentParser( - description="Run the TWS Robot smoke suite.", - ) - parser.add_argument( - "pytest_args", - nargs="*", - help="Optional extra pytest args (for example: -k emergency -x).", - ) - # Allow direct pytest flags (for example: -q, -k, -x) without forcing - # users to add "--" in front of them. - return parser.parse_known_args() +from tests.smoke_manifest import SMOKE_TEST_MODULES def main() -> int: - args, passthrough = parse_args() + pytest_args = _normalise_pytest_args(sys.argv[1:]) + pytest_cmd = ["-m", "smoke"] + if not _requests_coverage(pytest_args): + pytest_cmd.append("--no-cov") + pytest_cmd.extend(pytest_args) - pytest_cmd = [*DEFAULT_SMOKE_TARGETS, *args.pytest_args, *passthrough] - print("Running smoke suite:") - for target in DEFAULT_SMOKE_TARGETS: - print(f" - {target}") - extra_args = [*args.pytest_args, *passthrough] - if extra_args: - print("Extra pytest args:", " ".join(extra_args)) + print(f"Running smoke suite from {len(SMOKE_TEST_MODULES)} marked modules.") + if pytest_args: + print("Extra pytest args:", " ".join(pytest_args)) return pytest.main(pytest_cmd) +def _normalise_pytest_args(args: list[str]) -> list[str]: + """Allow both ``run_all_smoke.py -k x`` and ``run_all_smoke.py -- -k x``.""" + + if args and args[0] == "--": + return args[1:] + return args + + +def _requests_coverage(args: list[str]) -> bool: + return any( + arg == "--cov" or arg == "--no-cov" or arg.startswith("--cov=") + for arg in args + ) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/tests/smoke_manifest.py b/tests/smoke_manifest.py new file mode 100644 index 0000000..f1dc34b --- /dev/null +++ b/tests/smoke_manifest.py @@ -0,0 +1,70 @@ +"""Smoke-suite inventory for safety-critical TWS Robot paths. + +The smoke marker is applied during collection from this manifest so the suite +can be run with ``pytest -m smoke`` without repeating a long file list in +scripts, CI jobs, or PR notes. +""" + +from __future__ import annotations + + +SMOKE_TEST_MODULES = { + # Original operator-facing smoke coverage. + "test_safety_regression.py", + "test_web_api.py", + "test_portfolio_analysis.py", + "test_auth.py", + "test_config_security.py", + "test_order_executor.py", + "test_tws_bridge.py", + "test_fx_research.py", + + # Autonomous trading API, engine, runner, and dashboard safety paths. + "test_api_autonomous.py", + "test_api_autonomous_evidence.py", + "test_api_autonomous_live.py", + "test_api_autonomous_runner.py", + "test_api_trading_readiness.py", + "test_assisted_live_stop_requirement.py", + "test_autonomous_dashboard.py", + "test_autonomous_engine.py", + "test_autonomous_engine_basket.py", + "test_autonomous_engine_evidence.py", + "test_autonomous_engine_vix_gate.py", + "test_autonomous_exit_manager.py", + "test_autonomous_live_runner.py", + "test_autonomous_live_runner_basket.py", + "test_autonomous_paper_adapter.py", + "test_autonomous_runner.py", + "test_autonomous_trade_store.py", + "test_controlled_live_trading.py", + "test_live_dry_run_guard.py", + + # Order lifecycle, broker-state, recovery, market-data, and replay guards. + "test_basket_planner.py", + "test_broker_fill_ingestor.py", + "test_continuous_supervisor.py", + "test_idempotency.py", + "test_market_data_health.py", + "test_market_data_provider.py", + "test_order_lifecycle.py", + "test_recovery_manager.py", + "test_replay_engine.py", + + # Trade planning, evidence learning, risk lifecycle, and promotion gates. + "test_adaptive_edge_estimator.py", + "test_capital_promotion.py", + "test_emergency_controls.py", + "test_evidence_aware_sizer.py", + "test_evidence_calibrator.py", + "test_evidence_learning_summary.py", + "test_risk_lifecycle.py", + "test_setup_eligibility.py", + "test_setup_registry.py", + "test_trade_evidence_store.py", + "test_trade_planner.py", + "test_trade_planner_evidence_sizing.py", + "test_trade_planner_execution_quality.py", + "test_trade_planner_fractional_drawdown.py", + "test_trade_planner_sizing.py", +} diff --git a/tests/test_assisted_live_stop_requirement.py b/tests/test_assisted_live_stop_requirement.py index 92a0728..d094604 100644 --- a/tests/test_assisted_live_stop_requirement.py +++ b/tests/test_assisted_live_stop_requirement.py @@ -46,6 +46,7 @@ def test_assisted_live_share_plan_requires_stop_price(): AutonomousTradingConfig( mode=AutonomousMode.ASSISTED_LIVE, require_stop_price_for_assisted_live=True, + market_data_health_guard_enabled=False, ) ) reasons = [] @@ -66,6 +67,7 @@ def test_assisted_live_share_plan_allows_valid_support_stop(): AutonomousTradingConfig( mode=AutonomousMode.ASSISTED_LIVE, require_stop_price_for_assisted_live=True, + market_data_health_guard_enabled=False, ) ) diff --git a/tests/test_autonomous_engine.py b/tests/test_autonomous_engine.py index dd61481..86111ba 100644 --- a/tests/test_autonomous_engine.py +++ b/tests/test_autonomous_engine.py @@ -268,6 +268,7 @@ def test_live_execution_blocked_unless_explicitly_enabled(tmp_path): cfg = AutonomousTradingConfig( mode=AutonomousMode.ASSISTED_LIVE, allow_live_execution=False, + market_data_health_guard_enabled=False, emergency_stop_file=str(tmp_path / "EMERGENCY_STOP"), audit_log_dir=str(tmp_path), ) @@ -284,6 +285,7 @@ def test_live_execution_with_flag_but_no_confirm_requires_confirmation(tmp_path) cfg = AutonomousTradingConfig( mode=AutonomousMode.ASSISTED_LIVE, allow_live_execution=True, + market_data_health_guard_enabled=False, emergency_stop_file=str(tmp_path / "EMERGENCY_STOP"), audit_log_dir=str(tmp_path), ) @@ -410,6 +412,7 @@ def test_live_execution_returns_live_plan_ready_when_allow_live_true_and_confirm mode=AutonomousMode.ASSISTED_LIVE, allow_live_execution=True, require_user_confirmation=True, + market_data_health_guard_enabled=False, max_trades_per_day=5, emergency_stop_file=str(tmp_path / "EMERGENCY_STOP"), audit_log_dir=str(tmp_path), diff --git a/tests/test_autonomous_engine_basket.py b/tests/test_autonomous_engine_basket.py index 5b3c987..dff2eeb 100644 --- a/tests/test_autonomous_engine_basket.py +++ b/tests/test_autonomous_engine_basket.py @@ -166,6 +166,7 @@ def test_engine_assisted_live_returns_basket_live_plan_ready(tmp_path): cfg = _basket_config( mode=AutonomousMode.ASSISTED_LIVE, allow_live_execution=True, + market_data_health_guard_enabled=False, audit_log_dir=str(tmp_path), ) engine = _engine(cfg) diff --git a/tests/test_order_lifecycle.py b/tests/test_order_lifecycle.py index 4bbd1fe..dfad9fb 100644 --- a/tests/test_order_lifecycle.py +++ b/tests/test_order_lifecycle.py @@ -114,6 +114,46 @@ def execute_signal(self, strategy_name, signal, current_equity, positions): return result +class _LiveMarketDataProvider: + def __init__(self, symbol: str = "AAA"): + now = datetime.now(timezone.utc) + self.quote = MarketDataQuote( + symbol=symbol, + bid=99.95, + ask=100.05, + last=100.0, + timestamp=now, + bid_timestamp=now, + ask_timestamp=now, + last_timestamp=now, + source=IBKR_SOURCE, + market_data_type=IBKR_MARKET_DATA_TYPE_LIVE, + feed_healthy=True, + ) + self.subscribed: list[str] = [] + + def subscribe(self, symbols): + self.subscribed.extend([str(symbol).upper() for symbol in symbols]) + + def unsubscribe(self, symbols): + pass + + def latest_quote(self, symbol): + if str(symbol).upper() == self.quote.symbol: + return self.quote + return None + + def status(self): + return MarketDataProviderStatus( + provider=IBKR_SOURCE, + connected=True, + healthy=True, + subscribed_symbols=list(self.subscribed), + market_data_type=IBKR_MARKET_DATA_TYPE_LIVE, + reason="test market-data provider", + ) + + def _signal(symbol: str = "AAA") -> CandidateSignal: return CandidateSignal( symbol=symbol, @@ -191,9 +231,9 @@ def _runner( rejected_order_ids_provider=rejected_order_ids_provider, filled_order_ids_provider=filled_order_ids_provider, broker_open_orders_provider=broker_open_orders_provider, + market_data_provider=_LiveMarketDataProvider(), order_lifecycle_store=lifecycle_store, idempotency_store=idempotency_store, - market_data_provider=_LiveMarketDataProvider(), ) diff --git a/tests/test_recovery_manager.py b/tests/test_recovery_manager.py index f5d1271..24365e7 100644 --- a/tests/test_recovery_manager.py +++ b/tests/test_recovery_manager.py @@ -92,6 +92,46 @@ def execute_signal(self, strategy_name, signal, current_equity, positions): ) +class _LiveMarketDataProvider: + def __init__(self, symbol: str = "AAA"): + now = datetime.now(timezone.utc) + self.quote = MarketDataQuote( + symbol=symbol, + bid=99.95, + ask=100.05, + last=100.0, + timestamp=now, + bid_timestamp=now, + ask_timestamp=now, + last_timestamp=now, + source=IBKR_SOURCE, + market_data_type=IBKR_MARKET_DATA_TYPE_LIVE, + feed_healthy=True, + ) + self.subscribed: list[str] = [] + + def subscribe(self, symbols): + self.subscribed.extend([str(symbol).upper() for symbol in symbols]) + + def unsubscribe(self, symbols): + pass + + def latest_quote(self, symbol): + if str(symbol).upper() == self.quote.symbol: + return self.quote + return None + + def status(self): + return MarketDataProviderStatus( + provider=IBKR_SOURCE, + connected=True, + healthy=True, + subscribed_symbols=list(self.subscribed), + market_data_type=IBKR_MARKET_DATA_TYPE_LIVE, + reason="test market-data provider", + ) + + def _trade( *, trade_id: str = "trade-1", @@ -204,9 +244,9 @@ def _runner( deployable_cash_provider=lambda: 50_000.0, broker_positions_provider=lambda: broker_positions, broker_open_orders_provider=lambda: broker_open_orders, + market_data_provider=_LiveMarketDataProvider(), order_lifecycle_store=lifecycle_store, idempotency_store=idempotency_store, - market_data_provider=_LiveMarketDataProvider(), ) From 517f1f094fd4f8fc60f9eae9f8a730bce9b65da6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:30:53 +0000 Subject: [PATCH 2/2] Fix review comments: normalize quote symbol comparison and enable market-data health guard in tests --- tests/test_assisted_live_stop_requirement.py | 2 -- tests/test_autonomous_engine.py | 3 --- tests/test_autonomous_engine_basket.py | 1 - tests/test_order_lifecycle.py | 4 ++-- tests/test_recovery_manager.py | 4 ++-- 5 files changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/test_assisted_live_stop_requirement.py b/tests/test_assisted_live_stop_requirement.py index d094604..92a0728 100644 --- a/tests/test_assisted_live_stop_requirement.py +++ b/tests/test_assisted_live_stop_requirement.py @@ -46,7 +46,6 @@ def test_assisted_live_share_plan_requires_stop_price(): AutonomousTradingConfig( mode=AutonomousMode.ASSISTED_LIVE, require_stop_price_for_assisted_live=True, - market_data_health_guard_enabled=False, ) ) reasons = [] @@ -67,7 +66,6 @@ def test_assisted_live_share_plan_allows_valid_support_stop(): AutonomousTradingConfig( mode=AutonomousMode.ASSISTED_LIVE, require_stop_price_for_assisted_live=True, - market_data_health_guard_enabled=False, ) ) diff --git a/tests/test_autonomous_engine.py b/tests/test_autonomous_engine.py index 86111ba..dd61481 100644 --- a/tests/test_autonomous_engine.py +++ b/tests/test_autonomous_engine.py @@ -268,7 +268,6 @@ def test_live_execution_blocked_unless_explicitly_enabled(tmp_path): cfg = AutonomousTradingConfig( mode=AutonomousMode.ASSISTED_LIVE, allow_live_execution=False, - market_data_health_guard_enabled=False, emergency_stop_file=str(tmp_path / "EMERGENCY_STOP"), audit_log_dir=str(tmp_path), ) @@ -285,7 +284,6 @@ def test_live_execution_with_flag_but_no_confirm_requires_confirmation(tmp_path) cfg = AutonomousTradingConfig( mode=AutonomousMode.ASSISTED_LIVE, allow_live_execution=True, - market_data_health_guard_enabled=False, emergency_stop_file=str(tmp_path / "EMERGENCY_STOP"), audit_log_dir=str(tmp_path), ) @@ -412,7 +410,6 @@ def test_live_execution_returns_live_plan_ready_when_allow_live_true_and_confirm mode=AutonomousMode.ASSISTED_LIVE, allow_live_execution=True, require_user_confirmation=True, - market_data_health_guard_enabled=False, max_trades_per_day=5, emergency_stop_file=str(tmp_path / "EMERGENCY_STOP"), audit_log_dir=str(tmp_path), diff --git a/tests/test_autonomous_engine_basket.py b/tests/test_autonomous_engine_basket.py index dff2eeb..5b3c987 100644 --- a/tests/test_autonomous_engine_basket.py +++ b/tests/test_autonomous_engine_basket.py @@ -166,7 +166,6 @@ def test_engine_assisted_live_returns_basket_live_plan_ready(tmp_path): cfg = _basket_config( mode=AutonomousMode.ASSISTED_LIVE, allow_live_execution=True, - market_data_health_guard_enabled=False, audit_log_dir=str(tmp_path), ) engine = _engine(cfg) diff --git a/tests/test_order_lifecycle.py b/tests/test_order_lifecycle.py index dfad9fb..fde0ed2 100644 --- a/tests/test_order_lifecycle.py +++ b/tests/test_order_lifecycle.py @@ -58,7 +58,7 @@ def unsubscribe(self, symbols): pass def latest_quote(self, symbol): - if str(symbol).upper() == self.quote.symbol: + if str(symbol).upper() == self.quote.symbol.upper(): return self.quote return None @@ -139,7 +139,7 @@ def unsubscribe(self, symbols): pass def latest_quote(self, symbol): - if str(symbol).upper() == self.quote.symbol: + if str(symbol).upper() == self.quote.symbol.upper(): return self.quote return None diff --git a/tests/test_recovery_manager.py b/tests/test_recovery_manager.py index 24365e7..e68274d 100644 --- a/tests/test_recovery_manager.py +++ b/tests/test_recovery_manager.py @@ -64,7 +64,7 @@ def unsubscribe(self, symbols): pass def latest_quote(self, symbol): - if str(symbol).upper() == self.quote.symbol: + if str(symbol).upper() == self.quote.symbol.upper(): return self.quote return None @@ -117,7 +117,7 @@ def unsubscribe(self, symbols): pass def latest_quote(self, symbol): - if str(symbol).upper() == self.quote.symbol: + if str(symbol).upper() == self.quote.symbol.upper(): return self.quote return None