From 3619c6c535270593107c9e41f65bc52a446ed7e6 Mon Sep 17 00:00:00 2001 From: "Arta Asadi(workstation)" Date: Fri, 12 Jun 2026 21:47:30 +0330 Subject: [PATCH] feat(env): update environment configuration and API key handling - Modify .env.example and env.example.prod to include RADA_API_KEY and RADA_ENV variables. - Update docker-compose files to use environment variables for Grafana admin credentials. - Adjust Makefile test command to exclude GPU and integration tests. - Enhance mkdocs.yml navigation and update license in pyproject.toml to Apache-2.0. - Improve dashboard Nginx configuration for security and API key handling. - Refactor dashboard components to handle API keys and improve error handling. - Add new audit export script and update documentation for audit endpoints. - Introduce synthetic context in decision-making processes and enhance metrics handling. --- .env.example | 3 +- .github/workflows/ci.yml | 23 +++++ .github/workflows/integration.yml | 8 ++ Makefile | 2 +- apps/streamlit/dashboard.py | 24 ++++- dashboard/nginx.conf | 45 +++++++++ dashboard/src/pages/Decisions.jsx | 28 ++++-- dashboard/src/pages/Overview.jsx | 21 ++-- dashboard/src/pages/ReviewQueue.jsx | 21 ++-- docker-compose.monitoring.yml | 4 +- docker-compose.prod.yml | 5 +- docs/audit.md | 5 +- docs/conf.py | 2 +- docs/human-validation.md | 20 +++- docs/index.md | 14 +++ docs/monitoring.md | 15 ++- docs/runbook.md | 5 +- env.example.prod | 10 +- mkdocs.yml | 3 +- pyproject.toml | 3 +- runpod/train.py | 39 ++++++-- scripts/compare_pre_post_train.py | 5 +- scripts/export_audit.py | 46 +++++++++ scripts/reflection_train.py | 11 ++- src/rada/adapters/scenario_reasoner.py | 38 ++++++-- src/rada/audit/api.py | 23 ++++- src/rada/audit/store.py | 8 ++ src/rada/audit/writer.py | 44 ++++++++- src/rada/calc/engine.py | 18 +++- src/rada/calc/runner.py | 24 ++++- src/rada/core/decision_loop.py | 50 +++++++--- src/rada/core/reflection_loop.py | 95 ++++++++++++++----- src/rada/core/search_loop.py | 36 +++++-- src/rada/data/analytics.py | 2 + src/rada/data/bus.py | 25 ++++- src/rada/data/storage.py | 23 +++++ src/rada/data/timescale_store.py | 48 ++++++++-- src/rada/feedback/api.py | 27 ++++-- src/rada/feedback/auto_flag.py | 7 ++ src/rada/feedback/schemas.py | 24 ++++- src/rada/feedback/store.py | 83 ++++++++++------ src/rada/interfaces/data_store.py | 4 + .../llm_integration/adapters/_http_base.py | 68 +++++++++++-- .../adapters/custom_adapter.py | 6 +- .../adapters/litellm_adapter.py | 6 +- src/rada/llm_integration/context.py | 42 ++++++-- src/rada/main.py | 42 ++++++-- src/rada/policies/registry.py | 15 ++- src/rada/schemas.py | 1 + src/rada/search/constants.py | 3 + src/rada/search/game_theory.py | 3 +- src/rada/search/mcts.py | 4 +- src/rada/search/risk_selection.py | 12 ++- src/rada/security/__init__.py | 6 ++ src/rada/security/auth.py | 32 +++++++ src/rada/security/validation.py | 25 +++++ src/rada/utils/artifact_manifest.py | 14 +-- storage/ftp_sync.py | 3 + storage/sync_daemon.sh | 4 +- tests/integration/test_api_auth.py | 50 ++++++++++ tests/integration/test_feedback_api.py | 14 +-- .../test_search_decision_integration.py | 7 +- tests/unit/test_bus.py | 1 + tests/unit/test_data_query.py | 49 ++++++++++ tests/unit/test_feedback_export_cards.py | 10 +- tests/unit/test_http_adapter.py | 59 ++++++++++++ tests/unit/test_model_registry.py | 6 +- tests/unit/test_smoke.py | 45 ++++++++- tests/unit/test_utils_metrics.py | 14 +++ 69 files changed, 1252 insertions(+), 230 deletions(-) create mode 100644 docs/index.md create mode 100644 scripts/export_audit.py create mode 100644 src/rada/search/constants.py create mode 100644 src/rada/security/__init__.py create mode 100644 src/rada/security/auth.py create mode 100644 src/rada/security/validation.py create mode 100644 tests/integration/test_api_auth.py create mode 100644 tests/unit/test_data_query.py create mode 100644 tests/unit/test_http_adapter.py create mode 100644 tests/unit/test_utils_metrics.py diff --git a/.env.example b/.env.example index c173375..b077781 100644 --- a/.env.example +++ b/.env.example @@ -4,7 +4,8 @@ RADA_EVENT_BUS_MODE=inmemory RADA_REDIS_URL=redis://localhost:6379/0 RADA_DATABASE_URL=postgresql+asyncpg://rada:rada@localhost:5432/rada -RADA_SQLITE_URL=sqlite+aiosqlite:///./rada.db +RADA_SQLITE_URL=sqlite:///./rada.db +RADA_API_KEY= # Change these in real environments. POSTGRES_USER=rada diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cda530..a3c95ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,3 +26,26 @@ jobs: - name: Unit tests run: pytest tests/unit -q --strict-markers -m "not gpu and not integration" + + - name: Integration tests + run: pytest tests/integration -q --strict-markers -m integration + + dashboard: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: dashboard/package-lock.json + + - name: Install dashboard dependencies + working-directory: dashboard + run: npm ci + + - name: Build dashboard + working-directory: dashboard + run: npm run build diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 98ce855..4d70d03 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -2,6 +2,8 @@ name: Integration (manual) on: workflow_dispatch: + pull_request: + branches: [main, develop] jobs: integration: @@ -27,3 +29,9 @@ jobs: - uses: actions/checkout@v4 - name: Build Docker image run: docker build -t rada:ci . + - name: Smoke test image health + run: | + docker run -d --name rada-ci -p 18000:8000 rada:ci + sleep 5 + curl -fsS http://127.0.0.1:18000/health + docker rm -f rada-ci diff --git a/Makefile b/Makefile index 8500b56..6dd1517 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ down: docker compose down test: - pytest + pytest tests/unit -m "not gpu and not integration" lint: ruff check . diff --git a/apps/streamlit/dashboard.py b/apps/streamlit/dashboard.py index bfa552c..2bb8322 100644 --- a/apps/streamlit/dashboard.py +++ b/apps/streamlit/dashboard.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os import httpx @@ -10,6 +11,14 @@ from rada.utils.metrics import get_metrics_snapshot API_URL = os.getenv("RADA_API_URL", "http://localhost:8000") +API_KEY = os.getenv("RADA_API_KEY", "") + + +def _headers() -> dict[str, str]: + headers: dict[str, str] = {} + if API_KEY: + headers["X-API-Key"] = API_KEY + return headers def _submit(*, client_action: str, decision_id: str, note: str) -> None: @@ -21,7 +30,12 @@ def _submit(*, client_action: str, decision_id: str, note: str) -> None: } try: with httpx.Client(timeout=10.0) as client: - client.post(f"{API_URL}/feedback/submit", json=payload) + response = client.post( + f"{API_URL}/feedback/submit", + json=payload, + headers=_headers(), + ) + response.raise_for_status() st.success(f"{client_action} submitted for {decision_id}") except httpx.HTTPError as exc: st.error(str(exc)) @@ -42,7 +56,13 @@ def _submit(*, client_action: str, decision_id: str, note: str) -> None: st.subheader("Flagged decisions") try: with httpx.Client(timeout=10.0) as client: - pending = client.get(f"{API_URL}/feedback/pending").json().get("pending", []) + response = client.get(f"{API_URL}/feedback/pending", headers=_headers()) + response.raise_for_status() + try: + pending = response.json().get("pending", []) + except json.JSONDecodeError as exc: + st.error(f"Invalid JSON from feedback API: {exc}") + pending = [] except httpx.HTTPError as exc: st.error(f"Cannot reach feedback API at {API_URL}: {exc}") pending = [] diff --git a/dashboard/nginx.conf b/dashboard/nginx.conf index bd58c98..c922610 100644 --- a/dashboard/nginx.conf +++ b/dashboard/nginx.conf @@ -4,31 +4,76 @@ server { root /usr/share/nginx/html; index index.html; + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + add_header X-XSS-Protection "1; mode=block" always; + location / { try_files $uri $uri/ /index.html; } location /health { proxy_pass http://rada:8000/health; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; } location /metrics { + limit_except GET { + deny all; + } proxy_pass http://rada:8000/metrics; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; } location /ingest { + limit_except POST { + deny all; + } proxy_pass http://rada:8000/ingest; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-API-Key $http_x_api_key; } location /bootstrap-demo { + limit_except POST { + deny all; + } proxy_pass http://rada:8000/bootstrap-demo; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-API-Key $http_x_api_key; } location /audit/ { + limit_except GET { + deny all; + } proxy_pass http://rada:8000/audit/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-API-Key $http_x_api_key; } location /feedback/ { proxy_pass http://rada:8000/feedback/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-API-Key $http_x_api_key; } } diff --git a/dashboard/src/pages/Decisions.jsx b/dashboard/src/pages/Decisions.jsx index 8287533..a42e50e 100644 --- a/dashboard/src/pages/Decisions.jsx +++ b/dashboard/src/pages/Decisions.jsx @@ -2,6 +2,14 @@ import { useState } from "react"; import { useMutation } from "@tanstack/react-query"; import { apiFetch } from "../api/rest"; +function parsePositiveNumber(value, field) { + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`${field} must be a positive number`); + } + return parsed; +} + export function Decisions() { const [symbol, setSymbol] = useState("BTCUSD"); const [price, setPrice] = useState("42000"); @@ -10,12 +18,22 @@ export function Decisions() { const [audit, setAudit] = useState(null); const [error, setError] = useState(""); + const loadAudit = async (decisionId) => { + try { + const chain = await apiFetch(`/audit/decision/${decisionId}`); + setAudit(chain); + } catch (err) { + setError(`Audit fetch failed: ${err.message}`); + setAudit(null); + } + }; + const ingest = useMutation({ mutationFn: async () => { const body = { symbol, - price: Number(price), - volume: Number(volume), + price: parsePositiveNumber(price, "Price"), + volume: parsePositiveNumber(volume, "Volume"), timestamp: new Date().toISOString(), }; return apiFetch("/ingest", { method: "POST", body: JSON.stringify(body) }); @@ -23,8 +41,7 @@ export function Decisions() { onSuccess: async (data) => { setError(""); setLastDecisionId(data.decision_id); - const chain = await apiFetch(`/audit/decision/${data.decision_id}`); - setAudit(chain); + await loadAudit(data.decision_id); }, onError: (err) => setError(err.message), }); @@ -34,8 +51,7 @@ export function Decisions() { onSuccess: async (data) => { setError(""); setLastDecisionId(data.decision_id); - const chain = await apiFetch(`/audit/decision/${data.decision_id}`); - setAudit(chain); + await loadAudit(data.decision_id); }, onError: (err) => setError(err.message), }); diff --git a/dashboard/src/pages/Overview.jsx b/dashboard/src/pages/Overview.jsx index c79c53a..fe705d1 100644 --- a/dashboard/src/pages/Overview.jsx +++ b/dashboard/src/pages/Overview.jsx @@ -42,15 +42,24 @@ export function Overview() {

Unreachable: {health.error.message}

)} {health.isSuccess && ( -

Healthy — {health.data.status}

+

+ Healthy — {health.data?.status ?? "unknown"} +

)} -
- - - -
+ {metrics.isLoading &&

Loading metrics…

} + {metrics.isError && ( +

Metrics unavailable: {metrics.error.message}

+ )} + + {metrics.isSuccess && ( +
+ + + +
+ )} ); } diff --git a/dashboard/src/pages/ReviewQueue.jsx b/dashboard/src/pages/ReviewQueue.jsx index af22518..f31aa1b 100644 --- a/dashboard/src/pages/ReviewQueue.jsx +++ b/dashboard/src/pages/ReviewQueue.jsx @@ -4,7 +4,7 @@ import { apiFetch } from "../api/rest"; export function ReviewQueue() { const qc = useQueryClient(); - const [note, setNote] = useState(""); + const [notes, setNotes] = useState({}); const pending = useQuery({ queryKey: ["feedback-pending"], @@ -15,8 +15,12 @@ export function ReviewQueue() { const submit = useMutation({ mutationFn: (payload) => apiFetch("/feedback/submit", { method: "POST", body: JSON.stringify(payload) }), - onSuccess: () => { - setNote(""); + onSuccess: (_data, variables) => { + setNotes((prev) => { + const next = { ...prev }; + delete next[variables.decision_id]; + return next; + }); qc.invalidateQueries({ queryKey: ["feedback-pending"] }); }, }); @@ -34,6 +38,7 @@ export function ReviewQueue() { {pending.isLoading &&

Loading queue…

} {pending.isError &&

{pending.error.message}

} + {submit.isError &&

Submit failed: {submit.error.message}

} {items.length === 0 && pending.isSuccess && (

No pending reviews.

@@ -42,7 +47,7 @@ export function ReviewQueue() {