Refactoring

.env.exemple

@@ -1,8 +1,9 @@
 QTSBE_PORT=5002
-QTSBE_AUTO_FETCH_PORT=5004
 QTSBE_HOST=0.0.0.0
 QTSBE_DEBUG=false
-QTSBE_CACHE_TYPE=simple
+QTSBE_CACHE_TYPE=SimpleCache
 QTSBE_CACHE_DEFAULT_TIMEOUT=300
 QTSBE_CORS_ORIGINS=http://127.0.0.1:1337,http://localhost:1337
-QTSBE_CORS_METHODS='GET,PUT,POST,DELETE,OPTIONS'
+QTSBE_CORS_METHODS='GET,PUT,POST,DELETE,OPTIONS'
+QTSBE_CRON_PORT=5004
+QTSBE_CRON_INTERVAL_SECONDS=10

.github/workflows/pytest.yml

@@ -0,0 +1,34 @@
+name: Python Tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install pytest pytest-flask
+
+      - name: Setup Test Environment
+        run: |
+          cp .env.exemple .env
+          mkdir -p data/bank
+          python3 -c "import h5py, numpy as np; f=h5py.File('data/bank/qtsbe_data.h5', 'w'); d=np.ones((100, 6)); d[:,0]=np.arange(1700000000000, 1700000000000 + 100*86400000, 86400000); d[:,1:5]=np.random.uniform(50000, 60000, (100, 4)); f.create_dataset('Binance_BTCUSDC_1d', data=d); f.create_dataset('Binance_BTCUSDC_1w', data=d); f.create_dataset('Binance_BTCUSDC_1s', data=d); f.close()"
+
+      - name: Run Pytest
+        run: pytest

.gitignore

@@ -1,22 +1,11 @@
 *.pyc
 .DS_Store
 
-logs/*
-
-config/*
-!config/README.md
-
-
 data/bank/*
 
 api/strategies/*
 !api/strategies/default.py
 !api/strategies/rsi_example.py
 !api/strategies/README.md
-integrations/plotly/saved_results/*
-
-
-tools/strategy_scanner/_results/*
-!tools/strategy_scanner/_results/README.md
 
 .env

Dockerfile

@@ -3,6 +3,8 @@ FROM python:3.11-slim as builder
 RUN apt-get update && apt-get install -y \
     gcc \
     python3-dev \
+    libhdf5-dev \
+    pkg-config \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
@@ -15,10 +17,13 @@ RUN pip install --no-cache-dir -r requirements.txt
 
 FROM python:3.11-slim
 
+RUN apt-get update && apt-get install -y \
+    libhdf5-dev \
+    && rm -rf /var/lib/apt/lists/*
+
 WORKDIR /app
 
 COPY --from=builder /opt/venv /opt/venv
-
 COPY . .
 
 ENV PATH="/opt/venv/bin:$PATH"

README.md

@@ -11,7 +11,7 @@ QTSBE is a robust, open-source platform designed for backtesting quantitative tr
   <img src="https://img.shields.io/badge/flask-%23000.svg?style=for-the-badge&logo=flask&logoColor=white">
   <img src="https://img.shields.io/badge/pandas-%23150458.svg?style=for-the-badge&logo=pandas&logoColor=white">
   <img src="https://img.shields.io/badge/numpy-%23013243.svg?style=for-the-badge&logo=numpy&logoColor=white">
-  <img src="https://img.shields.io/badge/Plotly-%233F4F75.svg?style=for-the-badge&logo=plotly&logoColor=white">
+  <img src="https://img.shields.io/badge/pytest-%23C1E81C.svg?style=for-the-badge&logo=pytest&logoColor=black">
   <img src="https://img.shields.io/badge/Binance-FCD535?style=for-the-badge&logo=binance&logoColor=white">
   <img src="https://img.shields.io/badge/Yahoo!-6001D2?style=for-the-badge&logo=Yahoo!&logoColor=white">
   <img src="https://img.shields.io/badge/Linux-FCC624?style=for-the-badge&logo=linux&logoColor=black">
@@ -92,53 +92,49 @@ docker-compose up
 1. **Strategy Development**
    - Base template: `api/strategies/default.py`
    - Example implementation: `api/strategies/rsi_example`
-   - Create your strategy file with an `analyse` function
-   - Implement technical indicators directly in your strategy code
+   - Create your strategy file with required functions (`buy_signal`, `sell_signal`, `Indicators`)
 
 2. **API Deployment**
 ```bash
 python api/api.py
 ```
 
 3. **API Documentation**
-   - Access the Swagger UI documentation: `http://127.0.0.1:5002/docs`
-   - Comprehensive endpoint documentation and testing interface
-
-4. **Running Tests**
-> [!TIP]
-> Run fixtures of the API with : `pytest tests/test_api_endpoints.py`
-
-5. **Visualization Tools**
-
-   a. Generate Plotly Charts:
-   ```bash
-   sh tests/integrations/plotly_unit.sh
-   ```
-
-   b. Discord Integration:
-   - Configure: `integrations/discord_chat_bot/bot.py`
-   - Launch:
-   ```bash
-   sh sh/discord_chat_bot.sh
-   ```
-
-   c. Custom Interface Development
-   - Framework available for building custom web interfaces
-   - Reference the Smartswap project for implementation examples
-
-6. **Automated Data Collection**
-   - Configure: `tools/auto_fetch/config.json`
-   - Launch collector:
-   ```bash
-   sh sh/auto_fetch.sh
-   ```
-
-## Visualization Examples
+   - API Documentation is available via **Postman**.
+   - Import the collection located in `docs/postman/QTSBE_API_Collection.json`.
+   - See [Postman Documentation](docs/postman.md) for detailed instructions.
 
-<p align="center">
-  <img src="https://github.com/simonpotel/QTSBE/blob/master/assets/integration/plotly/white_3.png?raw=true" width="100%">
-  <img src="https://github.com/simonpotel/QTSBE/blob/master/assets/integration/plotly/black_2.png?raw=true" width="100%">
-</p>
+4. **Automated Data Collection**
+   - Configure: `config/data_cron.json`
+   - Launch collector: `python tools/data/cron.py`
+
+## Documentation
+
+For more detailed information, please refer to the following guides:
+
+- [Strategy Development](docs/strategies.md) : How to build and add your own trading strategies.
+- [Configuration](docs/configuration.md) : Detailed explanation of all configuration files.
+- [Data Management](docs/data.md) : Understanding HDF5 storage and data structures.
+- [Tooling](docs/tools.md) : Overview of data fetching and maintenance tools.
+- [Postman Guide](docs/postman.md) : How to import and use the API collection.
+
+## Testing
+
+QTSBE uses **Pytest** for automated integration testing.
+
+| Endpoint | Test Objective |
+|----------|----------------|
+| `/QTSBE/health` | Service health and external connectivity |
+| `/QTSBE/get_tokens` | Listing of available HDF5 datasets |
+| `/QTSBE/get_tokens_stats` | Live performance and metadata validation |
+| `/QTSBE/get_strategies` | Discovery of local Python strategy modules |
+| `/QTSBE/analyse` | Core backtesting engine execution |
+| `/QTSBE/analyse_custom` | Dynamic execution of injected strategy code |
+
+To run the tests simply execute:
+```bash
+pytest
+```
 
 ## License
 

api/api.py

@@ -1,21 +1,14 @@
 from flask import Flask, request, jsonify
 from flask_cors import CORS
 from flask_caching import Cache
-from loguru import logger
-import os
-import sys
-import json
-import importlib.util
-from flask_swagger_ui import get_swaggerui_blueprint
-import shutil
+import os, sys, json, importlib.util
 from dotenv import load_dotenv
 
 load_dotenv()
 
 from core.analysis import analyse
 from routes.analyse import register_analyse_routes
 from routes.analyse_custom import register_analyse_custom_routes
-from routes.scan import register_scan_routes
 from routes.strategies import register_strategy_routes
 from routes.get_tokens import register_get_tokens_routes
 from routes.get_tokens_stats import register_get_tokens_stats_routes
@@ -25,127 +18,44 @@
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
 
 def load_config():
-    config_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'config', 'api.json')
-    try:
-        with open(config_path, 'r') as f:
-            return json.load(f)
-    except Exception as e:
-        logger.error(f"Failed to load config from {config_path}: {e}")
-        sys.exit(1)
-
-config = load_config()
-strategies_folder = r"api/strategies"
-strategies = {}
-
-def reload_loguru_config():
-    logger.remove()
-    logger.add(sys.stdout, level="DEBUG")
-    if config['server']['debug']:
-        logger.add(r"logs/api/debug.log", level="DEBUG")
-    else:
-        logger.add(r"logs/api/logs.log", level="INFO")
-
-def import_signals_and_indicators(strategies_folder="strategies"):
-    strategies = {}
-    for root, dirs, files in os.walk(strategies_folder):
-        for file_name in files:
-            if file_name.endswith(".py"):
-                file_path = os.path.join(root, file_name)
-                name_without_extension = os.path.splitext(file_name)[0]
-                strategy_name = os.path.relpath(file_path, strategies_folder).replace(
-                    os.sep, '_').rsplit('.', 1)[0]
+    path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'config', 'api.json')
+    with open(path, 'r') as f: return json.load(f)
+
+def import_strategies(folder):
+    strats = {}
+    if not os.path.exists(folder): return strats
+    for root, _, files in os.walk(folder):
+        for f in files:
+            if f.endswith(".py"):
+                path = os.path.join(root, f)
+                name = os.path.relpath(path, folder).replace(os.sep, '_').rsplit('.', 1)[0]
                 try:
-                    spec = importlib.util.spec_from_file_location(
-                        name_without_extension, file_path)
-                    module = importlib.util.module_from_spec(spec)
-                    spec.loader.exec_module(module)
-                    buy_signal_func = getattr(module, 'buy_signal', None)
-                    sell_signal_func = getattr(module, 'sell_signal', None)
-                    indicators_class = getattr(module, 'Indicators', None)
-                    if buy_signal_func and sell_signal_func and indicators_class:
-                        strategies[strategy_name] = {
-                            "buy_signal": buy_signal_func,
-                            "sell_signal": sell_signal_func,
-                            "Indicators": indicators_class
-                        }
-                        logger.debug(f"Imported strategy '{strategy_name}' from {file_path}")
-                    else:
-                        logger.warning(f"Strategy '{strategy_name}' is missing required functions/classes.")
-                except Exception as e:
-                    logger.error(f"Failed to import module '{strategy_name}' from {file_path}: {e}")
-    logger.info(f'Strategies: {strategies}')
-    return strategies
+                    spec = importlib.util.spec_from_file_location(os.path.splitext(f)[0], path)
+                    mod = importlib.util.module_from_spec(spec)
+                    spec.loader.exec_module(mod)
+                    if all(hasattr(mod, a) for a in ['buy_signal', 'sell_signal', 'Indicators']):
+                        strats[name] = {"buy_signal": mod.buy_signal, "sell_signal": mod.sell_signal, "Indicators": mod.Indicators}
+                except Exception: pass
+    return strats
 
 def create_app():
     app = Flask(__name__)
-
-    # Setup cache
-    cache_config = config['cache']
     app.config['CACHE_TYPE'] = os.getenv('QTSBE_CACHE_TYPE')
     app.config['CACHE_DEFAULT_TIMEOUT'] = int(os.getenv('QTSBE_CACHE_DEFAULT_TIMEOUT'))
-    cache = Cache(app)
+    Cache(app)
 
-    cors_origins = os.getenv('QTSBE_CORS_ORIGINS').split(',') if os.getenv('QTSBE_CORS_ORIGINS') != '*' else ["*"]
-    cors_methods = os.getenv('QTSBE_CORS_METHODS').split(',')
-
-    CORS(app, resources={
-        r"/QTSBE/*": {
-            "origins": cors_origins,
-            "methods": cors_methods,
-            "allow_headers": ["Content-Type", "Authorization", "Accept"],
-            "expose_headers": ["Content-Range", "X-Content-Range"]
-        }
-    })
-
-    @app.after_request
-    def after_request(response):
-        cors_origin = os.getenv('QTSBE_CORS_ORIGINS')
-        response.headers.add('Access-Control-Allow-Origin', cors_origin if cors_origin != '*' else '*')
-        response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
-        response.headers.add('Access-Control-Allow-Methods', os.getenv('QTSBE_CORS_METHODS'))
-        return response
-
-    SWAGGER_URL = '/docs'
-    API_URL = '/static/swagger.yaml'
+    origins = os.getenv('QTSBE_CORS_ORIGINS', '*').split(',')
+    CORS(app, resources={r"/QTSBE/*": {"origins": origins}})
 
-    swaggerui_blueprint = get_swaggerui_blueprint(
-        SWAGGER_URL,
-        API_URL,
-        config={
-            'app_name': "QTSBE API Documentation"
-        }
-    )
-
-    app.register_blueprint(swaggerui_blueprint, url_prefix=SWAGGER_URL)
-
-    os.makedirs('api/static', exist_ok=True)
-
-    if os.path.exists('swagger.yaml'):
-        shutil.copy('swagger.yaml', 'api/static/swagger.yaml')
-
-    register_analyse_routes(app, strategies, analyse)
+    strats = import_strategies("api/strategies")
+    register_analyse_routes(app, strats, analyse)
     register_analyse_custom_routes(app, analyse)
-    register_scan_routes(app, strategies, analyse)
-    register_strategy_routes(app, strategies)
+    register_strategy_routes(app, strats)
     register_get_tokens_routes(app)
     register_get_tokens_stats_routes(app)
     register_health_routes(app)
-
     return app
 
 if __name__ == '__main__':
-    reload_loguru_config()
-    strategies = import_signals_and_indicators(strategies_folder)
-    logger.debug("List of all strategies: {}", list(strategies.keys()))
-    logger.warning("API has been restarted.")
     app = create_app()
-
-    port = int(os.getenv('QTSBE_PORT'))
-    host = os.getenv('QTSBE_HOST')
-    debug = os.getenv('QTSBE_DEBUG').lower() == 'true'
-
-    app.run(
-        host=host,
-        port=port,
-        debug=debug
-    )
+    app.run(host=os.getenv('QTSBE_HOST'), port=int(os.getenv('QTSBE_PORT')), debug=os.getenv('QTSBE_DEBUG', 'false').lower() == 'true')