Add initial version of flexmeasures-openadr3 plugin

.github/issue-branch.yml

@@ -0,0 +1,2 @@
+openDraftPR: true
+autoCloseIssue: true

.github/workflows/lint-and-test.yml

@@ -0,0 +1,49 @@
+name: lint-and-test
+
+on: push
+
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    name: "Check (on Python3.11)"
+    steps:
+      - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb
+        with:
+          python-version: 3.12
+      - name: Install uv
+        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
+        with:
+          version: "0.9.22"
+          enable-cache: true
+      - uses: actions/checkout@v3
+      - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507
+
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: Install uv and set Python version
+        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
+        with:
+          version: "0.9.22"
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+
+      - id: install_dependencies
+        run: uv sync --locked --all-extras --dev
+
+      - id: run_tests
+        run: uv run pytest --cov . --cov-report xml
+
+      - name: Upload test coverage xml as artifact
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: coverage-results-${{ matrix.python-version }}
+          path: coverage.xml

.gitignore

@@ -0,0 +1,91 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Sphinx documentation
+docs/_build/
+
+# IPython
+profile_default/
+ipython_config.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+
+# custom project files
+
+.vscode
+*.pickle
+
+.ipynb_checkpoints/
+notebooks/.ipynb_checkpoints/
+
+flexmeasures.log
+
+.DS_Store
+
+
+docker-compose-data
+ci
+flexmeasures-instance
+
+.cursor

.pre-commit-config.yaml

@@ -0,0 +1,47 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+    -   id: check-added-large-files
+    -   id: check-json
+    -   id: check-symlinks
+    -   id: check-toml
+    -   id: check-xml
+    -   id: check-yaml
+    -   id: detect-private-key
+    -   id: end-of-file-fixer
+    -   id: fix-byte-order-marker
+    -   id: trailing-whitespace
+    -   id: mixed-line-ending
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.9
+    hooks:
+        - id: ruff
+        - id: ruff-format
+- repo: local
+  hooks:
+    - id: mypy
+      name: mypy
+      description: a hook the runs mypy with uv
+      entry: uv run mypy
+      language: system
+      types: [python]
+      stages: [pre-commit]
+- repo: local
+  hooks:
+    - id: ruff-check
+      name: ruff-check
+      description: a hook the runs ruff with uv
+      entry: uv run ruff check
+      language: system
+      types: [python]
+      stages: [pre-commit]
+- repo: local
+  hooks:
+    - id: ruff-check
+      name: ruff-check
+      description: a hook the runs ruff with uv
+      entry: uv run ruff format
+      language: system
+      types: [python]
+      stages: [pre-commit]

.python-version

@@ -0,0 +1 @@
+3.12

Dockerfile

@@ -0,0 +1,8 @@
+FROM lfenergy/flexmeasures:v0.33.0
+
+COPY --from=ghcr.io/astral-sh/uv:0.10-python3.12-trixie-slim /usr/local/bin/uv /usr/local/bin/uv
+
+COPY . /app/plugins/flexmeasures-openadr3
+
+# Install the plugin dependencies INTO the existing uv environment
+RUN uv pip install --python /app/.venv/bin/python /app/plugins/flexmeasures-openadr3

README.md

@@ -0,0 +1,131 @@
+# flexmeasures-openadr3
+
+FlexMeasures plugin that connects to an **OpenADR 3 Virtual Top Node (VTN)** as a **Virtual End Node (VEN)** and ingests demand-response capacity signals into FlexMeasures time-series sensors.
+
+Requires FlexMeasures **≥ v0.33.0** and Python **≥ 3.12**.
+
+## What is OpenADR?
+
+[OpenADR](https://www.openadr.org/) (Open Automated Demand Response) is an industry standard for exchanging demand-response signals between a grid operator or aggregator (**VTN — Virtual Top Node**) and flexible assets or sites (**VEN — Virtual End Node**). Events describe when and how much capacity is available or limited, typically as time-bounded intervals with numeric payloads.
+
+**OpenADR 3.x** modernises the protocol around REST APIs and JSON payloads (compared with the XML-centric OpenADR 2.0b profile). This plugin implements a **VEN client** using the [`openadr3-client`](https://pypi.org/project/openadr3-client/) Open-Source Python library and targets **OpenADR 3.1** VTNs.
+
+### Related: OpenLEADR
+
+[OpenLEADR](https://lfenergy.org/projects/openleadr/) is another [LF Energy](https://lfenergy.org/) project. It provides open-source OpenADR implementations, including a **Rust-based OpenADR 3.1 VTN** ([openleadr-rs](https://github.com/OpenLEADR/openleadr-rs)). OpenLEADR plays the **VTN** role; this FlexMeasures plugin plays the **VEN** role and can consume events published by VTNs such as OpenLEADR.
+
+## Overview
+
+The plugin adds **configuration UI only** — it does **not** extend the FlexMeasures CLI or REST API. All setup happens through the web dashboard.
+
+| Capability | Description |
+|------------|-------------|
+| **VEN clients** | Register connection details to a VTN: base URL, OAuth client credentials (stored encrypted), and scopes. Each VEN client is represented as a FlexMeasures **generic asset** of type `OpenADR VEN`. |
+| **Polling schedules** | Per VEN client, define one or more polling schedules. A polling schedule will run once daily at a configured time in UTC and will fetch demand response signals from the OpenADR VTN configured in the VEN client. |
+| **Sensor mapping** | For each polling schedule, the plugin creates or links FlexMeasures **sensors** on the VEN asset and writes incoming event values as **beliefs** (15-minute resolution, kW). |
+
+### Supported signals (current scope)
+
+Only two OpenADR event payload types are supported as of the current release of this plugin:
+
+| Payload type | FlexMeasures sensor | Unit |
+|--------------|----------------------|------|
+| `IMPORT_CAPACITY_LIMIT` | `import-capacity-limit` | kW |
+| `EXPORT_CAPACITY_LIMIT` | `export-capacity-limit` | kW |
+
+Additional OpenADR payload types may be supported in future releases. The ingestion pipeline is limited to these capacity-limit signals for now.
+
+### Dashboard
+
+After installation, open the plugin from the FlexMeasures main menu:
+
+- **Menu label:** OpenADR 3 Configuration
+- **URL:** `/flexmeasures-openadr3/dashboard`
+
+From the dashboard you can create and manage VEN clients, polling schedules, and inspect linked sensors.
+
+## How it works
+
+```mermaid
+flowchart LR
+    subgraph config [Configuration UI]
+        UI[Dashboard]
+        VC[VEN client]
+        SC[polling schedule]
+        UI --> VC
+        VC --> SC
+    end
+
+    subgraph fm [FlexMeasures]
+        GA[Generic asset: OpenADR VEN]
+        IS[import-capacity-limit sensor]
+        ES[export-capacity-limit sensor]
+        SC --> GA
+        SC --> IS
+        SC --> ES
+    end
+
+    subgraph runtime [Scheduled fetch]
+        RQ[RQ job at UTC trigger time]
+        VEN[openadr3-client VEN HTTP client]
+        VTN[(OpenADR 3 VTN)]
+        RQ --> VEN
+        VEN -->|GET events by targets| VTN
+        VEN -->|TimedBelief| IS
+        VEN -->|TimedBelief| ES
+    end
+
+    SC -.->|schedules daily| RQ
+```
+
+1. An administrator configures a **VEN client** (VTN URL, OAuth) and one or more **polling schedulles** with optional OpenADR targets to filter on and a daily UTC poll time.
+2. When signals are enabled, the plugin provisions **import** and/or **export capacity limit** sensors on the VEN asset.
+3. An **RQ job** runs at the configured time each day (and reschedules itself after each run). The job calls the VTN, filters events for the supported payload types, validates that active intervals do not overlap, and stores values on the mapped sensors with data source `OpenADR 3 VTN`.
+4. These sensors can then be coupled to assets within FlexMeasures. The most obvious mapping for the currently supported signals would be for an OpenADR sensor to map to the `site-consumption-capacity` (import capacity limit) and `site-production-capacity` (export capacity limit) on an asset of the `building` type. But other variations may exist.
+
+## Installation
+
+1. Add the plugin to your FlexMeasures config via `FLEXMEASURES_PLUGINS` (a list). Use either:
+   - The filesystem path to this repository, e.g. `/path/to/flexmeasures-plugin-oadr3`, or
+   - The installed package name: `flexmeasures_openadr3` (after `pip install` / `uv pip install`).
+
+2. Optionally set `OPENADR_SECRETS_ENCRYPTION_KEY`, if this environment variable is set, this key is used to encrypt secrets (such as oauth credentials) at rest in asset attributes. Optional, if not set, the global flexmeasures `SECRET_KEY` environment variable is used to encrypt secrets..
+
+3. Ensure an RQ worker is running on the queue used for fetch jobs (currently the `ingestion` queue), this is the default behaviour when using the FlexMeasures CLI run-worker when running FlexMeasures v0.33.0 or higher.
+
+4. Restart FlexMeasures and open **OpenADR 3 Configuration** in the menu.
+
+## Development
+
+We use pre-commit to keep code quality up.
+
+Install necessary tools with:
+
+```bash
+pip install pre-commit black flake8 mypy
+pre-commit install
+```
+
+or:
+
+```bash
+make install-for-dev
+```
+
+Try it:
+
+```bash
+pre-commit run --all-files --show-diff-on-failure
+```
+
+To inspect RQ jobs, you can use the rq-dashboard package which can be optionally installed as a dev dependency:
+
+```bash
+uv run rq-dashboard --redis-url redis://{USERNAME}:{PASSWORD}@{HOST}:{PORT}/{QUEUE_NAME}
+```
+
+In the case of the redis queue configured in the `docker-compose.yml`, this would result in the following command:
+
+```bash
+uv run rq-dashboard --redis-url redis://:fm-redis-pass@localhost:6379/0
+```

conftest.py

@@ -0,0 +1,58 @@
+"""Top-level pytest configuration (pytest_plugins must live here)."""
+
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING
+
+import pytest
+from flask import Flask
+from flexmeasures.app import create as create_flexmeasures_app
+
+from tests.test_container.flexmeasures_test_postgres_container import (
+    FlexMeasuresTestPostgres,
+    FlexMeasuresTestPostgresContainer,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+pytest_plugins = [
+    "flexmeasures.conftest",
+]
+
+
+@pytest.fixture(scope="session")
+def integration_test_flexmeasures_postgres() -> Iterator[FlexMeasuresTestPostgres]:
+    """Session-scoped PostgreSQL testcontainer for FlexMeasures during integration tests."""
+    with FlexMeasuresTestPostgresContainer() as postgres_container:
+        yield postgres_container.get_connection_info()
+
+
+@pytest.fixture(scope="session")
+def app(
+    integration_test_flexmeasures_postgres: FlexMeasuresTestPostgres,
+) -> Iterator[Flask]:
+    """FlexMeasures app with this plugin loaded (overrides flexmeasures.conftest.app)."""
+    # FlexMeasures reads this env var during create_app() (see read_config in testing mode).
+    # Setting SQLALCHEMY_DATABASE_URI on the app config afterwards is too late: db.init_app()
+    # already created a cached engine bound to TestingConfig's localhost default.
+    os.environ["SQLALCHEMY_TEST_DATABASE_URI"] = integration_test_flexmeasures_postgres.database_uri
+
+    test_app = create_flexmeasures_app(
+        env="testing",
+        plugins=["flexmeasures_openadr3"],
+    )
+    test_app.config["OPENADR_SECRETS_ENCRYPTION_KEY"] = "test-openadr-secrets-key"
+    test_app.config["ALLOW_INSECURE_HTTP_VTN"] = "true"
+    test_app.config["SERVER_NAME"] = "localhost"
+    test_app.config["PREFERRED_URL_SCHEME"] = "http"
+
+    ctx = test_app.app_context()
+    ctx.push()
+
+    yield test_app
+
+    ctx.pop()
+
+    os.environ.pop("SQLALCHEMY_TEST_DATABASE_URI", None)