diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000..448c862f --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,3 @@ +[env] +DUCKDB_DOWNLOAD_LIB = { value = "1", force = false } +DUCKDB_CMAKE_BUILD_TYPE = { value = "Release", force = false } diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..3deaaf9f --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,12 @@ +{ + "name": "sensapp", + "image": "mcr.microsoft.com/devcontainers/universal:2-linux", + "remoteEnv": { + "DUCKDB_DOWNLOAD_LIB": "1", + "DUCKDB_CMAKE_BUILD_TYPE": "Release" + }, + "containerEnv": { + "DUCKDB_DOWNLOAD_LIB": "1", + "DUCKDB_CMAKE_BUILD_TYPE": "Release" + } +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..509d0190 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.git +.github +current_tasks +done +ideas +target +tmp +reference +docker +compose.test-services.yml +*.db +*.duckdb +*.sqlite +*.log diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c0166d0..421d564f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,10 +7,13 @@ on: pull_request: release: types: [published] + workflow_dispatch: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 + DUCKDB_DOWNLOAD_LIB: "1" + DUCKDB_CMAKE_BUILD_TYPE: Release REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} @@ -29,7 +32,7 @@ jobs: components: rustfmt, clippy - name: Cache cargo-make - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.cargo/bin/cargo-make key: cargo-make-${{ runner.os }} @@ -43,9 +46,65 @@ jobs: - name: Format check run: cargo make fmt-check + security-audit: + name: Security Audit + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup Rust toolchain + uses: moonrepo/setup-rust@v1 + with: + channel: stable + + - name: Cache cargo-make + uses: actions/cache@v5 + with: + path: ~/.cargo/bin/cargo-make + key: cargo-make-${{ runner.os }} + + - name: Install cargo-make + run: | + if ! command -v cargo-make &> /dev/null; then + cargo install cargo-make + fi + - name: Security audit run: cargo make security-audit - continue-on-error: true + + frontend: + name: Frontend Quality + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: frontend/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Lint + run: npm run lint + + - name: Type check + run: npm run typecheck + + - name: Test + run: npm test + + - name: Build + run: npm run build test-matrix: name: Test Suite @@ -57,12 +116,20 @@ jobs: os: [ubuntu-latest] rust: [stable] #rust: [stable, beta] - feature: [sqlite, postgres] + feature: [sqlite, postgres, clickhouse, duckdb, timescaledb, rrdcached] include: - feature: sqlite test-env: "TEST_DATABASE_URL_SQLITE=sqlite://test.db" - feature: postgres - test-env: "TEST_DATABASE_URL_POSTGRES=postgres://postgres:postgres@localhost:5432/sensapp" + test-env: "TEST_DATABASE_URL_POSTGRES=postgres://postgres:postgres@localhost:5432/sensapp-test" + - feature: clickhouse + test-env: "TEST_DATABASE_URL_CLICKHOUSE=clickhouse://default:password@localhost:8123/sensapp_test" + - feature: duckdb + test-env: "TEST_DATABASE_URL_DUCKDB=duckdb://test.duckdb" + - feature: timescaledb + test-env: "TEST_DATABASE_URL_TIMESCALEDB=timescaledb://postgres:postgres@localhost:5433/sensapp-test" + - feature: rrdcached + test-env: "TEST_DATABASE_URL_RRDCACHED=rrdcached://127.0.0.1:42217?preset=hoarder" services: postgres: @@ -70,7 +137,7 @@ jobs: env: POSTGRES_PASSWORD: postgres POSTGRES_USER: postgres - POSTGRES_DB: sensapp + POSTGRES_DB: sensapp-test options: >- --health-cmd pg_isready --health-interval 10s @@ -78,7 +145,33 @@ jobs: --health-retries 5 ports: - 5432:5432 - + clickhouse: + image: clickhouse/clickhouse-server:24.8 + env: + CLICKHOUSE_DB: sensapp_test + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: password + options: >- + --health-cmd "clickhouse-client --password password --query 'SELECT 1'" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + ports: + - 8123:8123 + - 9000:9000 + timescaledb: + image: timescale/timescaledb:2.17.2-pg16 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + POSTGRES_DB: sensapp-test + options: >- + --health-cmd "pg_isready -U postgres -d sensapp-test" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + ports: + - 5433:5432 steps: - name: Checkout repository uses: actions/checkout@v6 @@ -90,7 +183,7 @@ jobs: components: clippy - name: Cache cargo-make - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.cargo/bin/cargo-make key: cargo-make-${{ runner.os }} @@ -104,6 +197,23 @@ jobs: - name: Install sqlx-cli run: cargo install sqlx-cli --no-default-features --features postgres,sqlite + - name: Start RRDCached test service + if: matrix.feature == 'rrdcached' + run: | + docker build -f docker/rrdcached/Dockerfile -t sensapp-rrdcached-test . + docker run -d --name sensapp-rrdcached -p 42217:42217 sensapp-rrdcached-test + + for attempt in $(seq 1 30); do + if nc -z 127.0.0.1 42217; then + exit 0 + fi + + sleep 1 + done + + docker logs sensapp-rrdcached + exit 1 + - name: Set up test environment run: | ${{ matrix.test-env }} @@ -117,9 +227,80 @@ jobs: if: matrix.feature == 'sqlite' run: cargo make migrate-sqlite + - name: Run migrations (TimescaleDB) + if: matrix.feature == 'timescaledb' + run: cargo make migrate-timescaledb + - name: Run tests + if: matrix.feature != 'rrdcached' run: cargo make check-${{ matrix.feature }} + - name: Run RRDCached checks + if: matrix.feature == 'rrdcached' + run: | + export TEST_DATABASE_URL='rrdcached://127.0.0.1:42217?preset=hoarder' + cargo build --features rrdcached --no-default-features + cargo test --features rrdcached --no-default-features storage::rrdcached --lib + cargo test --features rrdcached --no-default-features http::influxdb::tests::test_publish_influxdb -- --exact + cargo test --features rrdcached --no-default-features http::influxdb::tests::test_publish_influxdb_with_numeric_enabled -- --exact + cargo test --features rrdcached --no-default-features --test rrdcached_integration + + helm-validation: + name: Helm Validation + runs-on: ubuntu-latest + needs: quality-checks + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Helm + uses: azure/setup-helm@v4 + + - name: Lint chart + run: helm lint charts/sensapp + + - name: Render chart with SQLite defaults + run: helm template sensapp charts/sensapp + + - name: Render chart with PostgreSQL override + run: >- + helm template sensapp charts/sensapp + --set storage.connectionString=postgres://postgres:postgres@postgres:5432/sensapp + --set persistence.enabled=false + + docker-smoke: + name: Docker Smoke Build (${{ matrix.name }}) + runs-on: ubuntu-latest + needs: quality-checks + strategy: + fail-fast: false + matrix: + include: + - name: runtime + features: postgres,sqlite,timescaledb,duckdb,clickhouse,rrdcached + - name: runtime-bigquery + features: postgres,sqlite,timescaledb,duckdb,clickhouse,rrdcached,bigquery + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build image variant + uses: docker/build-push-action@v6 + with: + context: . + push: false + platforms: linux/amd64 + build-args: | + FEATURES=${{ matrix.features }} + NO_DEFAULT_FEATURES=true + cache-from: type=gha + cache-to: type=gha,mode=max + # Disabled temporarily due to timeout issues # coverage: # name: Test Coverage @@ -154,7 +335,7 @@ jobs: # uses: taiki-e/install-action@cargo-llvm-cov # - name: Cache cargo-make - # uses: actions/cache@v4 + # uses: actions/cache@v5 # with: # path: ~/.cargo/bin/cargo-make # key: cargo-make-${{ runner.os }} @@ -183,7 +364,7 @@ jobs: docker: name: Docker Build & Push runs-on: ubuntu-latest - needs: test-matrix + needs: [test-matrix, helm-validation, docker-smoke] if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop' || github.event_name == 'release') permissions: contents: read @@ -226,10 +407,36 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max build-args: | + FEATURES=postgres,sqlite,timescaledb,duckdb,clickhouse,rrdcached + NO_DEFAULT_FEATURES=true BUILDTIME=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }} VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }} REVISION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }} + helm-package: + name: Helm Package + runs-on: ubuntu-latest + needs: helm-validation + if: github.event_name != 'pull_request' + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Helm + uses: azure/setup-helm@v4 + + - name: Package chart + run: | + mkdir -p dist + helm package charts/sensapp --destination dist + + - name: Upload chart artifact + uses: actions/upload-artifact@v4 + with: + name: sensapp-helm-chart + path: dist/*.tgz + release: name: Release runs-on: ubuntu-latest @@ -249,7 +456,7 @@ jobs: channel: stable - name: Cache cargo-make - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.cargo/bin/cargo-make key: cargo-make-${{ runner.os }} diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index fe71e52d..719b2673 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v6 - uses: pre-commit/action@v3.0.0 env: SKIP: no-commit-to-branch diff --git a/.github/workflows/python-sdk-check.yaml b/.github/workflows/python-sdk-check.yaml new file mode 100644 index 00000000..e8136589 --- /dev/null +++ b/.github/workflows/python-sdk-check.yaml @@ -0,0 +1,94 @@ +name: Python SDK Check + +on: + push: + branches: [main, develop] + paths: + - "python/sensapp/**" + - ".github/workflows/python-sdk-check.yaml" + - "docs/PYTHON_SDK.md" + pull_request: + paths: + - "python/sensapp/**" + - ".github/workflows/python-sdk-check.yaml" + - "docs/PYTHON_SDK.md" + +jobs: + python-sdk-unit: + name: Python SDK Unit (${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13"] + defaults: + run: + working-directory: python/sensapp + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Install SDK dependencies + run: uv pip install --system -e '.[dev]' + + - name: Run Ruff check + if: matrix.python-version == '3.12' + run: uv run ruff check . + + - name: Run Ruff format check + if: matrix.python-version == '3.12' + run: uv run ruff format --check . + + - name: Run unit tests + run: python -m pytest -m 'not integration' + + - name: Build distributions + if: matrix.python-version == '3.12' + run: uv build + + - name: Smoke test wheel install + if: matrix.python-version == '3.12' + run: | + python -m venv /tmp/sensapp-wheel + /tmp/sensapp-wheel/bin/python -m pip install dist/*.whl + /tmp/sensapp-wheel/bin/python -c "import sensapp; print(sensapp.__all__[0])" + + python-sdk-integration: + name: Python SDK Integration + runs-on: ubuntu-latest + needs: python-sdk-unit + defaults: + run: + working-directory: python/sensapp + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Set up Rust toolchain + uses: moonrepo/setup-rust@v1 + with: + channel: stable + + - name: Install SDK dependencies + run: uv pip install --system -e '.[dev]' + + - name: Run live integration tests + run: python -m pytest -m integration diff --git a/.gitignore b/.gitignore index d3ae7a1d..88a058e0 100644 --- a/.gitignore +++ b/.gitignore @@ -22,9 +22,22 @@ coverage/ pki -test.db* +*.db +*.db-shm +*.db-wal +*.duckdb # Claude .claude/settings.local.json reference/ +integrations/ + +# Python +__pycache__/ +.venv/ +dist/ +python/demo-dashboard/ + +bin/ +target diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0776c84a..8dcb8257 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,14 +7,15 @@ repos: - id: mixed-line-ending - id: trailing-whitespace - id: check-json + exclude: ^frontend/tsconfig(\.[^.]+)?\.json$ - id: check-yaml + exclude: ^charts/sensapp/templates/.*\.ya?ml$ - id: check-xml - id: check-ast - - id: end-of-file-fixer - id: check-merge-conflict - id: no-commit-to-branch # no commits to main - repo: https://github.com/gitleaks/gitleaks - rev: v8.30.0 + rev: v8.30.1 hooks: - id: gitleaks - repo: https://github.com/jorisroovers/gitlint diff --git a/.vscode/settings.json b/.vscode/settings.json index 3a277a43..24808ac3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,10 +1,11 @@ { + "peacock.color": "fcd21c", "workbench.colorCustomizations": { - "commandCenter.border": "#15202b99", - "titleBar.activeBackground": "#fcd21c", - "titleBar.activeForeground": "#15202b", - "titleBar.inactiveBackground": "#fcd21c99", - "titleBar.inactiveForeground": "#15202b99" + "commandCenter.border": "#e7e7e799", + "titleBar.activeBackground": "#04052d", + "titleBar.activeForeground": "#e7e7e7", + "titleBar.inactiveBackground": "#04052d99", + "titleBar.inactiveForeground": "#e7e7e799" }, - "peacock.color": "fcd21c" + "peacock.remoteColor": "#04052d" } diff --git a/AGENTS.md b/AGENTS.md index 50051853..fc941144 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,3 +18,18 @@ Moreover, you can run the linters on the tests with `cargo clippy --tests`. - We follow a KISS (Keep It Simple, Stupid) approach. Avoid unnecessary abstractions and complexity. - During development, we focus on PostGreSQL as a main storage backend. Thus, you must only write tests in a generic way and not cut corners. - unit tests and integrations tests are very helpful and appreciated. Consider doing them even when not actively requested. Testing is important. + +## Task tracking + +We use a folder-based task tracking system to maintain visibility into ongoing work, future ideas, and completed tasks: + +- **`current_tasks/`** — Active tasks currently being worked on. Each task should be a separate markdown file describing the goal, context, and progress. When starting a major task, create a file here. +- **`ideas/`** — Ideas for future work, improvements, or features. Add a markdown file for each idea, even if it's rough or speculative. +- **`done/`** — Completed tasks. When a task from `current_tasks/` is finished, move it here. This provides a historical record of what has been accomplished. + +**Rules:** +- Always check `current_tasks/` before starting work to understand what's in progress. +- When completing a major task, move its file from `current_tasks/` to `done/`. +- New ideas should go in `ideas/`, not in `current_tasks/` unless actively being worked on. +- Keep file names descriptive (e.g., `mqtt-ingestor-integration.md`, `rrdcached-fetch-support.md`). +- Maintain these folders — they are the project's source of truth for work status. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b2199627..3ad429ea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,3 +73,27 @@ Examples: fix: 🐛 prevent infinite loop when it rains docs(architecture): 📝 correct spelling of banana ``` + +## Useful Commands + +```bash +# Build +cargo build + +# Test +cargo test +cargo make test-all # all storage backends +cargo make test-local-matrix # all local backends via Docker Compose (no BigQuery) + +# Lint (format + clippy) +cargo make lint +cargo make lint-all # all storage backends + +# Full validation +cargo make check-all # working features (postgres + sqlite) +cargo make check-all-storage # all storage backends +cargo make check-local-matrix # all local backends via Docker Compose (no BigQuery) + +# Setup (runs migrations) +cargo make setup-dev +``` diff --git a/Cargo.lock b/Cargo.lock index 661a75dc..1816a472 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,9 +21,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.11.2" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7926860314cbe2fb5d1f13731e387ab43bd32bca224e82e6e2db85de0a3dba49" +checksum = "f860ee6746d0c5b682147b2f7f8ef036d4f92fe518251a3a35ffa3650eafdf0e" dependencies = [ "actix-codec", "actix-rt", @@ -34,7 +34,7 @@ dependencies = [ "bytestring", "derive_more", "encoding_rs", - "foldhash", + "foldhash 0.1.5", "futures-core", "http 0.2.12", "httparse", @@ -52,9 +52,9 @@ dependencies = [ [[package]] name = "actix-router" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13d324164c51f63867b57e73ba5936ea151b8a41a1d23d1031eeb9f70d0236f8" +checksum = "14f8c75c51892f18d9c46150c5ac7beb81c95f78c8b83a634d49f4ca32551fe7" dependencies = [ "bytestring", "cfg-if", @@ -113,9 +113,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.12.1" +version = "4.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1654a77ba142e37f049637a3e5685f864514af11fcbc51cb51eb6596afe5b8d6" +checksum = "ff87453bc3b56e9b2b23c1cc0b1be8797184accf51d2abe0f8a33ec275d316bf" dependencies = [ "actix-codec", "actix-http", @@ -129,7 +129,7 @@ dependencies = [ "cfg-if", "derive_more", "encoding_rs", - "foldhash", + "foldhash 0.1.5", "futures-core", "futures-util", "impl-more", @@ -144,7 +144,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec 1.15.1", - "socket2 0.6.1", + "socket2 0.6.3", "time", "tracing", "url", @@ -171,7 +171,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "once_cell", "version_check", ] @@ -231,9 +231,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "approx" @@ -244,6 +244,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -258,97 +267,49 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "56.2.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" +checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" dependencies = [ - "arrow-arith 56.2.0", - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-cast 56.2.0", - "arrow-data 56.2.0", - "arrow-ord 56.2.0", - "arrow-row 56.2.0", - "arrow-schema 56.2.0", - "arrow-select 56.2.0", - "arrow-string 56.2.0", -] - -[[package]] -name = "arrow" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb372a7cbcac02a35d3fb7b3fc1f969ec078e871f9bb899bf00a2e1809bec8a3" -dependencies = [ - "arrow-arith 57.1.0", - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-cast 57.1.0", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", "arrow-csv", - "arrow-data 57.1.0", + "arrow-data", "arrow-ipc", "arrow-json", - "arrow-ord 57.1.0", - "arrow-row 57.1.0", - "arrow-schema 57.1.0", - "arrow-select 57.1.0", - "arrow-string 57.1.0", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", ] [[package]] name = "arrow-arith" -version = "56.2.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" +checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "chrono", - "num", -] - -[[package]] -name = "arrow-arith" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f377dcd19e440174596d83deb49cd724886d91060c07fec4f67014ef9d54049" -dependencies = [ - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-data 57.1.0", - "arrow-schema 57.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "num-traits", ] [[package]] name = "arrow-array" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" -dependencies = [ - "ahash 0.8.12", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "chrono", - "half", - "hashbrown 0.16.1", - "num", -] - -[[package]] -name = "arrow-array" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eaff85a44e9fa914660fb0d0bb00b79c4a3d888b5334adb3ea4330c84f002" +checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" dependencies = [ "ahash 0.8.12", - "arrow-buffer 57.1.0", - "arrow-data 57.1.0", - "arrow-schema 57.1.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "half", "hashbrown 0.16.1", @@ -359,20 +320,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "56.2.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" -dependencies = [ - "bytes", - "half", - "num", -] - -[[package]] -name = "arrow-buffer" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2819d893750cb3380ab31ebdc8c68874dd4429f90fd09180f3c93538bd21626" +checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" dependencies = [ "bytes", "half", @@ -382,55 +332,35 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "56.2.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" +checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "arrow-select 56.2.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", "atoi", - "base64 0.22.1", + "base64", "chrono", "comfy-table", "half", "lexical-core", - "num", - "ryu", -] - -[[package]] -name = "arrow-cast" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d131abb183f80c450d4591dc784f8d7750c50c6e2bc3fcaad148afc8361271" -dependencies = [ - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-data 57.1.0", - "arrow-ord 57.1.0", - "arrow-schema 57.1.0", - "arrow-select 57.1.0", - "atoi", - "base64 0.22.1", - "chrono", - "half", - "lexical-core", "num-traits", "ryu", ] [[package]] name = "arrow-csv" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2275877a0e5e7e7c76954669366c2aa1a829e340ab1f612e647507860906fb6b" +checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe" dependencies = [ - "arrow-array 57.1.0", - "arrow-cast 57.1.0", - "arrow-schema 57.1.0", + "arrow-array", + "arrow-cast", + "arrow-schema", "chrono", "csv", "csv-core", @@ -439,24 +369,12 @@ dependencies = [ [[package]] name = "arrow-data" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" -dependencies = [ - "arrow-buffer 56.2.0", - "arrow-schema 56.2.0", - "half", - "num", -] - -[[package]] -name = "arrow-data" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05738f3d42cb922b9096f7786f606fcb8669260c2640df8490533bb2fa38c9d3" +checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" dependencies = [ - "arrow-buffer 57.1.0", - "arrow-schema 57.1.0", + "arrow-buffer", + "arrow-schema", "half", "num-integer", "num-traits", @@ -464,29 +382,31 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d09446e8076c4b3f235603d9ea7c5494e73d441b01cd61fb33d7254c11964b3" +checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" dependencies = [ - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-data 57.1.0", - "arrow-schema 57.1.0", - "arrow-select 57.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "flatbuffers", + "lz4_flex 0.13.0", + "zstd", ] [[package]] name = "arrow-json" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "371ffd66fa77f71d7628c63f209c9ca5341081051aa32f9c8020feb0def787c0" +checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8" dependencies = [ - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-cast 57.1.0", - "arrow-data 57.1.0", - "arrow-schema 57.1.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "chrono", "half", "indexmap", @@ -502,127 +422,64 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "56.2.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" +checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "arrow-select 56.2.0", -] - -[[package]] -name = "arrow-ord" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc94fc7adec5d1ba9e8cd1b1e8d6f72423b33fe978bf1f46d970fafab787521" -dependencies = [ - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-data 57.1.0", - "arrow-schema 57.1.0", - "arrow-select 57.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", ] [[package]] name = "arrow-row" -version = "56.2.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" +checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "half", -] - -[[package]] -name = "arrow-row" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "169676f317157dc079cc5def6354d16db63d8861d61046d2f3883268ced6f99f" -dependencies = [ - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-data 57.1.0", - "arrow-schema 57.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half", ] [[package]] name = "arrow-schema" -version = "56.2.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" +checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" dependencies = [ "bitflags", ] -[[package]] -name = "arrow-schema" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27609cd7dd45f006abae27995c2729ef6f4b9361cde1ddd019dc31a5aa017e0" - -[[package]] -name = "arrow-select" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" -dependencies = [ - "ahash 0.8.12", - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "num", -] - [[package]] name = "arrow-select" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae980d021879ea119dd6e2a13912d81e64abed372d53163e804dfe84639d8010" +checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" dependencies = [ "ahash 0.8.12", - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-data 57.1.0", - "arrow-schema 57.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num-traits", ] [[package]] name = "arrow-string" -version = "56.2.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" +checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "arrow-select 56.2.0", - "memchr", - "num", - "regex", - "regex-syntax", -] - -[[package]] -name = "arrow-string" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf35e8ef49dcf0c5f6d175edee6b8af7b45611805333129c541a8b89a0fc0534" -dependencies = [ - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-data 57.1.0", - "arrow-schema 57.1.0", - "arrow-select 57.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "memchr", "num-traits", "regex", @@ -631,13 +488,12 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.34" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e86f6d3dc9dc4352edeea6b8e499e13e3f5dc3b964d7ca5fd411415a3498473" +checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" dependencies = [ "compression-codecs", "compression-core", - "futures-core", "pin-project-lite", "tokio", ] @@ -661,7 +517,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -672,7 +528,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -698,9 +554,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-rs" -version = "1.15.1" +version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b5ce75405893cd713f9ab8e297d8e438f624dde7d706108285f7e17a25a180f" +checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" dependencies = [ "aws-lc-sys", "zeroize", @@ -708,9 +564,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.34.0" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "179c3777a8b5e70e90ea426114ffc565b2c1a9f82f6c4a0c5a34aa6ef5e781b6" +checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" dependencies = [ "cc", "cmake", @@ -720,9 +576,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b098575ebe77cb6d14fc7f32749631a6e44edbef6b796f89b020e99ba20d425" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" dependencies = [ "axum-core", "axum-macros", @@ -755,9 +611,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59446ce19cd142f8833f856eb31f3eb097812d1479ab224f54d72428ca21ea22" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", @@ -780,7 +636,7 @@ checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -799,10 +655,10 @@ dependencies = [ ] [[package]] -name = "base64" -version = "0.21.7" +name = "base16ct" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" [[package]] name = "base64" @@ -812,9 +668,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" -version = "1.8.0" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "big-decimal-byte-string-encoder" @@ -830,9 +686,9 @@ dependencies = [ [[package]] name = "bigdecimal" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "560f42649de9fa436b73517378a147ec21f6c997a546581df4b4b31677828934" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", @@ -843,9 +699,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" dependencies = [ "serde_core", ] @@ -873,15 +729,16 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.2" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" +checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", + "cpufeatures 0.3.0", ] [[package]] @@ -928,7 +785,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -963,9 +820,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.19.0" +version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" [[package]] name = "byte-unit" @@ -1009,9 +866,12 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +dependencies = [ + "serde", +] [[package]] name = "bytestring" @@ -1024,32 +884,33 @@ dependencies = [ [[package]] name = "cached" -version = "0.56.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "801927ee168e17809ab8901d9f01f700cd7d8d6a6527997fee44e4b0327a253c" +checksum = "53b6f5d101f0f6322c8646a45b7c581a673e476329040d97565815c2461dd0c4" dependencies = [ "ahash 0.8.12", "async-trait", "cached_proc_macro", "cached_proc_macro_types", "futures", - "hashbrown 0.15.5", + "hashbrown 0.16.1", "once_cell", - "thiserror 2.0.17", + "parking_lot", + "thiserror 2.0.18", "tokio", "web-time", ] [[package]] name = "cached_proc_macro" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9225bdcf4e4a9a4c08bf16607908eb2fbf746828d5e0b5e019726dbf6571f201" +checksum = "8ebcf9c75f17a17d55d11afc98e46167d4790a263f428891b8705ab2f793eca3" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1066,9 +927,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.48" +version = "1.2.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c481bdbf0ed3b892f6f806287d72acd515b352a4ec27a208489b8c1bc839633a" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" dependencies = [ "find-msvc-tools", "jobserver", @@ -1090,9 +951,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.42" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", "js-sys", @@ -1110,9 +971,9 @@ checksum = "93a719913643003b84bd13022b4b7e703c09342cd03b679c4641c7d2e50dc34d" [[package]] name = "clickhouse" -version = "0.14.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4916807143285e80eefcaf741a29a98bbd687e4581352631a74ab223e66558b" +checksum = "35330181022b5f972325aa41fd3465f636be6527499c383cacd3fb2e51c5a8bc" dependencies = [ "bnum", "bstr", @@ -1125,13 +986,15 @@ dependencies = [ "http-body-util", "hyper", "hyper-util", - "lz4_flex", + "lz4_flex 0.11.6", "polonius-the-crab", "quanta", "serde", - "thiserror 2.0.17", + "serde_json", + "thiserror 2.0.18", "time", "tokio", + "tracing", "url", "uuid", ] @@ -1145,50 +1008,52 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "clickhouse-types" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "358fbfd439fb0bed02a3e2ecc5131f6a9d039ba5639aed650cf0e845f6ebfc16" +checksum = "30a5efddc880ce9e2573bd867413d9056fa2bea0206af88dec21e72178b9dc74" dependencies = [ "bytes", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] name = "clru" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbd0f76e066e64fdc5631e3bb46381254deab9ef1158292f27c8c57e3bf3fe59" +checksum = "197fd99cb113a8d5d9b6376f3aa817f32c1078f2343b714fff7d2ca44fdf67d5" +dependencies = [ + "hashbrown 0.16.1", +] [[package]] name = "cmake" -version = "0.1.54" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" dependencies = [ "cc", ] [[package]] name = "comfy-table" -version = "7.1.2" +version = "7.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" dependencies = [ - "strum 0.26.3", - "strum_macros 0.26.4", + "unicode-segmentation", "unicode-width", ] [[package]] name = "compression-codecs" -version = "0.4.33" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302266479cb963552d11bd042013a58ef1adc56768016c8b82b4199488f2d4ad" +checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" dependencies = [ "brotli", "compression-core", @@ -1233,7 +1098,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1257,16 +1122,16 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "once_cell", "tiny-keccak", ] [[package]] name = "constant_time_eq" -version = "0.3.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" [[package]] name = "convert_case" @@ -1277,16 +1142,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation" version = "0.10.1" @@ -1321,6 +1176,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -1394,6 +1258,18 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -1439,6 +1315,33 @@ dependencies = [ "memchr", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "darling" version = "0.20.11" @@ -1460,7 +1363,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1471,7 +1374,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1515,34 +1418,45 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.5" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ "powerfmt", "serde_core", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "derive_more" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10b768e943bed7bf2cab53df09f4bc34bfd217cdb57d971e769874c9a6710618" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" dependencies = [ "derive_more-impl", ] [[package]] name = "derive_more-impl" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d286bfdaf75e988b4a78e013ecd79c581e06399ab53fbacd2d916c2f904f30b" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" dependencies = [ "convert_case", "proc-macro2", "quote", "rustc_version", - "syn 2.0.111", + "syn 2.0.117", "unicode-xid", ] @@ -1560,9 +1474,9 @@ dependencies = [ [[package]] name = "dispatch2" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89a09f22a6c6069a18470eb92d2298acf25463f14256d24778e1230d789a2aec" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ "bitflags", "objc2", @@ -1576,7 +1490,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1591,13 +1505,19 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + [[package]] name = "duckdb" -version = "1.4.2" +version = "1.10501.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46d5568337ee1f7ea8779e1d9aa2eafcdf156458713ce65afb246c5d2cf5850" +checksum = "f13bc6d6487032fc2825a62ef8b4924b2378a2eb3166e132e5f3141ae9dd633f" dependencies = [ - "arrow 56.2.0", + "arrow", "cast", "fallible-iterator", "fallible-streaming-iterator", @@ -1605,7 +1525,7 @@ dependencies = [ "libduckdb-sys", "num-integer", "rust_decimal", - "strum 0.27.2", + "strum", ] [[package]] @@ -1630,6 +1550,44 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.15.0" @@ -1639,6 +1597,27 @@ dependencies = [ "serde", ] +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest", + "ff", + "generic-array", + "group", + "hkdf", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encoding_rs" version = "0.8.35" @@ -1661,7 +1640,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1704,23 +1683,38 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "filetime" -version = "0.2.26" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" dependencies = [ "cfg-if", "libc", "libredox", - "windows-sys 0.60.2", ] [[package]] name = "find-msvc-tools" -version = "0.1.5" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "findshlibs" @@ -1742,9 +1736,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flatbuffers" -version = "25.9.23" +version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b6620799e7340ebd9968d2e0708eb82cf1971e9a16821e2091b6d6e475eed5" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ "bitflags", "rustc_version", @@ -1752,12 +1746,13 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.5" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1789,6 +1784,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1827,9 +1828,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" dependencies = [ "futures-channel", "futures-core", @@ -1842,9 +1843,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", "futures-sink", @@ -1852,15 +1853,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ "futures-core", "futures-task", @@ -1880,38 +1881,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-channel", "futures-core", @@ -1921,15 +1922,14 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] [[package]] name = "gcp-bigquery-client" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb75ad03f4fffd15675c6fed7c37715176d7fc3f681b91c055a9f0a5cd34973d" +checksum = "b776bd336303c2dcd66a872d3a9f5c2a4678162dc3c90617e89ea665e2ce79cc" dependencies = [ "async-stream", "async-trait", @@ -1943,10 +1943,10 @@ dependencies = [ "prost", "prost-build", "prost-types", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", "tokio", "tokio-stream", @@ -1966,6 +1966,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -2003,18 +2004,18 @@ dependencies = [ [[package]] name = "geographiclib-rs" -version = "0.2.5" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f611040a2bb37eaa29a78a128d1e92a378a03e0b6e66ae27398d42b1ba9a7841" +checksum = "c5a7f08910fd98737a6eda7568e7c5e645093e073328eeef49758cfe8b0489c7" dependencies = [ "libm", ] [[package]] name = "getrandom" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", "js-sys", @@ -2032,22 +2033,46 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + [[package]] name = "gimli" version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "h2" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" dependencies = [ "atomic-waker", "bytes", @@ -2100,7 +2125,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -2108,6 +2133,11 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "hashlink" @@ -2164,16 +2194,16 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hifitime" -version = "4.2.3" +version = "4.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae664976c51b75b30deb25aa778bff665169ca129104fb5f427e7154fc611ad5" +checksum = "22f1dfc1be6cd1c3a44704db0a8800aa53ebb07a9509535a0261cd387eedee7c" dependencies = [ "js-sys", "lexical-core", "num-traits", "serde", "serde_derive", - "snafu 0.8.9", + "snafu 0.9.0", "wasm-bindgen", "web-sys", "web-time", @@ -2292,11 +2322,10 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hybridmap" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b28a248730c6c65e6201640a3e37806454291985af7834a042e86ee2c187f71" +checksum = "3aa2822e947ac001ca7b6b14a11f56cff5272a21803c8f2c2e30463ff4d08cb1" dependencies = [ - "rand 0.8.5", "smallvec 2.0.0-alpha.12", ] @@ -2372,14 +2401,13 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.18" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", - "futures-core", "futures-util", "http 1.4.0", "http-body", @@ -2388,7 +2416,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.1", + "socket2 0.6.3", "tokio", "tower-service", "tracing", @@ -2411,9 +2439,9 @@ checksum = "9190f86706ca38ac8add223b2aed8b1330002b5cdbbce28fb58b10914d38fc27" [[package]] name = "i_overlay" -version = "4.0.6" +version = "4.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcccbd4e4274e0f80697f5fbc6540fdac533cce02f2081b328e68629cce24f9" +checksum = "413183068e6e0289e18d7d0a1f661b81546e6918d5453a44570b9ab30cbed1b3" dependencies = [ "i_float", "i_key_sort", @@ -2439,9 +2467,9 @@ checksum = "35e6d558e6d4c7b82bc51d9c771e7a927862a161a7d87bf2b0541450e0e20915" [[package]] name = "iana-time-zone" -version = "0.1.64" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2509,9 +2537,9 @@ checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" dependencies = [ "icu_collections", "icu_locale_core", @@ -2523,9 +2551,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" [[package]] name = "icu_provider" @@ -2542,6 +2570,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "ident_case" version = "1.0.1" @@ -2577,9 +2611,9 @@ checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2" [[package]] name = "indexmap" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", "hashbrown 0.16.1", @@ -2602,15 +2636,15 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.11.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "iri-string" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" +checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" dependencies = [ "memchr", "serde", @@ -2645,9 +2679,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jobserver" @@ -2661,14 +2695,37 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" dependencies = [ "once_cell", "wasm-bindgen", ] +[[package]] +name = "jsonwebtoken" +version = "10.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1" +dependencies = [ + "base64", + "ed25519-dalek", + "getrandom 0.2.17", + "hmac", + "js-sys", + "p256", + "p384", + "pem", + "rand 0.8.5", + "rsa", + "serde", + "serde_json", + "sha2", + "signature", + "simple_asn1", +] + [[package]] name = "language-tags" version = "0.3.2" @@ -2684,6 +2741,12 @@ dependencies = [ "spin", ] +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "lexical-core" version = "1.0.6" @@ -2743,40 +2806,43 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.178" +version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "libduckdb-sys" -version = "1.4.2" +version = "1.10501.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6650a7ea86fce24fe1fbf5b037671a8b77c59d135703fc6085b8a1827e66e977" +checksum = "12096c1694924782b3fe21e790630b77bacb4fcb7ad9d7ee0fec626f985bf248" dependencies = [ "cc", "flate2", "pkg-config", + "reqwest 0.12.28", "serde", "serde_json", "tar", "vcpkg", + "zip", ] [[package]] name = "libm" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.10" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" +checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" dependencies = [ "bitflags", "libc", - "redox_syscall", + "plain", + "redox_syscall 0.7.3", ] [[package]] @@ -2792,9 +2858,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" @@ -2831,9 +2897,18 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.11.5" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" + +[[package]] +name = "lz4_flex" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +dependencies = [ + "twox-hash", +] [[package]] name = "macro_rules_attribute" @@ -2878,9 +2953,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "mime" @@ -2916,9 +2991,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "log", @@ -2951,9 +3026,9 @@ checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" [[package]] name = "native-tls" -version = "0.2.14" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" dependencies = [ "libc", "log", @@ -2961,7 +3036,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework 2.11.1", + "security-framework", "security-framework-sys", "tempfile", ] @@ -3009,21 +3084,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", + "windows-sys 0.60.2", ] [[package]] @@ -3063,9 +3124,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" [[package]] name = "num-integer" @@ -3087,17 +3148,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -3129,9 +3179,9 @@ dependencies = [ [[package]] name = "objc2" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c2599ce0ec54857b29ce62166b0ed9b4f6f1a70ccc9a71165b6154caca8c05" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" dependencies = [ "objc2-encode", ] @@ -3297,15 +3347,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf" dependencies = [ "bitflags", "cfg-if", @@ -3324,20 +3374,20 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "openssl-probe" -version = "0.1.6" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" dependencies = [ "cc", "libc", @@ -3347,9 +3397,9 @@ dependencies = [ [[package]] name = "os_info" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c39b5918402d564846d5aba164c09a66cc88d232179dfd3e3c619a25a268392" +checksum = "e4022a17595a00d6a369236fdae483f0de7f0a339960a53118b818238e132224" dependencies = [ "android_system_properties", "log", @@ -3361,6 +3411,30 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + [[package]] name = "parking" version = "2.2.1" @@ -3385,7 +3459,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec 1.15.1", "windows-link", ] @@ -3396,6 +3470,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -3413,39 +3497,40 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "petgraph" -version = "0.7.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", + "hashbrown 0.15.5", "indexmap", ] [[package]] name = "pin-project" -version = "1.1.10" +version = "1.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.10" +version = "1.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "pin-project-lite" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pin-utils" @@ -3480,6 +3565,12 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "polonius-the-crab" version = "0.5.0" @@ -3521,32 +3612,64 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.111", + "syn 2.0.117", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", ] [[package]] name = "proc-macro-crate" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ "toml_edit", ] [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] +[[package]] +name = "prometheus-client" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cca3d75b4566b9a29fe1ed623587fb058e826eb329a0be4b7c4da1ebb2d7a6ca" +dependencies = [ + "dtoa", + "itoa", + "parking_lot", + "prometheus-client-derive-encode", +] + +[[package]] +name = "prometheus-client-derive-encode" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9adf1691c04c0a5ff46ff8f262b58beb07b0dbb61f96f9f54f6cbd82106ed87f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "prost" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", "prost-derive", @@ -3554,15 +3677,14 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "itertools 0.14.0", "log", "multimap", - "once_cell", "petgraph", "prettyplease", "prost", @@ -3570,28 +3692,28 @@ dependencies = [ "pulldown-cmark", "pulldown-cmark-to-cmark", "regex", - "syn 2.0.111", + "syn 2.0.117", "tempfile", ] [[package]] name = "prost-derive" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "prost-types" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" dependencies = [ "prost", ] @@ -3618,9 +3740,9 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" +checksum = "83c41efbf8f90ac44de7f3a868f0867851d261b56291732d0cbf7cceaaeb55a6" dependencies = [ "bitflags", "memchr", @@ -3629,9 +3751,9 @@ dependencies = [ [[package]] name = "pulldown-cmark-to-cmark" -version = "21.1.0" +version = "22.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8246feae3db61428fd0bb94285c690b460e4517d83152377543ca802357785f1" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" dependencies = [ "pulldown-cmark", ] @@ -3664,8 +3786,8 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.1", - "thiserror 2.0.17", + "socket2 0.6.3", + "thiserror 2.0.18", "tokio", "tracing", "web-time", @@ -3673,9 +3795,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "bytes", "getrandom 0.3.4", @@ -3686,7 +3808,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.17", + "thiserror 2.0.18", "tinyvec", "tracing", "web-time", @@ -3701,16 +3823,16 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.1", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.42" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -3721,6 +3843,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "radium" version = "0.7.0" @@ -3745,7 +3873,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.3", + "rand_core 0.9.5", ] [[package]] @@ -3765,7 +3893,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.3", + "rand_core 0.9.5", ] [[package]] @@ -3774,14 +3902,14 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", ] [[package]] name = "rand_core" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" dependencies = [ "getrandom 0.3.4", ] @@ -3824,6 +3952,15 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_syscall" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" +dependencies = [ + "bitflags", +] + [[package]] name = "ref-cast" version = "1.0.25" @@ -3841,14 +3978,14 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "regex" -version = "1.12.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -3858,9 +3995,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -3869,15 +4006,15 @@ dependencies = [ [[package]] name = "regex-lite" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "rend" @@ -3890,15 +4027,56 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.24" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "reqwest" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" +checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", "futures-core", "futures-util", + "h2", "http 1.4.0", "http-body", "http-body-util", @@ -3911,16 +4089,12 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", - "quinn", - "rustls", "rustls-pki-types", "serde", "serde_json", - "serde_urlencoded", "sync_wrapper", "tokio", "tokio-native-tls", - "tokio-rustls", "tower", "tower-http", "tower-service", @@ -3928,7 +4102,16 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", ] [[package]] @@ -3939,7 +4122,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.16", + "getrandom 0.2.17", "libc", "untrusted", "windows-sys 0.52.0", @@ -3947,9 +4130,9 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.7.45" +version = "0.7.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" +checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" dependencies = [ "bitvec", "bytecheck", @@ -3965,9 +4148,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.45" +version = "0.7.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" +checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" dependencies = [ "proc-macro2", "quote", @@ -3982,20 +4165,20 @@ checksum = "4e27ee8bb91ca0adcf0ecb116293afa12d393f9c2b9b9cd54d33e8078fe19839" [[package]] name = "rrdcached-client" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57dfd6f5a3094934b1f0813199b7571be5bde0bcc985005fe5a3c3d6a738d4cd" +checksum = "dea5c1441ef6aac3268b11aba80e084758e286b0c14d68bad4b70a79d2fbfe7f" dependencies = [ "nom 8.0.0", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", ] [[package]] name = "rsa" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40a0376c50d0358279d9d643e4bf7b7be212f1f4ff1da9070a7b54d22ef75c88" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ "const-oid", "digest", @@ -4024,9 +4207,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.39.0" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35affe401787a9bd846712274d97654355d21b2a2c092a3139aabe31e9022282" +checksum = "2ce901f9a19d251159075a4c37af514c3b8ef99c22e02dd8c19161cf397ee94a" dependencies = [ "arrayvec", "borsh", @@ -4036,13 +4219,14 @@ dependencies = [ "rkyv", "serde", "serde_json", + "wasm-bindgen", ] [[package]] name = "rustc-demangle" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" [[package]] name = "rustc-hash" @@ -4061,22 +4245,22 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.2" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] name = "rustls" -version = "0.23.35" +version = "0.23.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" dependencies = [ "aws-lc-rs", "log", @@ -4090,30 +4274,21 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.5.1", -] - -[[package]] -name = "rustls-pemfile" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" -dependencies = [ - "rustls-pki-types", + "security-framework", ] [[package]] name = "rustls-pki-types" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "708c0f9d5f54ba0272468c1d306a52c495b31fa155e91bc25371e6df7996908c" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ "web-time", "zeroize", @@ -4121,9 +4296,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.8" +version = "0.103.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +checksum = "20a6af516fea4b20eccceaf166e8aa666ac996208e8a644ce3ef5aa783bc7cd4" dependencies = [ "aws-lc-rs", "ring", @@ -4147,24 +4322,24 @@ dependencies = [ "crc32c", "nom 8.0.0", "smallvec 2.0.0-alpha.12", - "thiserror 2.0.17", + "thiserror 2.0.18", "xxhash-rust", ] [[package]] name = "rusty-promql-parser" -version = "0.1.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a9f9a20037ce456faae4f9e123f950d36ad6e1fda51f165b8e065d76ec4fc26" +checksum = "75dce42a82b9ab7c37b523ddf2f04e89b18e3df5ec382c884d4d98315b28480b" dependencies = [ "nom 8.0.0", ] [[package]] name = "ryu" -version = "1.0.20" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "scc" @@ -4177,18 +4352,18 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] [[package]] name = "schemars" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ "dyn-clone", "ref-cast", @@ -4215,26 +4390,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] -name = "security-framework" -version = "2.11.1" +name = "sec1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "bitflags", - "core-foundation 0.9.4", - "core-foundation-sys", - "libc", - "security-framework-sys", + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", ] [[package]] name = "security-framework" -version = "3.5.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags", - "core-foundation 0.10.1", + "core-foundation", "core-foundation-sys", "libc", "security-framework-sys", @@ -4242,9 +4418,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.15.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", @@ -4261,11 +4437,13 @@ name = "sensapp" version = "0.3.0" dependencies = [ "anyhow", - "arrow 57.1.0", + "arrow", + "arrow-array", "arrow-ipc", + "arrow-schema", "async-trait", "axum", - "base64 0.22.1", + "base64", "big-decimal-byte-string-encoder", "bigdecimal", "blake3", @@ -4284,8 +4462,10 @@ dependencies = [ "hybridmap", "influxdb-line-protocol", "iso8601", + "jsonwebtoken", "nom 8.0.0", "once_cell", + "prometheus-client", "prost", "regex", "rrdcached-client", @@ -4298,13 +4478,14 @@ dependencies = [ "serde", "serde_json", "serial_test", + "simplify-polyline", "sindit-senml", "sinteflake", "smallvec 1.15.1", "snap", "sqlx", "temp-env", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-stream", "tokio-test", @@ -4323,13 +4504,14 @@ dependencies = [ [[package]] name = "sentry" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9794f69ad475e76c057e326175d3088509649e3aed98473106b9fe94ba59424" +checksum = "eb25f439f97d26fea01d717fa626167ceffcd981addaa670001e70505b72acbb" dependencies = [ + "cfg_aliases", "httpdate", "native-tls", - "reqwest", + "reqwest 0.13.2", "sentry-actix", "sentry-anyhow", "sentry-backtrace", @@ -4345,9 +4527,9 @@ dependencies = [ [[package]] name = "sentry-actix" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0fee202934063ace4f1d1d063113b8982293762628e563a2d2fba08fb20b110" +checksum = "9453d18fc9a45d841636004aad50288d80cc07c34a9e88cd4397cb66e6356f67" dependencies = [ "actix-http", "actix-web", @@ -4358,9 +4540,9 @@ dependencies = [ [[package]] name = "sentry-anyhow" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1832cd051f3198af8ebc5222da5fd213e49976d758b7defa7accbff3aed1909" +checksum = "e15257eeba11d42eb0d6893fed26033504b031abaef3b3809343e4967f476502" dependencies = [ "anyhow", "sentry-backtrace", @@ -4369,9 +4551,9 @@ dependencies = [ [[package]] name = "sentry-backtrace" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e81137ad53b8592bd0935459ad74c0376053c40084aa170451e74eeea8dbc6c3" +checksum = "46a8c2c1bd5c1f735e84f28b48e7d72efcaafc362b7541bc8253e60e8fcdffc6" dependencies = [ "backtrace", "regex", @@ -4380,9 +4562,9 @@ dependencies = [ [[package]] name = "sentry-contexts" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfb403c66cc2651a01b9bacda2e7c22cd51f7e8f56f206aa4310147eb3259282" +checksum = "9b88a90baa654d7f0e1f4b667f6b434293d9f72c71bef16b197c76af5b7d5803" dependencies = [ "hostname", "libc", @@ -4394,9 +4576,9 @@ dependencies = [ [[package]] name = "sentry-core" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc409727ae90765ca8ea76fe6c949d6f159a11d02e130b357fa652ee9efcada" +checksum = "0ac170a5bba8bec6e3339c90432569d89641fa7a3d3e4f44987d24f0762e6adf" dependencies = [ "rand 0.9.2", "sentry-types", @@ -4407,9 +4589,9 @@ dependencies = [ [[package]] name = "sentry-debug-images" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06a2778a222fd90ebb01027c341a72f8e24b0c604c6126504a4fe34e5500e646" +checksum = "dd9646a972b57896d4a92ed200cf76139f8e30b3cfd03b6662ae59926d26633c" dependencies = [ "findshlibs", "sentry-core", @@ -4417,9 +4599,9 @@ dependencies = [ [[package]] name = "sentry-panic" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df79f4e1e72b2a8b75a0ebf49e78709ceb9b3f0b451f13adc92a0361b0aaabe" +checksum = "6127d3d304ba5ce0409401e85aae538e303a569f8dbb031bf64f9ba0f7174346" dependencies = [ "sentry-backtrace", "sentry-core", @@ -4427,9 +4609,9 @@ dependencies = [ [[package]] name = "sentry-tower" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eec9885bceb8ba374858d015bb6fa39dbb341d94ca088bc8f13bee2e64e2c68" +checksum = "61c5253dc4ad89863a866b93aeaaac1c9d60f2f774663b5024afe2d57e0a101c" dependencies = [ "sentry-core", "tower-layer", @@ -4438,9 +4620,9 @@ dependencies = [ [[package]] name = "sentry-tracing" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2046f527fd4b75e0b6ab3bd656c67dce42072f828dc4d03c206d15dca74a93" +checksum = "27701acc51e68db5281802b709010395bfcbcb128b1d0a4e5873680d3b47ff0c" dependencies = [ "bitflags", "sentry-backtrace", @@ -4451,16 +4633,16 @@ dependencies = [ [[package]] name = "sentry-types" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7b9b4e4c03a4d3643c18c78b8aa91d2cbee5da047d2fa0ca4bb29bc67e6c55c" +checksum = "56780cb5597d676bf22e6c11d1f062eb4def46390ea3bfb047bcbcf7dfd19bdb" dependencies = [ "debugid", "hex", "rand 0.9.2", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", "url", "uuid", @@ -4493,7 +4675,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -4504,20 +4686,20 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "serde_json" -version = "1.0.145" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", "memchr", - "ryu", "serde", "serde_core", + "zmij", ] [[package]] @@ -4533,9 +4715,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24345aa0fe688594e73770a5f6d1b216508b4f93484c0026d521acd30134392" +checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" dependencies = [ "serde_core", ] @@ -4554,11 +4736,12 @@ dependencies = [ [[package]] name = "serial_test" -version = "3.2.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9" +checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f" dependencies = [ - "futures", + "futures-executor", + "futures-util", "log", "once_cell", "parking_lot", @@ -4568,13 +4751,13 @@ dependencies = [ [[package]] name = "serial_test_derive" -version = "3.2.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" +checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -4584,7 +4767,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -4595,7 +4778,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -4616,16 +4799,17 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "sif-itree" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142099cd6db3c4fab61e5133c62ff80b26674391e195860791fda0b1be3e5080" +checksum = "d7f45b8998ced5134fb1d75732c77842a3e888f19c1ff98481822e8fbfbf930b" [[package]] name = "signal-hook-registry" -version = "1.4.7" +version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ + "errno", "libc", ] @@ -4641,9 +4825,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" [[package]] name = "simdutf8" @@ -4651,19 +4835,40 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "simple_asn1" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "simplify-polyline" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d236f7d768695e098c0c026be222f48b6c26558ce472e4c1abcbc3cc7e0fd69e" +dependencies = [ + "num-traits", +] + [[package]] name = "sindit-senml" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36fd7adab1c6f40d6eda0cc8655f810c93e672efc7a4fcaa8700b50dd8becf3f" +checksum = "2aab5a430865bbb5114c3985b3dc78226bd3e032b15f82e59d9e8c4884260eec" dependencies = [ - "base64 0.21.7", + "base64", "chrono", "once_cell", "regex", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 2.0.18", ] [[package]] @@ -4682,15 +4887,15 @@ dependencies = [ [[package]] name = "siphasher" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" [[package]] name = "slab" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" @@ -4719,12 +4924,12 @@ dependencies = [ [[package]] name = "snafu" -version = "0.8.9" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" dependencies = [ "backtrace", - "snafu-derive 0.8.9", + "snafu-derive 0.9.0", ] [[package]] @@ -4741,14 +4946,14 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.8.9" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -4769,9 +4974,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", "windows-sys 0.60.2", @@ -4827,7 +5032,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "crc", "crossbeam-queue", @@ -4849,7 +5054,7 @@ dependencies = [ "serde_json", "sha2", "smallvec 1.15.1", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", "tokio", "tokio-stream", @@ -4868,7 +5073,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -4891,7 +5096,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.111", + "syn 2.0.117", "tokio", "url", ] @@ -4903,7 +5108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", - "base64 0.22.1", + "base64", "bitflags", "byteorder", "bytes", @@ -4934,7 +5139,7 @@ dependencies = [ "smallvec 1.15.1", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", "tracing", "uuid", @@ -4948,7 +5153,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", - "base64 0.22.1", + "base64", "bitflags", "byteorder", "crc", @@ -4974,7 +5179,7 @@ dependencies = [ "smallvec 1.15.1", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", "tracing", "uuid", @@ -5001,7 +5206,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", "tracing", "url", @@ -5031,32 +5236,13 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" - [[package]] name = "strum" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros 0.27.2", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.111", + "strum_macros", ] [[package]] @@ -5068,7 +5254,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -5090,9 +5276,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.111" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -5116,7 +5302,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -5127,9 +5313,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" dependencies = [ "filetime", "libc", @@ -5147,15 +5333,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.23.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5169,11 +5355,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.17", + "thiserror-impl 2.0.18", ] [[package]] @@ -5184,18 +5370,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -5209,9 +5395,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.44" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", @@ -5219,22 +5405,22 @@ dependencies = [ "num-conv", "num_threads", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.24" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", @@ -5261,9 +5447,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -5276,9 +5462,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.48.0" +version = "1.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +checksum = "f66bf9585cda4b724d3e78ab34b73fb2bbaba9011b9bfdf69dc836382ea13b8c" dependencies = [ "bytes", "libc", @@ -5286,20 +5472,20 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.1", + "socket2 0.6.3", "tokio-macros", "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -5324,9 +5510,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ "futures-core", "pin-project-lite", @@ -5335,12 +5521,10 @@ dependencies = [ [[package]] name = "tokio-test" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468baabc3311435b55dd935f702f42cd1b8abb7e754fb7dfb16bd36aa88f9f7" +checksum = "3f6d24790a10a7af737693a3e8f1d03faef7e6ca0cc99aae5066f533766de545" dependencies = [ - "async-stream", - "bytes", "futures-core", "tokio", "tokio-stream", @@ -5348,9 +5532,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.17" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", @@ -5361,14 +5545,14 @@ dependencies = [ [[package]] name = "toml" -version = "0.9.8" +version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ "indexmap", "serde_core", "serde_spanned", - "toml_datetime", + "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", "toml_writer", "winnow", @@ -5376,49 +5560,58 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.3" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_datetime" +version = "1.0.0+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" +checksum = "32c2555c699578a4f59f0cc68e5116c8d7cabbd45e1409b989d4be085b53f13e" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.23.7" +version = "0.25.4+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" +checksum = "7193cbd0ce53dc966037f54351dbbcf0d5a642c7f0038c382ef9e677ce8c13f2" dependencies = [ "indexmap", - "toml_datetime", + "toml_datetime 1.0.0+spec-1.1.0", "toml_parser", "winnow", ] [[package]] name = "toml_parser" -version = "1.0.4" +version = "1.0.9+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" +checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" dependencies = [ "winnow", ] [[package]] name = "toml_writer" -version = "1.0.4" +version = "1.0.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" +checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" [[package]] name = "tonic" -version = "0.14.2" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" +checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" dependencies = [ "async-trait", "axum", - "base64 0.22.1", + "base64", "bytes", "flate2", "h2", @@ -5431,7 +5624,7 @@ dependencies = [ "percent-encoding", "pin-project", "rustls-native-certs", - "socket2 0.6.1", + "socket2 0.6.3", "sync_wrapper", "tokio", "tokio-rustls", @@ -5445,21 +5638,21 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.14.2" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c40aaccc9f9eccf2cd82ebc111adc13030d23e887244bc9cfa5d1d636049de3" +checksum = "1882ac3bf5ef12877d7ed57aad87e75154c11931c2ba7e6cde5e22d63522c734" dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "tonic-prost" -version = "0.14.2" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" +checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" dependencies = [ "bytes", "prost", @@ -5468,25 +5661,25 @@ dependencies = [ [[package]] name = "tonic-prost-build" -version = "0.14.2" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4a16cba4043dc3ff43fcb3f96b4c5c154c64cbd18ca8dce2ab2c6a451d058a2" +checksum = "f3144df636917574672e93d0f56d7edec49f90305749c668df5101751bb8f95a" dependencies = [ "prettyplease", "proc-macro2", "prost-build", "prost-types", "quote", - "syn 2.0.111", + "syn 2.0.117", "tempfile", "tonic-build", ] [[package]] name = "tower" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", @@ -5504,12 +5697,12 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf146f99d442e8e68e585f5d798ccd3cad9a7835b917e09728880a862706456" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "async-compression", - "base64 0.22.1", + "base64", "bitflags", "bytes", "futures-core", @@ -5547,9 +5740,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "log", "pin-project-lite", @@ -5565,14 +5758,14 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "tracing-core" -version = "0.1.35" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", "valuable", @@ -5591,9 +5784,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ "matchers", "nu-ansi-term", @@ -5613,6 +5806,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + [[package]] name = "typenum" version = "1.19.0" @@ -5630,9 +5829,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.8.1" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" [[package]] name = "unicode-bidi" @@ -5642,9 +5841,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-normalization" @@ -5687,11 +5886,11 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "3.1.4" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39cb1dbab692d82a977c0392ffac19e188bd9186a9f32806f0aaa859d75585a" +checksum = "fdc97a28575b85cfedf2a7e7d3cc64b3e11bd8ac766666318003abbacc7a21fc" dependencies = [ - "base64 0.22.1", + "base64", "der", "log", "native-tls", @@ -5708,7 +5907,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d81f9efa9df032be5934a46a068815a10a042b494b6a58cb0a1a97bb5467ed6f" dependencies = [ - "base64 0.22.1", + "base64", "http 1.4.0", "httparse", "log", @@ -5716,14 +5915,15 @@ dependencies = [ [[package]] name = "url" -version = "2.5.7" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -5771,7 +5971,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -5788,11 +5988,11 @@ dependencies = [ [[package]] name = "uuid" -version = "1.19.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.2", "js-sys", "serde_core", "wasm-bindgen", @@ -5833,9 +6033,18 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.1+wasi-0.2.4" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ "wit-bindgen", ] @@ -5848,24 +6057,26 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" dependencies = [ "cfg-if", "once_cell", "rustversion", + "serde", "wasm-bindgen-macro", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.56" +version = "0.4.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" dependencies = [ "cfg-if", + "futures-util", "js-sys", "once_cell", "wasm-bindgen", @@ -5874,9 +6085,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5884,31 +6095,65 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + [[package]] name = "web-sys" -version = "0.3.83" +version = "0.3.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" dependencies = [ "js-sys", "wasm-bindgen", @@ -5926,18 +6171,18 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.4" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee3e3b5f5e80bc89f30ce8d0343bf4e5f12341c51f3e26cbeecbc7c85443e85b" +checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" dependencies = [ "rustls-pki-types", ] [[package]] name = "webpki-roots" -version = "1.0.4" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" dependencies = [ "rustls-pki-types", ] @@ -5995,7 +6240,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -6006,7 +6251,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -6257,18 +6502,100 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" dependencies = [ "memchr", ] [[package]] name = "wit-bindgen" -version = "0.46.0" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck 0.5.0", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck 0.5.0", + "indexmap", + "prettyplease", + "syn 2.0.117", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] [[package]] name = "writeable" @@ -6320,18 +6647,18 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "synstructure", ] [[package]] name = "yup-oauth2" -version = "12.1.1" +version = "12.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f16c9757965c8c261c64d2ca27a8335688b54cc2147f135fd36449d7bbfcff00" +checksum = "ef19a12dfb29fe39f78e1547e1be49717b84aef8762a4001359ed4f94d3accc1" dependencies = [ "async-trait", - "base64 0.22.1", + "base64", "http 1.4.0", "http-body-util", "hyper", @@ -6340,11 +6667,10 @@ dependencies = [ "log", "percent-encoding", "rustls", - "rustls-pemfile", "seahash", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", "tokio", "url", @@ -6352,22 +6678,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.31" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.31" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -6387,7 +6713,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "synstructure", ] @@ -6427,7 +6753,45 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", +] + +[[package]] +name = "zip" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap", + "memchr", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 73ddb385..dc67621d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ version = "0.3.0" edition = "2024" [features] -default = ["postgres"] +default = ["postgres", "sqlite"] # Storage backends postgres = ["sqlx/postgres"] sqlite = ["sqlx/sqlite", "sqlx/regexp"] @@ -14,6 +14,7 @@ bigquery = [ "dep:gcp-bigquery-client", "dep:bigdecimal", "dep:big-decimal-byte-string-encoder", + "dep:tonic", ] rrdcached = ["dep:rrdcached-client"] clickhouse = ["dep:clickhouse"] @@ -33,9 +34,10 @@ test-utils = [] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -# Apache Arrow support for efficient columnar data format -arrow = { version = "57", features = ["ipc"] } -arrow-ipc = "57" +arrow = { version = "58", features = ["ipc", "ipc_compression"] } +arrow-ipc = { version = "58", features = ["lz4"] } +arrow-schema = "58" +arrow-array = "58" anyhow = "1.0" async-trait = "0.1" thiserror = "2.0" @@ -49,21 +51,22 @@ sqlx = { version = "0.8", features = [ "time", "rust_decimal", ] } -tokio = { version = "1.47", features = ["full"] } +tokio = { version = "1.48", features = ["full"] } tokio-stream = { version = "0.1", features = ["io-util"] } tokio-util = "0.7" tower = { version = "0.5", features = ["full"] } tower-http = { version = "0.6", features = ["full"] } tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } uuid = { version = "1.18", features = ["serde"] } csv-async = "1.3" -rust_decimal = "1.37" +rust_decimal = "1.39" geo = "0.32" -cached = { version = "0.56", features = ["async", "tokio", "async-trait"] } +cached = { version = "0.59", features = ["async", "tokio", "async-trait"] } nom = "8.0" +sindit-senml = "0.3" serde_json = "1.0" -hifitime = "4.1" +hifitime = "4.2" iso8601 = "0.6" duckdb = { version = "1.3", features = ["bundled"], optional = true } serde = "1.0" @@ -73,37 +76,40 @@ prost = "0.14" snap = "1.1" rusty-chunkenc = "0.1" blake3 = "1.8" -regex = "1.11" -sindit-senml = "0.2" +regex = "1.12" influxdb-line-protocol = "2.0" -flate2 = "1.1" +flate2 = "1.1.4" smallvec = "1.15" -once_cell = "1.21" +once_cell = "1.21.4" urlencoding = "2.1" -hybridmap = "0.1" -sentry = { version = "0.46", features = ["anyhow", "tower"] } +hybridmap = "0.1.3" +sentry = { version = "0.47", features = ["anyhow", "tower"] } url = "2.5" -gcp-bigquery-client = { version = "0.27", optional = true } +gcp-bigquery-client = { version = "0.28", optional = true } sinteflake = { version = "0.1", features = ["async"] } -tonic = "0.14" +tonic = { version = "0.14", optional = true } bigdecimal = { version = "0.4", optional = true } big-decimal-byte-string-encoder = { version = "0.1", optional = true } clru = "0.6" utoipa = { version = "5.4", features = ["axum_extras"] } utoipa-scalar = { version = "0.3", features = ["axum"] } -rrdcached-client = { version = "0.1", optional = true } -clickhouse = { version = "0.14", features = [ +rrdcached-client = { version = "0.2", optional = true } +clickhouse = { version = "0.15", features = [ "lz4", "uuid", "time", "inserter", ], optional = true } +jsonwebtoken = { version = "10", features = ["rust_crypto"] } rustls = "0.23" -rusty-promql-parser = "0.1" +rusty-promql-parser = "0.2.1" +prometheus-client = "0.24" +simplify-polyline = "0.5" [dev-dependencies] temp-env = "0.3" tokio-test = "0.4" serial_test = "3.1" +jsonwebtoken = { version = "10", features = ["rust_crypto"] } # Re-import the same crate with test-utils feature for integration tests sensapp = { path = ".", features = ["test-utils"], default-features = false } diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..6d883c95 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,55 @@ +FROM rust:1.89-slim-bookworm AS builder + +ARG FEATURES="postgres,sqlite,timescaledb,duckdb,clickhouse,rrdcached" +ARG NO_DEFAULT_FEATURES="true" +ARG DUCKDB_DOWNLOAD_LIB="1" + +ENV DUCKDB_DOWNLOAD_LIB=${DUCKDB_DOWNLOAD_LIB} + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + clang \ + cmake \ + libssl-dev \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +# Prime the dependency cache before copying the full source tree. +COPY Cargo.toml Cargo.lock build.rs settings.toml ./ +COPY src ./src + +RUN if [ "$NO_DEFAULT_FEATURES" = "true" ]; then \ + cargo build --locked --release --bin sensapp --no-default-features --features "$FEATURES"; \ + else \ + cargo build --locked --release --bin sensapp --features "$FEATURES"; \ + fi + +FROM debian:bookworm-slim AS runtime + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + libgcc-s1 \ + libssl3 \ + libstdc++6 \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --system --gid 10001 sensapp \ + && useradd --system --uid 10001 --gid sensapp --home-dir /var/lib/sensapp --create-home sensapp + +WORKDIR /var/lib/sensapp + +COPY --from=builder /app/target/release/sensapp /usr/local/bin/sensapp + +ENV SENSAPP_ENDPOINT=0.0.0.0 \ + SENSAPP_PORT=3000 \ + SENSAPP_STORAGE_CONNECTION_STRING=sqlite:///var/lib/sensapp/sensapp.db \ + RUST_LOG=info + +EXPOSE 3000 + +USER sensapp:sensapp + +ENTRYPOINT ["/usr/local/bin/sensapp"] diff --git a/HUMAN-TODO.md b/HUMAN-TODO.md new file mode 100644 index 00000000..3b7c5b74 --- /dev/null +++ b/HUMAN-TODO.md @@ -0,0 +1,11 @@ +# List of TODO for the human point of view + +I would like you to analyse the state of the project, and notably start working on merging stale remote branches: + +sprint-antoine, vibe-coding-2, rrdcached + +Perhaps start the work branch per branch. I would then enjoy if you can make a summary of the what has been achieved, what has been left to do and so on. We will also start working with a new pattern for keeping up to date: + +We will have a "current_tasks" folder and a "ideas" folder, and a "done" folder. Please create those and add empty .keep files in each of them. Then, you will add for instructions in the AGENTS.md that major tasks left to be done should be added in the "current_tasks" folder, and that any new ideas should be added in the "ideas" folder. Once a task is done, it should be moved to the "done" folder. It's important to maintain those. + +This way, we will have a clear overview of what is currently being worked on, what ideas we have for the future, and what has already been accomplished. It will also help us to stay organized and focused on our goals. diff --git a/Makefile.toml b/Makefile.toml index 3db12791..bc2c5888 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -11,7 +11,7 @@ RUST_BACKTRACE = { value = "1", condition = { env_not_set = ["RUST_BACKTRACE"] } # PostgreSQL defaults (can be overridden) POSTGRES_USER = { value = "postgres", condition = { env_not_set = ["POSTGRES_USER"] } } POSTGRES_PASSWORD = { value = "postgres", condition = { env_not_set = ["POSTGRES_PASSWORD"] } } -POSTGRES_DB = { value = "sensapp", condition = { env_not_set = ["POSTGRES_DB"] } } +POSTGRES_DB = { value = "sensapp-test", condition = { env_not_set = ["POSTGRES_DB"] } } POSTGRES_HOST = { value = "localhost", condition = { env_not_set = ["POSTGRES_HOST"] } } POSTGRES_PORT = { value = "5432", condition = { env_not_set = ["POSTGRES_PORT"] } } DATABASE_URL = { value = "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}", condition = { env_not_set = ["DATABASE_URL"] } } @@ -19,7 +19,10 @@ DATABASE_URL = { value = "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POS # Test database URLs with conditional setting TEST_DATABASE_URL_POSTGRES = { value = "${DATABASE_URL}", condition = { env_not_set = ["TEST_DATABASE_URL_POSTGRES"] } } TEST_DATABASE_URL_SQLITE = { value = "sqlite://test.db", condition = { env_not_set = ["TEST_DATABASE_URL_SQLITE"] } } +TEST_DATABASE_URL_TIMESCALEDB = { value = "timescaledb://postgres:postgres@localhost:5433/sensapp-test", condition = { env_not_set = ["TEST_DATABASE_URL_TIMESCALEDB"] } } +TEST_DATABASE_URL_DUCKDB = { value = "duckdb://test.duckdb", condition = { env_not_set = ["TEST_DATABASE_URL_DUCKDB"] } } TEST_DATABASE_URL_CLICKHOUSE = { value = "clickhouse://default:password@localhost:8123/sensapp_test", condition = { env_not_set = ["TEST_DATABASE_URL_CLICKHOUSE"] } } +TEST_DATABASE_URL_RRDCACHED = { value = "rrdcached://127.0.0.1:42217?preset=hoarder", condition = { env_not_set = ["TEST_DATABASE_URL_RRDCACHED"] } } # Environment profiles [env.ci] @@ -49,6 +52,24 @@ args = ["test"] description = "Test all storage backend features" dependencies = ["test-sqlite", "test-postgres", "test-clickhouse", "test-timescaledb", "test-duckdb", "test-bigquery", "test-rrdcached"] +[tasks.start-local-test-stack] +description = "Start the local Docker test services (PostgreSQL, TimescaleDB, ClickHouse, RRDCached)" +command = "docker" +args = ["compose", "-f", "compose.test-services.yml", "up", "-d", "--build"] + +[tasks.stop-local-test-stack] +description = "Stop the local Docker test services" +command = "docker" +args = ["compose", "-f", "compose.test-services.yml", "down"] + +[tasks.test-local-matrix] +description = "Run the local backend test matrix (SQLite, PostgreSQL, TimescaleDB, ClickHouse, DuckDB, RRDCached)" +dependencies = ["start-local-test-stack", "test-sqlite", "test-postgres", "test-clickhouse", "test-timescaledb", "test-duckdb", "test-rrdcached"] + +[tasks.check-local-matrix] +description = "Run build, test, and clippy for the local backend matrix (excludes BigQuery)" +dependencies = ["start-local-test-stack", "check-sqlite", "check-postgres", "check-clickhouse", "check-timescaledb", "check-duckdb", "check-rrdcached"] + [tasks.check-all] description = "Run all checks for working features (PostgreSQL and SQLite)" dependencies = ["check-sqlite", "check-postgres"] @@ -149,7 +170,7 @@ env = { "CARGO_MAKE_TASK_FEATURE" = "postgres" } extend = "template-test" private = false description = "Test with PostgreSQL feature" -env = { "CARGO_MAKE_TASK_FEATURE" = "postgres", "TEST_DATABASE_URL" = "${TEST_DATABASE_URL_POSTGRES}" } +env = { "CARGO_MAKE_TASK_FEATURE" = "postgres", "TEST_DATABASE_URL" = "${TEST_DATABASE_URL_POSTGRES}", "SENSAPP_PG_POOL_MAX_CONNECTIONS" = "1" } [tasks.clippy-postgres] extend = "template-clippy" @@ -217,7 +238,7 @@ env = { "CARGO_MAKE_TASK_FEATURE" = "timescaledb" } extend = "template-test" private = false description = "Test with TimescaleDB feature" -env = { "CARGO_MAKE_TASK_FEATURE" = "timescaledb", "TEST_DATABASE_URL" = "${TEST_DATABASE_URL_POSTGRES}" } +env = { "CARGO_MAKE_TASK_FEATURE" = "timescaledb", "TEST_DATABASE_URL" = "${TEST_DATABASE_URL_TIMESCALEDB}", "SENSAPP_PG_POOL_MAX_CONNECTIONS" = "1" } [tasks.clippy-timescaledb] extend = "template-clippy" @@ -251,7 +272,7 @@ env = { "CARGO_MAKE_TASK_FEATURE" = "duckdb" } extend = "template-test" private = false description = "Test with DuckDB feature" -env = { "CARGO_MAKE_TASK_FEATURE" = "duckdb" } +env = { "CARGO_MAKE_TASK_FEATURE" = "duckdb", "TEST_DATABASE_URL" = "${TEST_DATABASE_URL_DUCKDB}" } [tasks.clippy-duckdb] extend = "template-clippy" @@ -282,10 +303,11 @@ description = "Build with BigQuery feature" env = { "CARGO_MAKE_TASK_FEATURE" = "bigquery" } [tasks.test-bigquery] -extend = "template-test" private = false description = "Test with BigQuery feature" env = { "CARGO_MAKE_TASK_FEATURE" = "bigquery" } +command = "cargo" +args = ["test", "--features", "bigquery", "--no-default-features", "--lib", "--bins"] [tasks.clippy-bigquery] extend = "template-clippy" @@ -316,10 +338,20 @@ description = "Build with RRDCached feature" env = { "CARGO_MAKE_TASK_FEATURE" = "rrdcached" } [tasks.test-rrdcached] -extend = "template-test" private = false -description = "Test with RRDCached feature" -env = { "CARGO_MAKE_TASK_FEATURE" = "rrdcached" } +description = "Test RRDCached with unit/bin tests plus the dedicated RRDCached integration suite" +command = "cargo" +args = [ + "test", + "--features", + "rrdcached", + "--no-default-features", + "--lib", + "--bins", + "--test", + "rrdcached_integration", +] +env = { "TEST_DATABASE_URL" = "${TEST_DATABASE_URL_RRDCACHED}" } [tasks.clippy-rrdcached] extend = "template-clippy" @@ -389,7 +421,9 @@ dependencies = ["fmt-check", "clippy-all"] [tasks.security-audit] description = "Run security audit" command = "cargo" -args = ["audit"] +# Ignore the unfixed RSA Marvin advisory for now: this codebase uses HS256 JWTs, +# and the remaining `rsa` edge is currently transitive via `jsonwebtoken`/`sqlx-mysql`. +args = ["audit", "--ignore", "RUSTSEC-2023-0071"] [tasks.clippy-all] description = "Run clippy on all storage backend features" @@ -440,6 +474,15 @@ script = ''' sqlx migrate run --source src/storage/postgresql/migrations ''' +[tasks.migrate-timescaledb] +description = "Run TimescaleDB migrations" +script_runner = "@shell" +script = ''' +# sqlx CLI doesn't understand timescaledb:// scheme, so convert to postgres:// +export DATABASE_URL=$(echo "$TEST_DATABASE_URL_TIMESCALEDB" | sed 's|^timescaledb://|postgres://|') +sqlx migrate run --source src/storage/postgresql/migrations +''' + [tasks.migrate-clickhouse] description = "Run ClickHouse migrations" env = { "DATABASE_URL" = "${TEST_DATABASE_URL_CLICKHOUSE}" } diff --git a/README.md b/README.md index aa81f1a0..656ae234 100644 --- a/README.md +++ b/README.md @@ -8,22 +8,82 @@ It handles time-series data ingestion, storage, and retrieval. From small edge d SensApp is compatible with Prometheus and InfluxDB, but with an alternative architecture that prioritise data analysis and long-term storage over ingestion performance and real-time monitoring. -Handling CPU stats for the last 24 hours? InfluxDB or Prometheus are excellent choices. Fetching average bathroom temperature over the last 10 years grouped by day? SensApp will compute that instantly while InfluxDB or Prometheus will take a while as they must read many chunks of data. +Dealing with system statistics for the last 24 hours? InfluxDB or Prometheus are excellent choices. Fetching average bathroom temperatures over the last 10 years grouped by day? SensApp will compute that instantly while InfluxDB or Prometheus will take a little while. -But you don't have to chose, both InfluxDB and Prometheus can replicate their data to SensApp for long-term storage and analysis. +But you don't have to chose, both InfluxDB and Prometheus can replicate their data to SensApp for long-term storage and analysis. So you get the best of both worlds. -Of course you can also use Sensapp as a standalone time-series database. +You can also use Sensapp as a standalone time-series database. + +## Quickstart + +The quickest way to run SensApp is with SQLite so no external database is required. + +Start SensApp with SQLite: + +```bash +SENSAPP_STORAGE_CONNECTION_STRING=sqlite://sensapp.db \ +cargo run +``` + +By default, SensApp listens on [http://127.0.0.1:3000](http://127.0.0.1:3000). + +Ingest one sample: + +```bash +curl --json '[{"n": "temperature", "v": 21.5}]' \ + http://127.0.0.1:3000/publish +``` + +Query the stored data: + +```bash +curl 'http://127.0.0.1:3000/api/v1/query?query=temperature' +# or +curl 'http://127.0.0.1:3000/api/v1/query?query=temperature&format=csv' +``` + +## Python Quickstart + +Check the [python/sensapp](./python/sensapp) documentation for more details. + +```python +import asyncio +from sensapp import SensAppClient + +async def main(): + async with SensAppClient() as client: + await client.publish("temperature", 21.5) + [series] = await client.query("temperature[1h]") + print(series.frame) + +asyncio.run(main()) +``` + +### Using Containers + +```bash +docker compose up +``` + +You can deploy SensApp on Kubernetes using [the included Helm chart](./charts/sensapp). + +```bash +helm install sensapp ./charts/sensapp +``` ## Features - **HTTP REST API** -- **Compatible with existing sensor data pipelines**: +- **Prometheus Compatibility** - **Prometheus Remote Write**: Prometheus can push data to SensApp. - **Prometheus Remote Read**: Prometheus can also read data from SensApp. - - **InfluxDB Line Protocol**: InfluxDB can push data to SensApp, or you can use SensApp instead of InfluxDB, with [Telegraf](https://github.com/influxdata/telegraf) for example. + - **Prometheus Scrape Endpoint**: SensApp exposes internal service metrics on `/prometheus/metrics`. +- **InfluxDB Compatibility**: + - **InfluxDB Line Protocol**: You can use SensApp instead of InfluxDB, with [Telegraf](https://github.com/influxdata/telegraf) for example. + - **InfluxDB Data Replication**: InfluxDB [can replicate its data to SensApp](https://docs.influxdata.com/influxdb/cloud/write-data/replication/replicate-data/). - **Data formats**: - **JSON**: Simple and widely used format for data interchange. - - **CSV**: Many users *love* CSV. + - **CSV**: The classic. - **SenML**: Standardized format for sensor data representation, that is almost unheard of but actually pretty good. - **Apache Arrow IPC Support**: Efficient IPC format for high-performance data interchange. - **Flexible Time Series DataBase Storage**: @@ -48,29 +108,9 @@ SensApp storage is based on the findings of the paper [TSM-Bench: Benchmarking T Check the [ARCHITECTURE.md](docs/ARCHITECTURE.md) file for more details. -## Development - -```bash -# Build -cargo build - -# Test -cargo test -cargo make test-all # all storage backends - -# Lint (format + clippy) -cargo make lint -cargo make lint-all # all storage backends - -# Full validation -cargo make check-all # working features (postgres + sqlite) -cargo make check-all-storage # all storage backends - -# Setup (runs migrations) -cargo make setup-dev -``` +## Authentication -Override environment variables as needed: `DATABASE_URL`, `POSTGRES_USER`, etc. +SensApp supports **optional JWT authentication**. By default, all endpoints are open. Visit [docs/JWT_AUTH.md](./docs/JWT_AUTH.md) for the authentication documentation. ## Built With Rust™️ @@ -94,7 +134,7 @@ The SensApp software is provided "as is," with no warranties, and the creators o ## You may not want to use it in production (yet) -SensApp is currently under development. It is not yet ready for production. +SensApp is currently under development. It is not ready for production. ## Acknowledgments diff --git a/TODO.md b/TODO.md index ad94285a..cc0f20b0 100644 --- a/TODO.md +++ b/TODO.md @@ -1,280 +1,67 @@ -# SensApp Research Prototype Refactoring TODO +# SensApp TODO -This document tracks the comprehensive refactoring plan for SensApp to create a focused research prototype for storage backend comparison. +This file tracks the main remaining work for SensApp. -## Phase 1: Remove Event Bus & Simplify Architecture (Week 1) +Most of the large refactoring work is done: HTTP-only ingestion, direct storage calls, DCAT/query/export endpoints, Prometheus and InfluxDB compatibility, health endpoints, Prometheus service metrics, and a substantial integration test suite are already in place. -### 🔄 Architecture Simplification +The next phase is not to add more features. It is to make the existing system solid enough for pre-production, with ClickHouse as the first serious target backend, while keeping the other backends reasonably healthy. -- [x] Create proper storage factory with runtime selection via `settings.toml` -- [x] Simplify `main.rs` initialization -- [x] Keep ALL storage backends for research comparison -- [x] **Event Bus Removal**: Completely removed event bus from `main.rs` and all components -- [x] **Direct Storage Calls**: All components now use direct storage access instead of event bus -- [x] **Storage Factory**: Implemented runtime storage backend selection via connection strings -- [x] **Architecture Simplification**: `main.rs` initialization significantly simplified -- [x] **HttpServerState Refactored**: Now contains direct storage reference instead of event bus +## Current Priorities -## Phase 2: Add ClickHouse Storage Backend (Week 1) +### 1. Pre-production readiness -### 🗄️ ClickHouse Implementation +- [ ] Make ClickHouse the reference pre-production backend +- [ ] Validate the full ingestion/query/export lifecycle against a real ClickHouse service +- [ ] Harden ClickHouse operational behavior: migrations, health checks, error messages, and recovery paths +- [ ] Add deployment and operating guidance for a ClickHouse-based setup +- [ ] Define what pre-production ready means for SensApp and document the acceptance criteria -- [x] Create `src/storage/clickhouse/` module -- [x] Implement ClickHouse storage trait -- [x] Add ClickHouse migrations -- [x] Include ClickHouse in storage factory -- [x] Document ClickHouse-specific optimizations (LowCardinality, etc.) -- [x] Add ClickHouse connection string parsing -- [x] Test ClickHouse backend with sample data +### 2. Backend quality and consistency -### ✅ Phase 2 Completed +- [ ] Keep PostgreSQL, SQLite, TimescaleDB, DuckDB, RRDCached, and ClickHouse aligned with the current `StorageInstance` trait +- [ ] Add cross-backend consistency tests for the core workflows: publish, list metrics, list series, query by UUID, query by labels, export formats +- [ ] Decide which backends are actively maintained versus experimental -- [x] **ClickHouse Module**: Full implementation in `src/storage/clickhouse/` with 3 main files -- [x] **Storage Trait**: Complete `StorageInstance` trait implementation with all methods -- [x] **Database Schema**: 11 tables with monthly partitioning, bloom filters, and materialized views -- [x] **Connection Parsing**: `clickhouse://user:password@host:port/database` format supported -- [x] **Data Types**: All 8 sensor types (Integer, Numeric, Float, String, Boolean, Location, JSON, Blob) -- [x] **Factory Integration**: Storage factory correctly routes ClickHouse URLs -- [x] **Integration Tests**: Full test suite in `tests/clickhouse_integration.rs` -- [x] **Minor Gaps**: Example config in `settings.toml` would be nice to add +### 3. Codebase cleanup -## Phase 3: Add Comprehensive Read API (Week 2) +- [ ] Remove structural duplication between the library crate and the binary crate where practical +- [ ] Ensure the project builds cleanly without duplicated code paths, duplicated tests, or unnecessary warnings +- [ ] Keep module boundaries simple and avoid reintroducing architectural complexity -### 📊 Data Retrieval API +### 4. Observability and resilience -- [x] Implement metrics catalog endpoint (`GET /metrics`) with DCAT format -- [x] Implement series catalog endpoint (`GET /series`) with DCAT format -- [x] Implement series data endpoint (`GET /series/{uuid}`) with format selection -- [x] Support time range queries with `start/end` parameters -- [x] Support data format selection: CSV, JSON Lines, SenML, Apache Arrow -- [x] Add optional limit parameter for pagination -- [x] Add aggregation endpoints for research metrics (metrics catalog) -- [x] Implement data export in multiple formats (SenML, CSV, JSONL, Arrow) -- [ ] Add query performance metrics collection -- [ ] Add offset parameter for full pagination support -- [x] **DCAT Catalog Format**: Both metrics and series endpoints use W3C DCAT standard -- [x] **Multiple Export Formats**: SenML, CSV, JSON Lines, Apache Arrow all implemented -- [x] **Time Range Queries**: Full support for ISO 8601 datetime parsing with timezone handling -- [x] **UUID-based Series Access**: Clean UUID-based series identification and querying -- [x] **Prometheus-style IDs**: Series catalog includes Prometheus-compatible identifiers -- [x] **Rich Metadata**: Comprehensive sensor metadata with labels, units, and types +- [ ] Add query latency metrics +- [ ] Add write latency metrics +- [ ] Add ingestion rate and error rate metrics +- [ ] Add structured logging with enough context to diagnose backend failures +- [ ] Add request correlation or tracing identifiers where useful +- [ ] Add ingestion rate limiting and backpressure handling -## Phase 4: Testing Infrastructure (Week 2) +### 5. Documentation -### 🧪 Unit Testing +- [ ] Write a pre-production deployment guide for ClickHouse +- [ ] Document backend trade-offs clearly +- [ ] Write a short configuration reference for common deployments +- [ ] Document which features are production-oriented and which remain research-oriented -- [ ] Unit tests for SQLite storage backend -- [ ] Unit tests for PostgreSQL storage backend -- [ ] Unit tests for TimescaleDB storage backend -- [ ] Unit tests for DuckDB storage backend -- [ ] Unit tests for BigQuery storage backend -- [ ] Unit tests for RRDCached storage backend -- [ ] Unit tests for ClickHouse storage backend (when implemented) -- [x] Unit tests for data type inference (`src/infer/`) -- [x] Unit tests for CSV parsing (`src/importers/`) -- [x] Unit tests for datetime parsing and timezone handling -- [x] Unit tests for export format handling and content types -- [x] Unit tests for Prometheus ID generation +## Deferred Work -### 🔗 Integration Testing +These are valid tasks, but not the current focus. -- [x] Integration tests for HTTP CRUD/DCAT endpoints (`tests/crud_dcat_api.rs`) -- [x] Integration tests for data ingestion (`tests/ingestion.rs`) -- [x] Integration tests for query and export functionality (`tests/query_export.rs`) -- [x] Integration tests for Apache Arrow export (`tests/arrow_integration.rs`) -- [x] Integration tests for datamodel edge cases (`tests/datamodel.rs`) -- [x] Integration tests for parser edge cases (`tests/parser_edge_cases.rs`) -- [ ] Integration tests for InfluxDB compatibility endpoints -- [ ] Integration tests for Prometheus compatibility endpoints -- [ ] Cross-storage backend data consistency tests +- [ ] Bring BigQuery back in sync with the current storage trait and query model +- [ ] Add benchmark tooling for storage backend comparison +- [ ] Add research-specific comparison endpoints and reporting helpers +- [ ] Add storage-space and latency comparison reports across backends -### ⚡ Performance Benchmarks +## Working Position -- [ ] Write latency benchmarks per storage backend -- [ ] Read latency benchmarks per storage backend -- [ ] Storage space efficiency comparison -- [ ] Concurrent write performance tests -- [ ] Test harness for automated storage backend comparison +The project currently sits between two goals: -### ✅ Phase 4 Completed +- a real tool that should be good enough for pre-production use +- a research platform that supports storage backend comparison -- [x] **Comprehensive Test Suite**: 6+ integration test files covering major functionality -- [x] **Export Format Testing**: All export formats (SenML, CSV, JSONL, Arrow) tested -- [x] **HTTP API Testing**: CRUD and DCAT API endpoints fully tested -- [x] **Data Model Testing**: Edge cases and type handling tested -- [x] **Parser Testing**: CSV and other format parsers tested +That balance is useful, but it also creates maintenance pressure. The practical rule for now should be: -## Phase 5: Configuration & Observability (Week 3) - -### ⚙️ Configuration Management - -- [x] Move ALL storage configs to `settings.toml` -- [x] Add runtime storage backend selection -- [ ] Add connection pooling configuration -- [ ] Add batch processing configuration -- [ ] Environment variable override support - -### 📈 Metrics & Monitoring - -- [ ] Add Prometheus metrics endpoint (`/metrics`) -- [ ] Write latency metrics per storage backend -- [ ] Query latency metrics per storage backend -- [ ] Data ingestion rate metrics -- [ ] Storage-specific metrics (space usage, connections) -- [ ] Error rate metrics per backend -- [ ] Queue depth and processing metrics - -### 📝 Structured Logging - -- [ ] Implement structured logging with `tracing` -- [ ] Log storage backend selection decisions -- [ ] Log performance metrics -- [ ] Log error details with context -- [ ] Add request tracing correlation IDs - -### 🏥 Health Monitoring - -- [x] Health check endpoint (`/health/live` for liveness, `/health/ready` for readiness) -- [x] Storage backend connectivity checks (all 7 backends: PostgreSQL, SQLite, TimescaleDB, ClickHouse, BigQuery, DuckDB, RRDCached) -- [x] Unit tests for health check handlers and response serialization -- [x] Integration tests for health endpoints with database verification -- [ ] Database migration status checks -- [ ] Resource usage health indicators - -### ✅ Phase 5 Completed - -- [x] **Storage Backend Selection**: Connection string based runtime selection working -- [x] **Settings Configuration**: All major settings moved to settings.toml -- [x] **Multi-Backend Support**: PostgreSQL, SQLite, DuckDB, BigQuery, TimescaleDB, RRDCached -- [x] **Network Configuration**: HTTP server endpoint and port configuration -- [x] **Sentry Integration**: Optional error tracking configuration - -## Phase 6: Streamline Ingestion (Week 3) - -### 🔧 Ingestion Simplification - -- [x] Keep HTTP ingestion (all format support) -- [x] Remove OPC UA to separate crate/service -- [x] Remove AMQP planning/references -- [x] Remove MQTT client (use Telegraf/Vector bridge instead) -- [x] Optimize batch processing without event bus -- [ ] Add ingestion rate limiting -- [ ] Add backpressure handling - -### ✅ Phase 6 Completed - -- [x] **Event Bus Removal**: Event bus completely removed from all components -- [x] **Direct Storage Access**: All ingestion now uses direct storage calls -- [x] **MQTT Client Removed**: Keeps SensApp stateless; use Telegraf/Vector for MQTT bridging -- [x] **Batch Processing**: Simplified batch processing without event bus overhead - -## Phase 7: Research Tools & Documentation (Week 4) - -### 🔬 Research Features - -- [ ] Add storage backend comparison endpoints -- [ ] Create benchmark suite for storage comparison -- [ ] Add metrics dashboard configuration (Grafana) -- [ ] Performance comparison report generation -- [ ] Data consistency verification tools -- [ ] Storage backend switching without data loss - -### 📚 Documentation - -- [ ] Document storage backend trade-offs -- [ ] Create research data collection guide -- [ ] Performance comparison methodology -- [ ] Deployment guide for each storage backend -- [ ] API documentation updates -- [ ] Configuration reference guide - -### ✅ Phase 7 Completed - -- [x] **Export Format Support**: SenML, CSV, JSONL, Apache Arrow exporters implemented -- [x] **DCAT Catalog API**: W3C DCAT standard catalog endpoints for research compatibility -- [x] **Storage Backend Comparison**: All storage backends maintained for research -- [x] **Data Format Flexibility**: Multiple ingestion and export formats supported - -## Key Principles for Research Prototype - -- ✅ Maintain all storage backends for comparison -- ✅ Focus on measurement and observability -- ✅ Make storage selection runtime configurable -- ✅ Prioritize research flexibility over production optimization -- ✅ Keep ingestion simple (HTTP only) -- ✅ Remove unnecessary complexity (event bus, OPC UA, MQTT client) - -## Session Notes - -### Session 1: Analysis and Planning (Current) - -**Completed:** - -- Created comprehensive project plan and TODO document -- Analyzed event bus usage patterns across codebase -- Identified all locations where event bus is used for storage calls -- Designed direct storage call pattern to replace event bus -- Started refactoring: Updated `HttpServerState` to remove event bus dependency - -**Key Findings:** - -- Event bus creates single-threaded bottleneck in `main.rs:178-210` -- All storage calls are serialized through single consumer task -- Event bus is used in 11 files: CSV importers, OPC UA clients, HTTP endpoints, batch builder -- Current pattern: `event_bus.publish(batch)` → single consumer → `storage.publish(batch)` -- New pattern: Direct `storage.publish(batch)` calls for parallel processing - -**What Has Been Accomplished:** - -Major refactoring completed across multiple phases: - -### Session 2-17: Major Implementation Work (🌊 waves 2-17) - -**Event Bus Removal & Architecture Simplification (✅ COMPLETED)** - -- Completely removed event bus from main.rs and all components -- Updated all HTTP endpoints to use direct storage access -- Removed MQTT client to keep SensApp stateless -- Simplified main.rs initialization significantly - -**Comprehensive Read API (✅ COMPLETED)** - -- Implemented DCAT-compliant catalog endpoints: `/metrics` and `/series` -- Added full series data endpoint: `/series/{uuid}` with format selection -- Support for multiple export formats: SenML, CSV, JSONL, Apache Arrow -- Time range queries with ISO 8601 datetime parsing and timezone handling - -**Testing Infrastructure (✅ MAJOR PROGRESS)** - -- 6+ comprehensive integration test files implemented -- All export formats tested (SenML, CSV, JSONL, Arrow) -- HTTP API endpoints fully tested -- Data model and parser edge cases tested - -**Configuration & Settings (✅ COMPLETED)** - -- All storage backends configured via settings.toml -- Runtime storage backend selection working -- Sentry and network configuration implemented - -**Research-Ready Features (✅ COMPLETED)** - -- All storage backends maintained for comparison -- DCAT catalog format for research compatibility -- Multiple data format support for flexibility - -### Next Session Plan - -**High Priority Remaining Items:** - -- [x] ClickHouse storage backend implementation (Phase 2) -- [x] Health check endpoints (/health/live, /health/ready) -- [ ] Performance benchmarks for storage backend comparison -- [ ] Metrics endpoints (/metrics for Prometheus) -- [ ] Connection pooling configuration - -**Medium Priority:** - -- [ ] Storage backend unit tests -- [ ] Query performance metrics collection -- [ ] Rate limiting and backpressure handling +- prefer making the existing core reliable over adding new capabilities +- treat ClickHouse as the first backend that must feel operationally credible +- keep other backends working, but accept that not all of them need the same maturity at the same time diff --git a/charts/sensapp/.helmignore b/charts/sensapp/.helmignore new file mode 100644 index 00000000..ad2c2700 --- /dev/null +++ b/charts/sensapp/.helmignore @@ -0,0 +1,5 @@ +.DS_Store +.git/ +.github/ +tmp/ +target/ diff --git a/charts/sensapp/Chart.yaml b/charts/sensapp/Chart.yaml new file mode 100644 index 00000000..5f878eae --- /dev/null +++ b/charts/sensapp/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v2 +name: sensapp +description: Helm chart for deploying SensApp +type: application +version: 0.1.0 +appVersion: "0.3.0" +keywords: + - sensapp + - time-series + - prometheus + - influxdb +home: https://github.com/SINTEF/sensapp +sources: + - https://github.com/SINTEF/sensapp +maintainers: + - name: SINTEF diff --git a/charts/sensapp/templates/NOTES.txt b/charts/sensapp/templates/NOTES.txt new file mode 100644 index 00000000..b7a927de --- /dev/null +++ b/charts/sensapp/templates/NOTES.txt @@ -0,0 +1,26 @@ +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "sensapp.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + kubectl get --namespace {{ .Release.Namespace }} svc {{ include "sensapp.fullname" . }} -w +{{- else if .Values.ingress.enabled }} + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "sensapp.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://{{ (index .Values.ingress.hosts 0).host }} +{{- else }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "sensapp.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 3000:3000 + echo http://127.0.0.1:3000 +{{- end }} + +{{- if .Values.auth.jwtSecret }} +2. JWT authentication is ENABLED. + Generate a token with: + sensapp generate-token --scope readwrite --duration 3600 + Use it in requests: + curl -H "Authorization: Bearer " http:///metrics +{{- else }} +2. JWT authentication is DISABLED. All endpoints are open. + To enable, set auth.jwtSecret to a random string (≥ 32 chars). +{{- end }} diff --git a/charts/sensapp/templates/_helpers.tpl b/charts/sensapp/templates/_helpers.tpl new file mode 100644 index 00000000..06dca6a9 --- /dev/null +++ b/charts/sensapp/templates/_helpers.tpl @@ -0,0 +1,41 @@ +{{- define "sensapp.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "sensapp.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "sensapp.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "sensapp.labels" -}} +helm.sh/chart: {{ include "sensapp.chart" . }} +app.kubernetes.io/name: {{ include "sensapp.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{- define "sensapp.selectorLabels" -}} +app.kubernetes.io/name: {{ include "sensapp.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "sensapp.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} +{{- default (include "sensapp.fullname" .) .Values.serviceAccount.name -}} +{{- else -}} +{{- default "default" .Values.serviceAccount.name -}} +{{- end -}} +{{- end -}} diff --git a/charts/sensapp/templates/configmap.yaml b/charts/sensapp/templates/configmap.yaml new file mode 100644 index 00000000..91825052 --- /dev/null +++ b/charts/sensapp/templates/configmap.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "sensapp.fullname" . }} + labels: + {{- include "sensapp.labels" . | nindent 4 }} +data: + SENSAPP_PORT: {{ .Values.service.port | quote }} + {{- range $key, $value := .Values.env }} + {{- if ne $key "SENSAPP_PORT" }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} diff --git a/charts/sensapp/templates/deployment.yaml b/charts/sensapp/templates/deployment.yaml new file mode 100644 index 00000000..e5b8c60c --- /dev/null +++ b/charts/sensapp/templates/deployment.yaml @@ -0,0 +1,85 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "sensapp.fullname" . }} + labels: + {{- include "sensapp.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "sensapp.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "sensapp.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "sensapp.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: sensapp + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + envFrom: + - configMapRef: + name: {{ include "sensapp.fullname" . }} + - secretRef: + name: {{ include "sensapp.fullname" . }} + {{- with .Values.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml .Values.startupProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: /var/lib/sensapp + volumes: + - name: data + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ include "sensapp.fullname" . }} + {{- else }} + emptyDir: {} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/sensapp/templates/hpa.yaml b/charts/sensapp/templates/hpa.yaml new file mode 100644 index 00000000..0509c9b1 --- /dev/null +++ b/charts/sensapp/templates/hpa.yaml @@ -0,0 +1,22 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "sensapp.fullname" . }} + labels: + {{- include "sensapp.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "sensapp.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} +{{- end }} diff --git a/charts/sensapp/templates/ingress.yaml b/charts/sensapp/templates/ingress.yaml new file mode 100644 index 00000000..b43f124a --- /dev/null +++ b/charts/sensapp/templates/ingress.yaml @@ -0,0 +1,35 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "sensapp.fullname" . }} + labels: + {{- include "sensapp.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- with .Values.ingress.tls }} + tls: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "sensapp.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/sensapp/templates/pvc.yaml b/charts/sensapp/templates/pvc.yaml new file mode 100644 index 00000000..f189eca1 --- /dev/null +++ b/charts/sensapp/templates/pvc.yaml @@ -0,0 +1,17 @@ +{{- if .Values.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "sensapp.fullname" . }} + labels: + {{- include "sensapp.labels" . | nindent 4 }} +spec: + accessModes: + {{- toYaml .Values.persistence.accessModes | nindent 4 }} + resources: + requests: + storage: {{ .Values.persistence.size }} + {{- if .Values.persistence.storageClassName }} + storageClassName: {{ .Values.persistence.storageClassName }} + {{- end }} +{{- end }} diff --git a/charts/sensapp/templates/secret.yaml b/charts/sensapp/templates/secret.yaml new file mode 100644 index 00000000..b1e81bd3 --- /dev/null +++ b/charts/sensapp/templates/secret.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "sensapp.fullname" . }} + labels: + {{- include "sensapp.labels" . | nindent 4 }} +type: Opaque +stringData: + SENSAPP_STORAGE_CONNECTION_STRING: {{ .Values.storage.connectionString | quote }} + {{- if .Values.auth.jwtSecret }} + SENSAPP_JWT_SECRET: {{ .Values.auth.jwtSecret | quote }} + {{- end }} + {{- range $key, $value := .Values.secretEnv }} + {{ $key }}: {{ $value | quote }} + {{- end }} diff --git a/charts/sensapp/templates/service.yaml b/charts/sensapp/templates/service.yaml new file mode 100644 index 00000000..f84ab1e6 --- /dev/null +++ b/charts/sensapp/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "sensapp.fullname" . }} + labels: + {{- include "sensapp.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "sensapp.selectorLabels" . | nindent 4 }} diff --git a/charts/sensapp/templates/serviceaccount.yaml b/charts/sensapp/templates/serviceaccount.yaml new file mode 100644 index 00000000..2e5dbd8d --- /dev/null +++ b/charts/sensapp/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "sensapp.serviceAccountName" . }} + labels: + {{- include "sensapp.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/sensapp/values.yaml b/charts/sensapp/values.yaml new file mode 100644 index 00000000..734c17ac --- /dev/null +++ b/charts/sensapp/values.yaml @@ -0,0 +1,110 @@ +replicaCount: 1 + +image: + repository: ghcr.io/sintef/sensapp + tag: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: + fsGroup: 10001 + +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + +service: + type: ClusterIP + port: 3000 + +ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: sensapp.local + paths: + - path: / + pathType: Prefix + tls: [] + +resources: {} + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + +nodeSelector: {} +tolerations: [] +affinity: {} + +persistence: + enabled: false + accessModes: + - ReadWriteOnce + size: 8Gi + storageClassName: "" + +env: + SENSAPP_ENDPOINT: 0.0.0.0 + SENSAPP_PORT: "3000" + SENSAPP_HTTP_BODY_LIMIT: 64MiB + SENSAPP_HTTP_SERVER_TIMEOUT_SECONDS: "30" + SENSAPP_BATCH_SIZE: "8192" + SENSAPP_INFLUXDB_WITH_NUMERIC: "false" + +storage: + connectionString: sqlite:///var/lib/sensapp/sensapp.db + +# Optional JWT authentication. +# When jwtSecret is set, read/write endpoints require a valid Bearer token. +# Health checks, docs, and prometheus scrape remain open. +# Use `sensapp generate-token ` to create tokens. +auth: + # Set to a random string of at least 32 characters to enable JWT auth. + # Leave empty (or omit) to run without authentication (default). + jwtSecret: "" + +secretEnv: {} +extraEnv: [] +extraEnvFrom: [] + +livenessProbe: + httpGet: + path: /health/live + port: http + initialDelaySeconds: 10 + periodSeconds: 15 + +readinessProbe: + httpGet: + path: /health/ready + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + +startupProbe: + httpGet: + path: /health/live + port: http + failureThreshold: 30 + periodSeconds: 5 diff --git a/compose.prometheus-roleplay.yml b/compose.prometheus-roleplay.yml new file mode 100644 index 00000000..49117fdd --- /dev/null +++ b/compose.prometheus-roleplay.yml @@ -0,0 +1,30 @@ +services: + sensapp: + build: + context: . + args: + FEATURES: duckdb + NO_DEFAULT_FEATURES: "true" + container_name: sensapp-roleplay + environment: + SENSAPP_ENDPOINT: 0.0.0.0 + SENSAPP_PORT: 3017 + SENSAPP_STORAGE_CONNECTION_STRING: duckdb:///workspaces/sensapp/python/roleplay-datascience/roleplay-session.duckdb + RUST_LOG: info + ports: + - "3017:3017" + volumes: + - .:/workspaces/sensapp + + prometheus: + image: prom/prometheus:v2.55.1 + container_name: sensapp-roleplay-prometheus + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.enable-lifecycle + depends_on: + - sensapp + ports: + - "9090:9090" + volumes: + - ./docker/prometheus-roleplay/prometheus.yml:/etc/prometheus/prometheus.yml:ro diff --git a/compose.test-services.yml b/compose.test-services.yml new file mode 100644 index 00000000..4fd55121 --- /dev/null +++ b/compose.test-services.yml @@ -0,0 +1,77 @@ +services: + postgres: + image: postgres:18 + container_name: sensapp-postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: sensapp-test + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d sensapp-test"] + interval: 5s + timeout: 5s + retries: 20 + volumes: + - postgres-test-data:/var/lib/postgresql + + timescaledb: + image: timescale/timescaledb:2.17.2-pg16 + container_name: sensapp-timescaledb + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: sensapp-test + ports: + - "5433:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d sensapp-test"] + interval: 5s + timeout: 5s + retries: 20 + volumes: + - timescaledb-test-data:/var/lib/postgresql/data + + clickhouse: + image: clickhouse/clickhouse-server:24.8 + container_name: sensapp-clickhouse + environment: + CLICKHOUSE_DB: sensapp_test + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: password + ports: + - "8123:8123" + - "9000:9000" + ulimits: + nofile: + soft: 262144 + hard: 262144 + healthcheck: + test: ["CMD-SHELL", "clickhouse-client --password password --query 'SELECT 1'"] + interval: 5s + timeout: 5s + retries: 30 + volumes: + - clickhouse-test-data:/var/lib/clickhouse + + rrdcached: + build: + context: . + dockerfile: docker/rrdcached/Dockerfile + container_name: sensapp-rrdcached + ports: + - "42217:42217" + healthcheck: + test: ["CMD", "nc", "-z", "127.0.0.1", "42217"] + interval: 5s + timeout: 5s + retries: 20 + volumes: + - rrdcached-test-data:/var/lib/rrdcached + +volumes: + postgres-test-data: + timescaledb-test-data: + clickhouse-test-data: + rrdcached-test-data: diff --git a/current_tasks/.keep b/current_tasks/.keep new file mode 100644 index 00000000..e69de29b diff --git a/current_tasks/clickhouse-preproduction-readiness.md b/current_tasks/clickhouse-preproduction-readiness.md new file mode 100644 index 00000000..09dcd69e --- /dev/null +++ b/current_tasks/clickhouse-preproduction-readiness.md @@ -0,0 +1,33 @@ +# ClickHouse Pre-Production Readiness + +## Goal + +Make ClickHouse the first operationally credible SensApp backend for pre-production deployments. + +## Context + +- `TODO.md` identifies ClickHouse as the main hardening target. +- Core ClickHouse query and pagination support already exist. +- Real-service tests already cover repeated migrations and basic health checks. +- The remaining work is now mostly about end-to-end validation, failure-path confidence, and deployment guidance. + +## Current focus + +1. ~~Add or strengthen tests for ClickHouse operational behaviors such as repeated migrations and health checks.~~ +2. ~~Validate the ingest/query/export lifecycle against a real ClickHouse service.~~ +3. ~~Add deployment and operating guidance for ClickHouse-based setups.~~ + +## Completed in this pass + +- Added integration coverage for repeated `create_or_migrate()` runs. +- Added integration coverage for ClickHouse storage `health_check()`. +- Added end-to-end ClickHouse-backed HTTP lifecycle coverage for readiness, CSV ingestion, Influx ingestion, series listing, query, and CSV export. +- Refactored ClickHouse aggregated query helpers so the ClickHouse feature set passes `cargo clippy --tests`. +- Added `docs/CLICKHOUSE.md` with connection-string, Helm, validation, and operator guidance. +- Refreshed a small set of low-risk dependencies: `cached`, `hybridmap`, `once_cell`, and `tracing-subscriber`. +4. Define and document what "pre-production ready" means for SensApp with ClickHouse. + +## Notes + +- Keep tests generic where practical, but accept backend-specific validation where operational behavior differs. +- Prefer simple, explicit checks over adding new abstractions. diff --git a/current_tasks/root-dependency-upgrade-pass.md b/current_tasks/root-dependency-upgrade-pass.md new file mode 100644 index 00000000..adaa2191 --- /dev/null +++ b/current_tasks/root-dependency-upgrade-pass.md @@ -0,0 +1,17 @@ +# Root Dependency Upgrade Pass + +## Goal + +Update the root Rust dependencies to the newest available versions that SensApp can realistically adopt now, then make the codebase pass the full validation matrix again. + +## Scope + +- refresh root dependencies reported by `cargo outdated --root-deps-only` +- fix compile, clippy, and test breakages caused by the upgrades +- run the local backend matrix against real services +- run BigQuery checks if the environment supports them + +## Notes + +- Existing branch work already included a partial low-risk dependency refresh. +- This pass should preserve unrelated in-flight changes and focus on dependency fallout only. \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..d234a735 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +services: + postgres: + image: postgres:18 + container_name: sensapp-postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: sensapp,sensapp-test + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d sensapp"] + interval: 5s + timeout: 5s + retries: 20 + volumes: + - postgres-data:/var/lib/postgresql +volumes: + postgres-data: {} diff --git a/docker/prometheus-roleplay/prometheus.yml b/docker/prometheus-roleplay/prometheus.yml new file mode 100644 index 00000000..2ff0db71 --- /dev/null +++ b/docker/prometheus-roleplay/prometheus.yml @@ -0,0 +1,20 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - prometheus:9090 + + - job_name: sensapp + metrics_path: /prometheus/metrics + static_configs: + - targets: + - sensapp:3017 + +remote_read: + - url: http://sensapp:3017/api/v1/prometheus_remote_read + read_recent: true + remote_timeout: 1m diff --git a/docker/rrdcached/Dockerfile b/docker/rrdcached/Dockerfile new file mode 100644 index 00000000..eabe417a --- /dev/null +++ b/docker/rrdcached/Dockerfile @@ -0,0 +1,11 @@ +FROM debian:bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends rrdcached netcat-openbsd \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /var/lib/rrdcached/db /var/lib/rrdcached/journal + +EXPOSE 42217 + +CMD ["rrdcached", "-g", "-B", "-R", "-l", "0.0.0.0:42217", "-b", "/var/lib/rrdcached/db", "-j", "/var/lib/rrdcached/journal", "-w", "1", "-f", "1"] diff --git a/docs/CLICKHOUSE.md b/docs/CLICKHOUSE.md new file mode 100644 index 00000000..dead88a2 --- /dev/null +++ b/docs/CLICKHOUSE.md @@ -0,0 +1,142 @@ +# ClickHouse Deployment Guide + +This guide describes the current recommended way to run SensApp with ClickHouse when the goal is operational credibility rather than local experimentation. + +## Position + +For SensApp, ClickHouse is the first backend that should feel pre-production ready. + +That means: + +- the database must be external and persistent +- SensApp instances should stay stateless +- readiness should depend on real ClickHouse connectivity +- migrations should be safe to run repeatedly +- operators should validate ingest and query paths explicitly after deployment + +## Connection String + +SensApp expects the ClickHouse HTTP interface, not the native TCP port. + +Use a connection string like this: + +```text +clickhouse://default:password@clickhouse:8123/sensapp +``` + +Notes: + +- Port `8123` is the expected HTTP port. +- Port `9000` is the native ClickHouse protocol and is not what the current Rust client path uses. +- SensApp creates the target database if it does not already exist. + +## Recommended Topology + +For a practical deployment: + +- run ClickHouse separately from SensApp +- persist ClickHouse data on durable storage +- run one or more stateless SensApp replicas behind a load balancer +- keep the SensApp container filesystem ephemeral +- use JWT auth if the deployment is not fully private + +## Local Docker Example + +Example ClickHouse container: + +```bash +docker run -d --name sensapp-clickhouse \ + -p 8123:8123 \ + -p 9000:9000 \ + -e CLICKHOUSE_DB=sensapp \ + -e CLICKHOUSE_USER=default \ + -e CLICKHOUSE_PASSWORD=password \ + clickhouse/clickhouse-server:24.8 +``` + +Example SensApp run command: + +```bash +SENSAPP_STORAGE_CONNECTION_STRING=clickhouse://default:password@127.0.0.1:8123/sensapp \ +cargo run --no-default-features --features clickhouse +``` + +Verify readiness: + +```bash +curl http://127.0.0.1:3000/health/ready +``` + +## Helm Example + +For Helm deployments, set the storage connection string to an external ClickHouse service: + +```bash +helm install sensapp ./charts/sensapp \ + --set storage.connectionString=clickhouse://default:password@clickhouse:8123/sensapp \ + --set persistence.enabled=false +``` + +`persistence.enabled=false` is intentional here because ClickHouse should own persistence, not the SensApp pod. + +## Startup And Readiness Expectations + +Current behavior: + +- SensApp runs ClickHouse migrations during startup +- repeated migrations are expected to be safe +- `/health/ready` depends on the storage backend health check +- ClickHouse readiness currently uses a simple `SELECT 1` + +Operational implication: + +- if ClickHouse is down, misconfigured, or unreachable, SensApp should fail readiness even if the HTTP server is alive + +## Post-Deploy Validation + +After deployment, validate the full basic lifecycle. + +### 1. Check readiness + +```bash +curl http://127.0.0.1:3000/health/ready +``` + +### 2. Publish one sample + +```bash +curl -X POST http://127.0.0.1:3000/publish \ + -H 'content-type: text/csv' \ + --data-raw $'datetime,sensor_name,value,unit\n2026-03-16T12:00:00Z,temperature,21.5,C' +``` + +### 3. Query it back + +```bash +curl 'http://127.0.0.1:3000/api/v1/query?query=temperature' +curl 'http://127.0.0.1:3000/series' +curl 'http://127.0.0.1:3000/metrics' +``` + +### 4. Check service metrics + +```bash +curl http://127.0.0.1:3000/prometheus/metrics +``` + +## Operational Caveats + +- SensApp currently targets ClickHouse through the HTTP endpoint only. +- Schema creation is embedded in the binary through the migration SQL file. +- The backend is tested against a real ClickHouse service, but production readiness still depends on good external operations: backups, disk sizing, and ClickHouse monitoring remain the operator's responsibility. +- If you expose SensApp beyond a trusted network, enable JWT auth. + +## Recommended Near-Term Checks + +Before calling a deployment pre-production ready, confirm: + +- repeated restarts do not break migrations +- `/health/ready` turns unhealthy when ClickHouse is unavailable +- publish, list, query, and export endpoints all work against the same ClickHouse instance +- ClickHouse storage and query latency are monitored externally +- backup and restore procedures are documented outside the app runtime diff --git a/docs/JWT_AUTH.md b/docs/JWT_AUTH.md new file mode 100644 index 00000000..d885eb37 --- /dev/null +++ b/docs/JWT_AUTH.md @@ -0,0 +1,121 @@ +# JWT Authentication + +SensApp supports **optional** JWT (JSON Web Token) authentication. By default, all endpoints are open — just like Prometheus. When enabled, JWT tokens control read and write access, and can optionally restrict which sensors a client may interact with. + +## Enabling Authentication + +Set the `SENSAPP_JWT_SECRET` environment variable (or `jwt_secret` in `settings.toml`) to a secret string of **at least 32 characters**: + +```bash +export SENSAPP_JWT_SECRET="my-super-secret-key-at-least-32-characters-long" +``` + +When unset, authentication is disabled and all endpoints remain open. + +## Generating Tokens + +Use the built-in CLI command to generate signed tokens: + +```bash +# Read + write access, 1 hour validity (default) +sensapp generate-token my-service + +# Read-only access, valid for 24 hours +sensapp generate-token prometheus-scraper --scope read --duration 86400 + +# Write-only, restricted to specific sensors +sensapp generate-token edge-device --scope write --sensors "temperature,humidity,pressure" + +# Read + write (explicit) +sensapp generate-token admin --scope readwrite --duration 3600 +``` + +The token is printed to stdout. + +### Options + +| Flag | Description | Default | +|------|-------------|---------| +| `--scope` | `read`, `write`, or `readwrite` | `read write` (both) | +| `--duration` | Token validity in seconds | `3600` (1 hour) | +| `--sensors` | Comma-separated sensor name allow list | all sensors | + +## Using Tokens + +Include the token in the `Authorization` header: + +``` +Authorization: Bearer +``` + +Example with curl: + +```bash +# Write data with a write token +curl -X POST http://localhost:3000/publish \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '[{"n":"temperature","u":"Cel","v":22.5,"t":1700000000}]' + +# Read metrics with a read token +curl http://localhost:3000/metrics \ + -H "Authorization: Bearer $TOKEN" +``` + +## Route Protection + +| Endpoint | Auth Required | Scope | +|----------|--------------|-------| +| `GET /` | No | — | +| `GET /docs` | No | — | +| `GET /health/live` | No | — | +| `GET /health/ready` | No | — | +| `GET /prometheus/metrics` | No | — | +| `GET /metrics` | Yes | `read` | +| `GET /series` | Yes | `read` | +| `GET /series/{uuid}` | Yes | `read` | +| `GET /api/v1/query` | Yes | `read` | +| `POST /api/v1/prometheus_remote_read` | Yes | `read` | +| `POST /publish` | Yes | `write` | +| `POST /api/v2/write` | Yes | `write` | +| `POST /api/v1/prometheus_remote_write` | Yes | `write` | +| `POST /api/v1/admin/vacuum` | Yes | `write` | + +Health checks, documentation, and Prometheus scrape endpoints are always public so that orchestration tools (Kubernetes probes, Prometheus scraper) work without tokens. + +## JWT Claims + +Tokens use the HS256 (HMAC-SHA256) algorithm with the following claims: + +```json +{ + "sub": "my-service", + "exp": 1700003600, + "iat": 1700000000, + "nbf": 1700000000, + "scope": "read write", + "sensors": ["temperature", "humidity"] +} +``` + +| Claim | Required | Description | +|-------|----------|-------------| +| `sub` | Yes | Subject — identifies who/what the token was issued to | +| `exp` | Yes | Expiration time (Unix timestamp). Tokens are rejected after this time | +| `iat` | No | Issued-at time (informational) | +| `nbf` | No | Not-before time. If present, the token is rejected before this time | +| `scope` | No | Space-separated: `"read"`, `"write"`, or `"read write"`. Defaults to `"read write"` | +| `sensors` | No | Array of allowed sensor names. If absent, all sensors are accessible | + +### Time Validation + +- **`exp`** is always required and validated. Expired tokens are rejected with `401 Unauthorized`. +- **`nbf`** is validated when present. Tokens presented before their not-before time are rejected. +- The `jsonwebtoken` library applies a default leeway of 60 seconds for clock skew tolerance. + +## Security Notes + +- The JWT secret must be kept private — anyone with the secret can create valid tokens. +- Use a cryptographically random secret of at least 32 characters. +- For production deployments, consider short-lived tokens and rotate secrets periodically. +- The sensor allow list uses exact string matching on sensor names. diff --git a/docs/PYTHON_SDK.md b/docs/PYTHON_SDK.md new file mode 100644 index 00000000..0a41aacb --- /dev/null +++ b/docs/PYTHON_SDK.md @@ -0,0 +1,96 @@ +# Python SDK + +SensApp now includes a small Python client in `python/sensapp`. + +## Design goals + +- keep the client thin and predictable +- make Apache Arrow the default choice for data exchange +- stay ready for PyPI publication with a standard `pyproject.toml` +- keep examples and tests short enough to copy into real code +- present a clean enough public face for demos and early adopters + +## Install locally + +```bash +cd python/sensapp +uv pip install -e '.[dev]' +uv run ruff check . +uv run ruff format --check . +uv run pytest +``` + +## Install from GitHub + +Before publishing to PyPI, end users can install the package directly from this repository: + +```bash +uv pip install 'git+https://github.com/SINTEF/sensapp.git@main#subdirectory=python/sensapp' +``` + +## Quick example + +```python +import asyncio + +from sensapp import SensAppClient + + +async def main() -> None: + async with SensAppClient.from_env() as client: + await client.publish("temperature", 21.5) + series = await client.query_one("temperature[1h]") + print(series.frame) + + +asyncio.run(main()) +``` + +## Arrow conventions + +The backend currently exposes two Arrow layouts: + +- upload uses a typed per-sensor table with `timestamp` and `value`, plus optional `sensor_id` and `sensor_name` +- download uses streamed Arrow IPC and the client normalizes that into `TimeSeries` objects backed by Polars + +The client keeps upload Arrow explicit and hides download wire-format details behind a compact API. + +Use `query()` when you expect multiple series and `query_one()` when you expect exactly one. + +The package also exposes `__version__` and includes runnable examples in `python/sensapp/examples/`. + +You can upload either simple scalar values, explicit `SamplePoint` lists, or a Polars DataFrame with `timestamp` and `value` columns. + +If you want the frame itself to carry upload metadata, `build_upload_table_from_polars()` also accepts uniform `sensor_name` and optional `sensor_id` columns. + +## What is included + +- `SensAppClient` for the HTTP API +- `SamplePoint` for lightweight upload payloads +- `TimeSeries` for query results with Polars and pandas conversion helpers +- `build_upload_table()` to build Arrow upload tables explicitly +- `build_upload_table_from_polars()` to normalize Polars frames into upload tables +- `serialize_arrow_table()` and `read_arrow_table()` helpers +- mocked unit tests in `python/sensapp/tests` + +## Packaging + +The package uses a normal `pyproject.toml` and can be built with: + +```bash +cd python/sensapp +uv build +``` + +That keeps the path to a future PyPI release straightforward. + +## Quality checks + +The SDK now uses Ruff for both linting and formatting. The intended local loop is: + +```bash +cd python/sensapp +uv run ruff check . +uv run ruff format . +uv run pytest +``` diff --git a/docs/PYTHON_SDK_BUGS.md b/docs/PYTHON_SDK_BUGS.md new file mode 100644 index 00000000..e210117e --- /dev/null +++ b/docs/PYTHON_SDK_BUGS.md @@ -0,0 +1,11 @@ +# Python SDK backend bugs + +No backend bugs have been confirmed by the Python SDK integration work so far. + +If the live SDK tests uncover backend defects, record them here with: + +- reproduction steps +- affected endpoint or format +- observed behavior +- expected behavior +- status or follow-up diff --git a/docs/PYTHON_SDK_DEMO_FINDINGS.md b/docs/PYTHON_SDK_DEMO_FINDINGS.md new file mode 100644 index 00000000..92639430 --- /dev/null +++ b/docs/PYTHON_SDK_DEMO_FINDINGS.md @@ -0,0 +1,113 @@ +# Junior Developer Findings: SensApp Python SDK Demo + +> **Who wrote this:** Me, a junior dev who just joined the project and tried to make something simple with SensApp. +> **Date:** 2026-03-16 +> **Demo code:** `python/demo-dashboard/` + +--- + +## What I built + +A tiny Streamlit dashboard that: +1. Connects to a local SensApp instance +2. Pushes a random temperature reading (18–28 °C) to the `demo_temperature` sensor every second +3. Queries the last 5 minutes of data back and renders it as a live line chart + +Stack: `uv` for project management, `streamlit` for the GUI, `sensapp-sdk` for talking to the API. + +--- + +## How to run it + +```bash +# Make sure SensApp is running on http://127.0.0.1:3000 with a database behind it + +cd python/demo-dashboard +uv run streamlit run app.py --server.port 8502 +# open http://localhost:8502 in your browser +# press ▶ Start and watch the chart fill up +``` + +No manual `pip install`, no `venv activate`, no pain. `uv` is great. + +--- + +## What worked well 👍 + +### The SDK is really clean + +`publish_samples` + `SamplePoint` is all you need to push data. +`query_rows` returns plain Python dicts — perfect for handing straight to `pandas`. +The `with SensAppClient(...) as client:` pattern is nice and explicit. + +### The Arrow-first design is smart + +Even though I never touched Arrow directly, the SDK wraps it so publishing just feels like "here is a list of `(timestamp, value)` pairs". Under the hood it sends efficient binary Arrow frames — you get the performance for free. + +### Streamlit + uv = zero-friction prototype + +`uv add streamlit` and `uv add --editable ../sensapp-sdk` and you are done. +Streamlit's `st.line_chart` needed zero configuration to render the data. + +--- + +## Problems I hit 🐛 + +### 1. Sensor names with hyphens break the query + +I first named my sensor `demo-temperature`. Pushing worked (mostly), but querying it back with `demo-temperature[5m]` returned a 400 error: + +``` +Binary operations (like +, -, *, /) are not supported. +Only simple selectors like 'metric_name{label="value"}' or 'metric_name[5m]' are supported. +``` + +The query parser treats the hyphen as a subtraction operator. I had to rename the sensor to `demo_temperature`. + +**Suggestion:** Either document this restriction clearly in the SDK (a docstring on `publish_samples` / `query_rows` explaining allowed name characters would help), or validate / reject names with hyphens at publish time, or make the parser handle quoted metric names. As a newbie I had no idea that sensor names were also query tokens. + +### 2. Push returns status 500 when the database is not connected + +When PostgreSQL was down I got a generic `500 Internal Server Error` with no body explaining what was wrong. The `/health/ready` endpoint *does* return a useful error message, but `publish` just returns 500. + +**Suggestion:** When the storage backend is unavailable, `/publish` should return a more descriptive error (e.g. 503 with `{"error": "database unavailable"}`) so client code can tell the difference between a bug and a configuration issue. Right now the SDK raises `SensAppHTTPError` with just the status code, which is not very helpful for debugging. + +### 3. No "readiness" check helper in the SDK + +I had to manually call `client.health_ready()` and inspect the dict myself to figure out if the database was up. It would be handy to have a `client.is_ready() -> bool` convenience method, or for `SensAppClient.__init__` to optionally raise if the server is not ready. + +Not a blocker at all, just a nice-to-have for quick scripts. + +### 4. Streamlit session state resets on page reload + +When you hard-refresh the browser, `st.session_state` is wiped. The push counter goes back to 0 and the app shows "pushed 0 samples" even though data is already in the database. This is normal Streamlit behaviour, not a SensApp problem, but it confused me for a second. + +**Not a SensApp issue**, just worth knowing if you are building something production-ish. + +### 5. The `time.sleep(1) + st.rerun()` polling loop is a hack + +Streamlit does not natively support background threads or websocket push, so the "stream every second" is implemented as: + +```python +if st.session_state.running: + time.sleep(1) + st.rerun() +``` + +This blocks the Streamlit server thread for 1 second on every cycle, which means the UI is unresponsive during that sleep. For a one-user demo it is fine. For anything real you would want a background thread or a proper async approach (or use a different tool like Dash/Panel). + +--- + +## What I would do differently next time + +- Use `client.query_arrow(...)` directly and pass the resulting `pyarrow.Table` to `pandas` — skipping `query_rows` avoids the list-of-dicts roundtrip. +- Move `SENSAPP_URL` to an environment variable or a `st.text_input` so it is easier to point at a remote server. +- Use `st.empty()` placeholders to update only the chart widget instead of re-rendering the whole page. + +--- + +## Summary + +The SDK is solid for a v0.1. Publish and query both work exactly as advertised. The main rough edge for a first-time user is the invisible restriction on sensor names — a hyphened name silently pushes (or 500s) but then fails to query, which wastes a lot of debugging time. A small validation or a better error message would make the onboarding experience much smoother. + +Overall: it took me about 30 minutes to go from zero to a live streaming chart. That is a good sign. diff --git a/docs/RRDCACHED.md b/docs/RRDCACHED.md new file mode 100644 index 00000000..9f257dd8 --- /dev/null +++ b/docs/RRDCACHED.md @@ -0,0 +1,157 @@ +# RRDCached Storage Backend + +## Overview + +RRDCached is an experimental storage backend for SensApp that uses [RRDtool's](https://oss.oetiker.ch/rrdtool/) caching daemon for time-series data storage. RRDtool is a mature, high-performance system originally designed for network monitoring that excels at storing and graphing time-series data with fixed-size storage requirements. + +## How it Works + +SensApp integrates with RRDCached through a TCP connection, allowing it to: + +- Create Round Robin Database (RRD) files for each sensor +- Publish numeric sensor data points +- Query historical data using RRD's consolidation functions +- List available sensors based on RRD files + +### Connection String + +```text +rrdcached://host:port?preset= +``` + +Example: + +```text +rrdcached://127.0.0.1:42217?preset=hoarder +``` + +### Available Presets + +- **munin**: Traditional Munin-style data retention (5-minute resolution) +- **hoarder**: High-resolution data retention for detailed analysis + +## Configuration + +To use RRDCached as a storage backend: + +1. Start an RRDCached daemon: + +```bash +rrdcached -l 127.0.0.1:42217 -b /var/lib/rrdcached/db -j /var/lib/rrdcached/journal +``` + +1. Configure SensApp with the RRDCached connection string in `settings.toml`: + +```toml +[database] +connection_string = "rrdcached://127.0.0.1:42217?preset=hoarder" +``` + +## Current Implementation Status + +### Supported Features + +- ✅ **Publishing numeric data** - Integer and Float sensor values +- ✅ **Creating RRD files** - Automatic RRD creation with configurable presets +- ✅ **Querying sensor data** - Fetch historical data with time ranges +- ✅ **Listing sensors** - Basic sensor enumeration based on RRD files + +### Limitations + +The RRDCached backend has significant limitations due to the fundamental design of RRDtool: + +#### Metadata Storage + +RRDtool only stores numeric time-series data. It cannot store: + +- **Sensor names** - Only UUIDs are used as RRD filenames +- **Sensor types** - All data is treated as numeric (Float) +- **Units** - No unit information is preserved +- **Labels/Tags** - Not supported by RRD format +- **Non-numeric types** - String, Boolean, Location, and JSON types cannot be stored + +#### Feature Limitations + +- **Arrow format export/import** - Not supported due to missing metadata +- **Sensor search by name** - Names are not stored, only UUIDs +- **Type preservation** - Integer values become Floats in RRD +- **Round-trip data integrity** - Original sensor metadata cannot be reconstructed + +#### Query Limitations + +- **Recent data** - RRD consolidation may return NaN for very recent data points +- **Exact timestamps** - RRD uses fixed time intervals, timestamps are rounded +- **Raw data access** - Only consolidated data is available after initial retention period + +## Testing + +Integration tests are available but **not enabled by default** due to the limitations described above. To run RRDCached-specific tests: + +```bash +# Ensure RRDCached is running on port 42217 +TEST_DATABASE_URL="rrdcached://127.0.0.1:42217?preset=hoarder" \ + cargo test --no-default-features --features rrdcached --test rrdcached_integration +``` + +Note: Generic storage tests that depend on full metadata support will fail. This includes: + +- Arrow format import/export tests +- Tests requiring sensor name lookups +- Tests using non-numeric data types +- Round-trip data integrity tests + +## Use Cases + +RRDCached is suitable for: + +- **Fixed-size storage** - RRD files don't grow over time +- **Network monitoring** - Traditional RRDtool use case +- **Simple numeric metrics** - Temperature, CPU usage, network traffic +- **Long-term trending** - Automatic data consolidation over time + +RRDCached is **not** suitable for: + +- **Rich sensor metadata** - When names, units, or labels are important +- **Non-numeric data** - String, Boolean, or complex data types +- **Data export/interchange** - When Arrow or other formats are needed +- **Exact data retrieval** - When original resolution must be preserved + +## Future Plans + +The RRDCached backend is currently experimental. Future enhancements being considered include: + +- **External metadata store** - SQLite or JSON sidecar for sensor metadata +- **Hybrid storage** - RRD for numeric data, separate store for metadata +- **Improved query support** - Better handling of consolidation functions +- **Grafana integration** - Direct RRD graph generation + +These enhancements would address the current limitations but require significant architectural changes. Implementation timeline is to be determined based on user needs and project priorities. + +## Technical Details + +### RRD File Structure + +Each sensor creates an RRD file named `{uuid}.rrd` with: + +- **Data Source (DS)**: Single gauge-type data source named "value" +- **Round Robin Archives (RRA)**: Multiple archives with different resolutions + - Munin preset: 5-minute, 30-minute, 2-hour, daily consolidations + - Hoarder preset: 1-second resolution with longer retention + +### Consolidation Functions + +When querying data, RRDCached uses consolidation functions: + +- **AVERAGE**: Mean value over the consolidation period +- **MIN**: Minimum value +- **MAX**: Maximum value +- **LAST**: Most recent value + +### Implementation Notes + +The implementation uses: + +- `rrdcached_client` crate for protocol communication +- Async/await pattern with Tokio runtime +- UUID v7 for time-ordered sensor identifiers +- In-memory tracking of created sensors per session diff --git a/done/.keep b/done/.keep new file mode 100644 index 00000000..e69de29b diff --git a/done/clickhouse-label-query-support.md b/done/clickhouse-label-query-support.md new file mode 100644 index 00000000..0f8cc69d --- /dev/null +++ b/done/clickhouse-label-query-support.md @@ -0,0 +1,22 @@ +# ClickHouse Label Query Support + +## Goal + +Implement `query_sensors_by_labels` for the ClickHouse backend so the core query path works for Prometheus-style label selection. + +## Context + +- ClickHouse is the current reference backend for pre-production hardening. +- The storage trait already requires label-based queries. +- ClickHouse was returning a not-implemented error for that path. + +## Completed Work + +- Added matcher-based sensor discovery for ClickHouse +- Reused the existing per-sensor sample query path for matched sensors +- Added ClickHouse integration coverage for exact-match and `numeric_only` queries +- Fixed the default ClickHouse test URL to use the HTTP port expected by the backend and test services + +## Status + +Completed. diff --git a/done/clickhouse-pagination-and-ci-feature-matrix.md b/done/clickhouse-pagination-and-ci-feature-matrix.md new file mode 100644 index 00000000..3a75ff85 --- /dev/null +++ b/done/clickhouse-pagination-and-ci-feature-matrix.md @@ -0,0 +1,30 @@ +# ClickHouse Pagination And CI Feature Matrix + +## Goal + +Finish ClickHouse cursor pagination and make CI exercise the ClickHouse backend through its explicit feature-gated check path. + +## Context + +- ClickHouse is the current reference backend for pre-production hardening. +- `list_series` was ignoring `limit` and `bookmark` for ClickHouse. +- GitHub Actions only exercised SQLite and PostgreSQL in the main test matrix even though backend-specific cargo-make tasks already existed. + +## Completed Work + +- Implemented real `list_series` cursor pagination for ClickHouse using `sensor_id` bookmarks +- Added ClickHouse integration coverage for pagination behavior and invalid bookmarks +- Added ClickHouse to the GitHub Actions backend matrix with the ClickHouse service and test environment +- Fixed ClickHouse numeric decimal storage to use a supported fixed-scale mapping +- Hardened config and pagination tests so the repo-defined ClickHouse check passes cleanly + +## Validation + +- `cargo test --no-default-features --features clickhouse,test-utils --test clickhouse_integration` +- `TEST_DATABASE_URL=clickhouse://default:password@localhost:8123/sensapp_test cargo test --no-default-features --features clickhouse,test-utils --test crud_dcat_api test_series_pagination_basic -- --nocapture` +- `TEST_DATABASE_URL=clickhouse://default:password@localhost:8123/sensapp_test cargo test --no-default-features --features clickhouse,test-utils --test crud_dcat_api test_series_pagination_with_bookmark -- --nocapture` +- `cargo make check-clickhouse` + +## Status + +Completed. diff --git a/done/containerization-and-helm.md b/done/containerization-and-helm.md new file mode 100644 index 00000000..29ac1c84 --- /dev/null +++ b/done/containerization-and-helm.md @@ -0,0 +1,22 @@ +# Containerization and Helm Packaging + +## Goal + +Add a production-oriented Docker image, a reusable Helm chart, and CI validation for both. + +## Scope + +- Create a multi-stage Dockerfile for the SensApp server +- Add a Helm chart for Kubernetes deployment +- Extend GitHub Actions to validate and build container artifacts +- Keep the published image focused on practical deployment backends + +## Notes + +- Primary image target: self-hosted deployments on Kubernetes and Docker +- Compile-time feature matrix is broader than the primary runtime image +- Helm should configure runtime behavior, not Rust compile-time features + +## Status + +Completed. diff --git a/done/cross-backend-pagination-and-ci-expansion.md b/done/cross-backend-pagination-and-ci-expansion.md new file mode 100644 index 00000000..2e5d2587 --- /dev/null +++ b/done/cross-backend-pagination-and-ci-expansion.md @@ -0,0 +1,32 @@ +# Cross-Backend Pagination And CI Expansion + +## Goal + +Bring `list_series` pagination semantics in line across the maintained backends and expand the explicit CI backend matrix beyond ClickHouse. + +## Context + +- ClickHouse now uses proper cursor pagination semantics. +- SQLite still ignores `limit` and `bookmark`. +- PostgreSQL and DuckDB still use the old `len == limit` heuristic for next-page detection. +- GitHub Actions only exercises a subset of the backend-specific check tasks that already exist in `cargo-make`. + +## Scope + +- Implement proper `list_series` pagination for SQLite +- Fix next-page detection in PostgreSQL and DuckDB +- Make the generic pagination tests backend-agnostic where needed +- Extend the CI backend matrix to additional supported backends + +## Outcome + +- SQLite now implements cursor pagination for `list_series` +- PostgreSQL, DuckDB, and TimescaleDB now use `limit + 1` next-page detection +- TimescaleDB now implements label-based queries needed by Prometheus remote read +- TimescaleDB timestamp conversion now round-trips at the intended microsecond precision +- CI now exercises `duckdb` and `timescaledb`, including a dedicated TimescaleDB migration step +- Local PostgreSQL 18 test compose configuration now uses a compatible data mount path + +## Status + +Completed. \ No newline at end of file diff --git a/done/last-sample-and-availability-api.md b/done/last-sample-and-availability-api.md new file mode 100644 index 00000000..ce80ba42 --- /dev/null +++ b/done/last-sample-and-availability-api.md @@ -0,0 +1,65 @@ +# Last Sample And Availability API + +## Goal + +Add the two follow-up APIs identified after the series downsampling work: + +- a dedicated last-sample endpoint for a single series +- a time-window availability endpoint exposing existence and optional bucket coverage + +## Delivered + +- added `GET /series/{series_uuid}/last` +- added `GET /series/{series_uuid}/availability` +- added a generic storage fallback for latest-sample retrieval +- added endpoint tests covering latest sample, bounded latest sample, and availability coverage +- cleaned up repository-wide clippy blockers encountered during validation + +## API Shape + +### Last sample + +`GET /series/{series_uuid}/last` + +Optional query parameters: + +- `start` +- `end` + +Response includes: + +- `series_uuid` +- `sensor_name` +- `sensor_type` +- `unit` +- `timestamp` +- `value` + +### Availability + +`GET /series/{series_uuid}/availability` + +Required query parameters: + +- `start` +- `end` + +Optional query parameter: + +- `step` + +Response includes: + +- `present` +- `sample_count` +- `first_sample_at` +- `last_sample_at` +- optional `covered_buckets` +- optional `total_buckets` +- optional `coverage_ratio` + +## Validation + +- `cargo check` +- `cargo test --test query_export` +- `cargo clippy --tests -- -D warnings` diff --git a/done/last-sample-and-availability-fast-paths.md b/done/last-sample-and-availability-fast-paths.md new file mode 100644 index 00000000..07338b2a --- /dev/null +++ b/done/last-sample-and-availability-fast-paths.md @@ -0,0 +1,17 @@ +# Last Sample And Availability Fast Paths + +## Goal + +Replace the generic full-series fallback used by the new last-sample and availability APIs with backend-native query paths. + +## Scope + +- add storage-native latest-sample queries +- add storage-native availability summary queries +- wire the HTTP availability endpoint to the summary API +- cover backend behavior with targeted tests + +## Notes + +- keep the generic fallback in the trait as a safety net +- prioritize the same backends that already have native advanced-query pushdown diff --git a/done/merge-stale-branches-march-2026.md b/done/merge-stale-branches-march-2026.md new file mode 100644 index 00000000..a0a9a87e --- /dev/null +++ b/done/merge-stale-branches-march-2026.md @@ -0,0 +1,60 @@ +# Merge Stale Remote Branches (March 2026) + +## Summary + +Analyzed and resolved three stale remote branches: `rrdcached`, `vibe-coding-2`, and `sprint-antoine`. + +## Branch: rrdcached (MERGED) + +**2 commits ahead of main, 0 behind.** + +Changes merged: +- Complete rrdcached storage backend implementation (`src/storage/rrdcached/mod.rs`) + - TCP and Unix socket client abstraction + - Create, batch insert, flush operations + - List series via `LIST` command + - Basic fetch support (WIP) +- RRDCached documentation (`docs/RRDCACHED.md`) +- Integration test framework (`tests/rrdcached_integration.rs`) +- Feature-gated behind `rrdcached` cargo feature + +Conflicts resolved: `Cargo.toml` and `Cargo.lock` (dependency versions). + +## Branch: vibe-coding-2 (MERGED) + +**10 commits ahead, 5 behind main.** + +Changes merged: +- **Directory restructure**: `src/ingestors/http/` → `src/http/` (simplification) +- **CRUD DCAT API**: New comprehensive CRUD endpoints with format negotiation (SenML, CSV, JSONL, Arrow) +- **Pagination**: `ListSeriesResult` with cursor-based pagination (`list_series` now supports `limit` + `bookmark`) +- **Storage trait update**: All backends updated for new `list_series` signature +- **PostgreSQL improvements**: Labels fetched via catalog view (JOIN in SQL) instead of N+1 queries +- **PromQL label selectors**: Parsing `{env="prod",region=~"us.*"}` style selectors +- **Metrics endpoints**: Summary views for sensor metadata +- **CRUD DCAT API tests** (`tests/crud_dcat_api.rs`) + +Conflicts resolved: 17 files including storage backends, HTTP routes, Cargo.toml. + +## Branch: sprint-antoine (NOT MERGED — too divergent) + +**10 commits ahead, 61 behind main.** + +This branch represents an older development path that was superseded by main's more recent work. Merging would have: +- Deleted ALL 17 test files +- Deleted all exporters (arrow, csv, jsonl, senml) +- Deleted the modern Prometheus read/write implementation +- Reverted the `src/http` rename back to `src/ingestors/http` +- Removed query.rs matchers, clickhouse support, and recent migration files + +**Valuable ideas extracted** (moved to `ideas/`): +- MQTT ingestor integration +- OPC-UA ingestor +- Event bus system +- Geobuf parsing support + +## Verification + +- `cargo check` — passes (1 warning: unused function `publish_prometheus`) +- `cargo clippy --tests` — clean (same 1 warning) +- `cargo test` — 169 passed, 3 failed (PostgreSQL connection timeouts, expected without a running database) diff --git a/done/prometheus-metrics-endpoint.md b/done/prometheus-metrics-endpoint.md new file mode 100644 index 00000000..898489ad --- /dev/null +++ b/done/prometheus-metrics-endpoint.md @@ -0,0 +1,22 @@ +# Add Prometheus Metrics Endpoint + +## Context + +SensApp already exposes API and health endpoints, but it does not expose Prometheus-compatible service metrics for scraping. + +## Goal + +1. Add a Prometheus scrape endpoint. +2. Expose a small set of useful runtime metrics. +3. Keep the implementation lightweight and aligned with the existing Axum server. + +## Outcome + +1. Added a Prometheus scrape endpoint at `/prometheus/metrics`. +2. Kept the existing DCAT metrics catalog endpoint at `/metrics`. +3. Exposed lightweight runtime metrics for HTTP request counts, request duration, in-flight requests, uptime, and storage readiness. +4. Added coverage for the new endpoint and updated existing tests for the catalog route change. + +## Notes + +- The implementation uses the `prometheus-client` crate instead of generating the text format manually. diff --git a/done/prometheus-remote-read-large-range-bug.md b/done/prometheus-remote-read-large-range-bug.md new file mode 100644 index 00000000..afe27507 --- /dev/null +++ b/done/prometheus-remote-read-large-range-bug.md @@ -0,0 +1,35 @@ +# Prometheus remote read large range bug + +## Goal + +Fix the Prometheus remote read streamed XOR response path so large query windows keep returning data instead of silently producing an empty stream. + +## Context + +- Storage queries succeeded and returned the expected sensor plus samples. +- The failure appeared only after the storage result was converted into streamed XOR chunks. +- Logs showed a working path around 40k samples and an empty response around 77k samples. + +## Root cause + +- The underlying XOR chunk encoder writes the per-chunk sample count as a big-endian `u16`. +- SensApp tried to encode an entire series as a single XOR chunk. +- Once a query returned more than `65535` samples, encoding failed with `too many samples for one chunk` and the handler silently skipped the series. + +## Fix + +- Split large streamed remote-read sample sets into multiple XOR chunks, each capped at `u16::MAX` samples. +- Keep a single `ChunkedSeries` per sensor while allowing multiple chunks inside it. +- Log chunk encoding failures with context instead of silently swallowing them. + +## Tests + +- Added a unit test for encoder chunk splitting beyond `65535` samples. +- Added an integration test covering `STREAMED_XOR_CHUNKS` for a `70000`-sample series. + +## Validation + +- `DUCKDB_DOWNLOAD_LIB=1 cargo test test_encode_series_splits_large_sample_sets`: passed +- `DUCKDB_DOWNLOAD_LIB=1 cargo test test_remote_read_streamed_xor_chunks_large_series`: passed +- `DUCKDB_DOWNLOAD_LIB=1 cargo check`: passed +- `DUCKDB_DOWNLOAD_LIB=1 cargo clippy --tests`: passed diff --git a/done/prometheus-roleplay-read-demo.md b/done/prometheus-roleplay-read-demo.md new file mode 100644 index 00000000..3a795e99 --- /dev/null +++ b/done/prometheus-roleplay-read-demo.md @@ -0,0 +1,25 @@ +# Prometheus roleplay read demo + +## Goal + +Create a small Docker Compose stack that runs SensApp against the existing roleplay DuckDB file and configures Prometheus to read historical series from SensApp through the Prometheus remote read API. + +## Constraints + +- Use the same effective runtime settings as the local DuckDB command used by the roleplay notebooks. +- Do not depend on an InfluxDB query API because SensApp only implements the InfluxDB write API. +- Keep the setup simple enough for local testing and manual querying. + +## Outcome + +- Added `compose.prometheus-roleplay.yml` to run a DuckDB-only SensApp service and a Prometheus service together. +- Added `docker/prometheus-roleplay/prometheus.yml` with a `remote_read` target pointing at SensApp. +- Updated the roleplay README with the stack startup command and basic checks. +- Verified the stack locally by querying the known `roleplay_zeb_*` metrics through Prometheus. + +## Validation + +- `docker compose -f compose.prometheus-roleplay.yml up --build -d`: passed +- `cargo check`: passed when run with `CARGO_TARGET_DIR=/tmp/sensapp-target-validate` +- `cargo clippy --tests`: passed when run with `CARGO_TARGET_DIR=/tmp/sensapp-target-validate` +- `cargo test`: failed due existing PostgreSQL pool timeouts in `http::influxdb::tests::test_publish_influxdb`, `http::influxdb::tests::test_publish_influxdb_with_numeric_enabled`, and `http::server::tests::test_frontpage_handler` diff --git a/done/python-sdk-client.md b/done/python-sdk-client.md new file mode 100644 index 00000000..bf18caf0 --- /dev/null +++ b/done/python-sdk-client.md @@ -0,0 +1,18 @@ +# Python SDK client + +## Goal + +Create a Python SDK for SensApp that feels natural to use from Python code, prefers Apache Arrow for data exchange, and is ready to evolve into a PyPI package. + +## Scope + +- standalone Python package inside this repository +- sync client for the current HTTP API +- Arrow-first helpers for upload and download +- tests with mocked HTTP transport and Arrow round-trips +- concise package documentation and simple examples + +## Notes + +- SensApp uses different Arrow schemas for upload and download, so the SDK should make that explicit instead of hiding it. +- Keep dependencies small and the public API boring and predictable. diff --git a/done/python-sdk-hardening.md b/done/python-sdk-hardening.md new file mode 100644 index 00000000..03e1ec31 --- /dev/null +++ b/done/python-sdk-hardening.md @@ -0,0 +1,17 @@ +# Python SDK hardening + +## Goal + +Strengthen the Python SDK so it is closer to a trustworthy PyPI package. + +## Scope + +- expand unit coverage for edge cases and request handling +- add live integration tests against a real SensApp process with SQLite +- add a dedicated GitHub workflow for linting, unit tests, integration tests, and wheel smoke tests +- document backend bugs in a repository file if the integration work exposes them + +## Notes + +- Prefer catching backend regressions with integration tests instead of compensating for them in the client. +- Keep the integration setup self-contained and SQLite-based so CI stays fast and deterministic. \ No newline at end of file diff --git a/done/python-sdk-safe-sensor-query-helpers.md b/done/python-sdk-safe-sensor-query-helpers.md new file mode 100644 index 00000000..ae0757dc --- /dev/null +++ b/done/python-sdk-safe-sensor-query-helpers.md @@ -0,0 +1,26 @@ +# Python SDK Safe Sensor Query Helpers + +## Goal + +- let SDK callers query by sensor name without hand-writing PromQL +- support hyphenated sensor names safely by generating `__name__` matchers +- cover the behavior with unit and live integration tests + +## Delivered + +- added structured sensor-name query helpers to `SensAppClient` +- generated safe `__name__="..."` selectors so hyphenated sensor names work without raw PromQL +- added unit tests for helper query construction +- added a live integration test covering a hyphenated sensor name against a real SensApp process +- updated the demo dashboard to use the helper API instead of interpolating raw PromQL +- updated the SDK README quick start and API list + +## Validation + +- `cd python/sensapp-sdk && uv run python -m pytest tests/test_client.py` +- `cd python/sensapp-sdk && uv run python -m pytest -m integration` +- `cd python/sensapp-sdk && uv run python -m pytest` + +## Notes + +- the live integration tests already start SensApp through `cargo run`, so extra Docker orchestration was not required for this fix diff --git a/done/query-feedback-and-storage-errors.md b/done/query-feedback-and-storage-errors.md new file mode 100644 index 00000000..e346c185 --- /dev/null +++ b/done/query-feedback-and-storage-errors.md @@ -0,0 +1,27 @@ +# Query Feedback And Storage Errors + +## Goal + +- keep hyphenated sensor names allowed +- make simple PromQL failures explain why hyphenated bare metric names break +- report storage backend unavailability more clearly during ingestion + +## Delivered + +- preserved hyphenated sensor names during ingestion and added coverage for that behavior +- improved simple PromQL binary-operation errors with a specific hint for hyphenated metric names +- documented the `__name__` matcher workaround in the Python SDK README +- preserved storage errors when they are wrapped in `anyhow` +- mapped backend-unavailable storage failures to HTTP `503` with a clear `Database unavailable` response + +## Validation + +- `cargo fmt` +- `cargo check` +- `cargo test --test simple_promql --test ingestion` +- `cargo test internal_server_error_preserves_storage_error_category` +- `cargo clippy --tests -- -D warnings` + +## Notes + +- existing `dead_code` warnings in `src/http/auth.rs` are still present during `cargo check` and `cargo test`, but `cargo clippy --tests -- -D warnings` passed unchanged diff --git a/done/roleplay-datascience-notebooks.md b/done/roleplay-datascience-notebooks.md new file mode 100644 index 00000000..59ed5726 --- /dev/null +++ b/done/roleplay-datascience-notebooks.md @@ -0,0 +1,19 @@ +# Roleplay Data Science Notebooks + +## Goal + +Create a beginner-friendly, notebook-based workflow that uploads the new parquet files to a local SensApp instance and then queries the uploaded data back for plotting and lightweight analysis. + +## Scope + +- run SensApp locally with DuckDB on a non-default port +- use `uv` for the Python environment in `python/roleplay-datascience` +- use the in-repo `sensapp-sdk` as an editable dependency +- create one notebook for upload and one notebook for analysis +- validate the workflow end to end and note issues or suggestions + +## Notes + +- The parquet files are large enough that upload chunking is likely required under the default HTTP body limit. +- Sensor names should avoid hyphens because the current simple query parser treats them as subtraction. +- Completed on 2026-03-17 with a validated DuckDB-backed workflow on port `3017`. diff --git a/done/rrdcached-backend-completion.md b/done/rrdcached-backend-completion.md new file mode 100644 index 00000000..8f9bc6f2 --- /dev/null +++ b/done/rrdcached-backend-completion.md @@ -0,0 +1,27 @@ +# Complete RRDCached Storage Backend + +## Summary + +The `rrdcached` branch work is now complete enough for the backend's intended experimental scope. + +## Completed Work + +- Added TCP and Unix socket client support +- Implemented create, batch insert, flush, fetch, and paginated `LIST` support +- Implemented label-based queries through `query_sensors_by_labels` +- Hardened reconnectable client error handling +- Added dedicated integration tests against real RRDCached instances +- Added explicit CI coverage for the `rrdcached` feature path +- Documented the backend and its constraints in `docs/RRDCACHED.md` + +## Final Assessment + +RRDCached should no longer be treated as an active delivery task. It works for its intended experimental use case, but it is not a candidate for the generic backend matrix because persistent `.rrd` files and limited metadata support do not satisfy the assumptions used by the shared storage tests. + +The remaining items are not blockers for closing the task. They are follow-up ideas that only matter if RRDCached is promoted beyond its current experimental role. + +## References + +- Docs: `docs/RRDCACHED.md` +- Tests: `tests/rrdcached_integration.rs` +- Code: `src/storage/rrdcached/mod.rs` diff --git a/done/series-downsampling-and-simplify.md b/done/series-downsampling-and-simplify.md new file mode 100644 index 00000000..612296e6 --- /dev/null +++ b/done/series-downsampling-and-simplify.md @@ -0,0 +1,27 @@ +# Series Downsampling And Simplify + +## Goal + +Add explicit query-time downsampling and simplification for series reads. + +## Delivered + +- Extended `GET /series/{series_uuid}` with explicit query parameters for temporal bucketing, aggregation selection, and explicit simplify enablement. +- Added shared query option types and Rust-side fallback processing for aggregation and simplify. +- Added Prometheus-style duration parsing for `step` values. +- Added native aggregation pushdown for PostgreSQL, SQLite, TimescaleDB, DuckDB, and ClickHouse. +- Kept simplify as a Rust-side post-processing step across all backends so the API semantics stay consistent. + +## Validation + +- `cargo check` +- `cargo check --features "sqlite timescaledb clickhouse"` +- `cargo check --features "duckdb"` +- `cargo test --test query_export advanced_series_query_tests` +- `cargo test --test advanced_backend_queries --features "sqlite timescaledb clickhouse"` +- `cargo test --test advanced_backend_queries --features "duckdb"` + +## Follow-Ups + +- Consider a dedicated last-sample API or a storage-level fast path for `aggregation=last` without requiring a caller-selected bucket. +- Consider an availability/existence API for a time window, returning either boolean presence or window coverage ratio. diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 00000000..a547bf36 --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1,24 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/frontend/README.md b/frontend/README.md new file mode 100644 index 00000000..4dcad1f9 --- /dev/null +++ b/frontend/README.md @@ -0,0 +1,73 @@ +# React + TypeScript + Vite + +This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules. + +Currently, two official plugins are available: + +- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Babel](https://babeljs.io/) (or [oxc](https://oxc.rs) when used in [rolldown-vite](https://vite.dev/guide/rolldown)) for Fast Refresh +- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh + +## React Compiler + +The React Compiler is currently not compatible with SWC. See [this issue](https://github.com/vitejs/vite-plugin-react/issues/428) for tracking the progress. + +## Expanding the ESLint configuration + +If you are developing a production application, we recommend updating the configuration to enable type-aware lint rules: + +```js +export default defineConfig([ + globalIgnores(['dist']), + { + files: ['**/*.{ts,tsx}'], + extends: [ + // Other configs... + + // Remove tseslint.configs.recommended and replace with this + tseslint.configs.recommendedTypeChecked, + // Alternatively, use this for stricter rules + tseslint.configs.strictTypeChecked, + // Optionally, add this for stylistic rules + tseslint.configs.stylisticTypeChecked, + + // Other configs... + ], + languageOptions: { + parserOptions: { + project: ['./tsconfig.node.json', './tsconfig.app.json'], + tsconfigRootDir: import.meta.dirname, + }, + // other options... + }, + }, +]) +``` + +You can also install [eslint-plugin-react-x](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-x) and [eslint-plugin-react-dom](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-dom) for React-specific lint rules: + +```js +// eslint.config.js +import reactX from 'eslint-plugin-react-x' +import reactDom from 'eslint-plugin-react-dom' + +export default defineConfig([ + globalIgnores(['dist']), + { + files: ['**/*.{ts,tsx}'], + extends: [ + // Other configs... + // Enable lint rules for React + reactX.configs['recommended-typescript'], + // Enable lint rules for React DOM + reactDom.configs.recommended, + ], + languageOptions: { + parserOptions: { + project: ['./tsconfig.node.json', './tsconfig.app.json'], + tsconfigRootDir: import.meta.dirname, + }, + // other options... + }, + }, +]) +``` diff --git a/frontend/eslint.config.js b/frontend/eslint.config.js new file mode 100644 index 00000000..dec9d6a9 --- /dev/null +++ b/frontend/eslint.config.js @@ -0,0 +1,23 @@ +import js from '@eslint/js' +import globals from 'globals' +import reactHooks from 'eslint-plugin-react-hooks' +import reactRefresh from 'eslint-plugin-react-refresh' +import tseslint from 'typescript-eslint' +import { defineConfig, globalIgnores } from 'eslint/config' + +export default defineConfig([ + globalIgnores(['dist', 'src/client']), + { + files: ['**/*.{ts,tsx}'], + extends: [ + js.configs.recommended, + tseslint.configs.recommended, + reactHooks.configs.flat.recommended, + reactRefresh.configs.vite, + ], + languageOptions: { + ecmaVersion: 2020, + globals: globals.browser, + }, + }, +]) diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 00000000..592cc7f1 --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,13 @@ + + + + + + + SensApp + + +
+ + + diff --git a/frontend/openapi-ts.config.ts b/frontend/openapi-ts.config.ts new file mode 100644 index 00000000..ddbcb2e8 --- /dev/null +++ b/frontend/openapi-ts.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from '@hey-api/openapi-ts'; + +export default defineConfig({ + input: { + path: './openapi.json', + }, + output: 'src/client', + plugins: [ + '@hey-api/client-fetch', + '@tanstack/react-query', + ], +}); diff --git a/frontend/openapi.json b/frontend/openapi.json new file mode 100644 index 00000000..3707078d --- /dev/null +++ b/frontend/openapi.json @@ -0,0 +1 @@ +{"openapi":"3.1.0","info":{"title":"sensapp","description":"","license":{"name":""},"version":"0.3.0"},"paths":{"/":{"get":{"tags":["SensApp"],"operationId":"frontpage","responses":{"200":{"description":"SensApp Frontpage","content":{"text/plain":{"schema":{"type":"string"}}}}}}},"/api/v1/admin/vacuum":{"post":{"tags":["Admin"],"summary":"Database Vacuuming","description":"Cleans up and optimizes the database by removing unused data and reclaiming space.\n(only if supported by the underlying storage engine).","operationId":"vacuum_database","responses":{"200":{"description":"Database vacuum completed successfully","content":{"text/plain":{"schema":{"type":"string"}}}},"500":{"description":"Failed to vacuum database","content":{"text/plain":{"schema":{"type":"string"}}}}}}},"/api/v1/prometheus_remote_read":{"post":{"tags":["Prometheus"],"summary":"Prometheus Remote Read API.","description":"Allows you to read data from SensApp using Prometheus remote read protocol.\n\nIt follows the [Prometheus Remote Read specification](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/).","operationId":"prometheus_remote_read","parameters":[{"name":"content-encoding","in":"header","description":"Content encoding, must be snappy","required":true,"schema":{"type":"string","format":"snappy"}},{"name":"content-type","in":"header","description":"Content type, must be application/x-protobuf","required":true,"schema":{"type":"string","format":"application/x-protobuf"}},{"name":"x-prometheus-remote-read-version","in":"header","description":"Prometheus Remote Read version, must be 0.1.0","required":true,"schema":{"type":"string","format":"0.1.0"}}],"requestBody":{"description":"Prometheus Remote Read endpoint. [Reference](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/)","content":{"application/x-protobuf":{}}},"responses":{"200":{"description":"Read Response","content":{"application/x-protobuf":{}}},"400":{"description":"Bad Request","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AppError"}}}},"500":{"description":"Internal Server Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AppError"}}}}}}},"/api/v1/prometheus_remote_write":{"post":{"tags":["Prometheus"],"summary":"Prometheus Remote Write API.","description":"Allows you to write data from Prometheus to SensApp.\n\nIt follows the [Prometheus Remote Write specification](https://prometheus.io/docs/concepts/remote_write_spec/).","operationId":"publish_prometheus","parameters":[{"name":"content-encoding","in":"header","description":"Content encoding, must be snappy","required":true,"schema":{"type":"string","format":"snappy"}},{"name":"content-type","in":"header","description":"Content type, must be application/x-protobuf","required":true,"schema":{"type":"string","format":"application/x-protobuf"}},{"name":"x-prometheus-remote-write-version","in":"header","description":"Prometheus Remote Write version, must be 0.1.0","required":true,"schema":{"type":"string","format":"0.1.0"}}],"requestBody":{"description":"Prometheus Remote Write endpoint. [Reference](https://prometheus.io/docs/concepts/remote_write_spec/)","content":{"application/x-protobuf":{}}},"responses":{"204":{"description":"No Content"},"400":{"description":"Bad Request","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AppError"}}}},"500":{"description":"Internal Server Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AppError"}}}}}}},"/api/v1/query":{"get":{"tags":["SensApp"],"summary":"Simple PromQL query endpoint.","description":"Parses a PromQL expression and returns matching time series data.\nOnly simple selectors (VectorSelector and MatrixSelector) are supported.\n\n# Examples\n\n- Simple metric: `GET /api/v1/query?query=my_metric`\n- With labels: `GET /api/v1/query?query=my_metric{env=\"prod\"}`\n- Range query: `GET /api/v1/query?query=my_metric[5m]`\n- With format: `GET /api/v1/query?query=my_metric&format=csv`","operationId":"simple_promql_query","parameters":[{"name":"query","in":"query","description":"PromQL query string (e.g., 'my_metric{label=\"value\"}' or 'my_metric[5m]')","required":true,"schema":{"type":"string"}},{"name":"format","in":"query","description":"Output format: senml (default), csv, jsonl, or arrow","required":false,"schema":{"type":"string"}}],"responses":{"200":{"description":"Query results in requested format","content":{"application/json":{"schema":{}}}},"400":{"description":"Invalid or unsupported PromQL query"},"500":{"description":"Internal server error"}}}},"/api/v2/write":{"post":{"tags":["InfluxDB"],"summary":"InfluxDB Compatible Write API.","description":"Allows you to write data from InfluxDB or Telegraf to SensApp.\n[More information.](https://github.com/SINTEF/sensapp/blob/main/docs/INFLUX_DB.md)","operationId":"publish_influxdb","parameters":[{"name":"bucket","in":"query","description":"Bucket name","required":true,"schema":{"type":"string"},"example":"sensapp"},{"name":"org","in":"query","description":"Organization name","required":false,"schema":{"type":"string"},"example":"sensapp"},{"name":"org_id","in":"query","description":"Organization ID","required":false,"schema":{"type":"string"}},{"name":"precision","in":"query","description":"Precision of the timestamps. One of ns, us, ms, s","required":false,"schema":{"type":"string"}}],"requestBody":{"description":"InfluxDB Line Protocol endpoint. [Reference](https://docs.influxdata.com/influxdb/v2/reference/syntax/line-protocol/).","content":{"text/plain":{"schema":{"type":"string"},"example":"cpu,host=A,region=west usage_system=64.2 1590488773254420000"}},"required":true},"responses":{"204":{"description":"No Content"},"400":{"description":"Bad Request","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AppError"}}}},"500":{"description":"Internal Server Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AppError"}}}}}}},"/health/live":{"get":{"tags":["Health"],"summary":"Liveness check","description":"Checks if the service is running.\nThis endpoint always returns 200 OK if the server is able to respond.","operationId":"liveness","responses":{"200":{"description":"Service is alive","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HealthResponse"}}}}}}},"/health/ready":{"get":{"tags":["Health"],"summary":"Readiness check","description":"Checks if the service is ready to accept traffic.\nThis includes checking if the database connection is working","operationId":"readiness","responses":{"200":{"description":"Service is ready","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReadinessResponse"}}}},"503":{"description":"Service is not ready","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReadinessResponse"}}}}}}},"/metrics":{"get":{"tags":["SensApp"],"summary":"List unique metrics (measurement types) with aggregated information in DCAT catalog format.","operationId":"list_metrics","parameters":[{"name":"name","in":"query","description":"Filter metrics by name (substring match)","required":false,"schema":{"type":"string"}},{"name":"name_regex","in":"query","description":"Filter metrics by name using regex pattern","required":false,"schema":{"type":"string"}},{"name":"type","in":"query","description":"Filter metrics by sensor type (float, integer, string, boolean, location, json, blob, numeric)","required":false,"schema":{"type":"string"}}],"responses":{"200":{"description":"Metrics catalog in DCAT format","content":{"application/json":{"schema":{}}}}}}},"/prometheus/metrics":{"get":{"tags":["Observability"],"operationId":"prometheus_metrics","responses":{"200":{"description":"Prometheus-compatible metrics","content":{"text/plain":{"schema":{"type":"string"}}}}}}},"/publish":{"post":{"tags":["SensApp"],"summary":"SensApp native data ingestion API supporting multiple formats.","description":"Accepts sensor data in one of the following formats:\n- **SenML JSON** (RFC 8428): `Content-Type: application/json`\n- **CSV**: `Content-Type: text/csv` or `application/csv`\n- **Apache Arrow IPC**: `Content-Type: application/vnd.apache.arrow.file`\n\nIf no Content-Type header is provided, defaults to CSV format.","operationId":"publish_sensors_data","requestBody":{"description":"Sensor data in SenML JSON, CSV, or Apache Arrow format","content":{"text/plain":{"schema":{"type":"string"}}},"required":true},"responses":{"200":{"description":"Data ingested successfully","content":{"text/plain":{"schema":{"type":"string"}}}},"400":{"description":"Bad Request - invalid data format","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AppError"}}}},"500":{"description":"Internal Server Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AppError"}}}}}}},"/series":{"get":{"tags":["SensApp"],"summary":"List all series (time series) in DCAT catalog format.","operationId":"list_series","parameters":[{"name":"metric","in":"query","description":"Filter series by metric name","required":false,"schema":{"type":"string"}},{"name":"selector","in":"query","description":"PromQL-style label selector (e.g., '{env=\"prod\",region=~\"us.*\"}')","required":false,"schema":{"type":"string"}}],"responses":{"200":{"description":"Time series catalog in DCAT format","content":{"application/json":{"schema":{}}}}}}},"/series/{series_uuid}":{"get":{"tags":["SensApp"],"summary":"Get series data in various formats based on query parameter.","operationId":"get_series_data","parameters":[{"name":"series_uuid","in":"path","description":"UUID of the series","required":true,"schema":{"type":"string"}},{"name":"format","in":"query","description":"Output format: senml, csv, or jsonl (default: senml)","required":false,"schema":{"type":"string"}},{"name":"start","in":"query","description":"Start datetime in ISO 8601 format (e.g., '2024-01-15T10:30:00Z')","required":false,"schema":{"type":"string"}},{"name":"end","in":"query","description":"End datetime in ISO 8601 format (e.g., '2024-01-15T11:00:00Z')","required":false,"schema":{"type":"string"}},{"name":"limit","in":"query","description":"Maximum number of samples (default: 10,000,000)","required":false,"schema":{"type":"integer","minimum":0}}],"responses":{"200":{"description":"Series data in requested format","content":{"application/json":{"schema":{}}}},"400":{"description":"Invalid format"},"404":{"description":"Series not found"}}}}},"components":{"schemas":{"AppError":{"oneOf":[{"type":"object","required":["InternalServerError"],"properties":{"InternalServerError":{"type":"string"}},"example":"Internal Server Error"},{"type":"object","required":["BadRequest"],"properties":{"BadRequest":{"type":"string"}},"example":"Bad Request"},{"type":"object","required":["NotFound"],"properties":{"NotFound":{"type":"string"}},"example":"Not Found"},{"type":"object","required":["Unauthorized"],"properties":{"Unauthorized":{"type":"string"}},"example":"Unauthorized"},{"type":"object","required":["Forbidden"],"properties":{"Forbidden":{"type":"string"}},"example":"Forbidden"},{"type":"object","required":["Storage"],"properties":{"Storage":{"type":"string"}},"example":"Storage Error"}]},"HealthResponse":{"type":"object","required":["status"],"properties":{"status":{"type":"string"}}},"ReadinessResponse":{"type":"object","required":["status","database"],"properties":{"database":{"type":"string"},"error":{"type":["string","null"]},"status":{"type":"string"}}}}},"tags":[{"name":"SensApp","description":"SensApp API"},{"name":"InfluxDB","description":"InfluxDB Write API"},{"name":"Observability","description":"Prometheus-compatible service metrics"},{"name":"Prometheus","description":"Prometheus Remote Write and Read API"},{"name":"Admin","description":"Administrative operations"},{"name":"Health","description":"Health check endpoints"}]} diff --git a/frontend/package-lock.json b/frontend/package-lock.json new file mode 100644 index 00000000..1908b7e2 --- /dev/null +++ b/frontend/package-lock.json @@ -0,0 +1,5839 @@ +{ + "name": "frontend", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "frontend", + "version": "0.0.0", + "dependencies": { + "@fontsource/roboto": "^5.2.10", + "@hey-api/client-fetch": "^0.13.1", + "@tailwindcss/vite": "^4.2.1", + "@tanstack/react-query": "^5.90.21", + "echarts": "^6.0.0", + "echarts-for-react": "^3.0.6", + "react": "^19.2.0", + "react-dom": "^19.2.0", + "react-router-dom": "^7.13.1", + "tailwindcss": "^4.2.1", + "zustand": "^5.0.11" + }, + "devDependencies": { + "@eslint/js": "^9.39.1", + "@hey-api/openapi-ts": "^0.94.0", + "@tanstack/react-query-devtools": "^5.91.3", + "@testing-library/jest-dom": "^6.9.1", + "@testing-library/react": "^16.3.2", + "@testing-library/user-event": "^14.6.1", + "@types/node": "^24.10.1", + "@types/react": "^19.2.7", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react-swc": "^4.2.2", + "daisyui": "^5.5.19", + "eslint": "^9.39.1", + "eslint-plugin-react-hooks": "^7.0.1", + "eslint-plugin-react-refresh": "^0.4.24", + "globals": "^16.5.0", + "jsdom": "^28.1.0", + "typescript": "~5.9.3", + "typescript-eslint": "^8.48.0", + "vite": "^7.3.1", + "vitest": "^4.0.18" + } + }, + "node_modules/@acemir/cssom": { + "version": "0.9.31", + "resolved": "https://registry.npmjs.org/@acemir/cssom/-/cssom-0.9.31.tgz", + "integrity": "sha512-ZnR3GSaH+/vJ0YlHau21FjfLYjMpYVIzTD8M8vIEQvIGxeOXyXdzCI140rrCY862p/C/BbzWsjc1dgnM9mkoTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@adobe/css-tools": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.4.4.tgz", + "integrity": "sha512-Elp+iwUx5rN5+Y8xLt5/GRoG20WGoDCQ/1Fb+1LiGtvwbDavuSk0jhD/eZdckHAuzcDzccnkv+rEjyWfRx18gg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@asamuzakjp/css-color": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.0.1.tgz", + "integrity": "sha512-2SZFvqMyvboVV1d15lMf7XiI3m7SDqXUuKaTymJYLN6dSGadqp+fVojqJlVoMlbZnlTmu3S0TLwLTJpvBMO1Aw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@csstools/css-calc": "^3.1.1", + "@csstools/css-color-parser": "^4.0.2", + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0", + "lru-cache": "^11.2.6" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/@asamuzakjp/css-color/node_modules/lru-cache": { + "version": "11.2.6", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.6.tgz", + "integrity": "sha512-ESL2CrkS/2wTPfuend7Zhkzo2u0daGJ/A2VucJOgQ/C48S/zB8MMeMHSGKYpXhIjbPxfuezITkaBH1wqv00DDQ==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@asamuzakjp/dom-selector": { + "version": "6.8.1", + "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-6.8.1.tgz", + "integrity": "sha512-MvRz1nCqW0fsy8Qz4dnLIvhOlMzqDVBabZx6lH+YywFDdjXhMY37SmpV1XFX3JzG5GWHn63j6HX6QPr3lZXHvQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@asamuzakjp/nwsapi": "^2.3.9", + "bidi-js": "^1.0.3", + "css-tree": "^3.1.0", + "is-potential-custom-element-name": "^1.0.1", + "lru-cache": "^11.2.6" + } + }, + "node_modules/@asamuzakjp/dom-selector/node_modules/lru-cache": { + "version": "11.2.6", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.6.tgz", + "integrity": "sha512-ESL2CrkS/2wTPfuend7Zhkzo2u0daGJ/A2VucJOgQ/C48S/zB8MMeMHSGKYpXhIjbPxfuezITkaBH1wqv00DDQ==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@asamuzakjp/nwsapi": { + "version": "2.3.9", + "resolved": "https://registry.npmjs.org/@asamuzakjp/nwsapi/-/nwsapi-2.3.9.tgz", + "integrity": "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@babel/code-frame": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", + "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.28.5", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.0.tgz", + "integrity": "sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.0.tgz", + "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/code-frame": "^7.29.0", + "@babel/generator": "^7.29.0", + "@babel/helper-compilation-targets": "^7.28.6", + "@babel/helper-module-transforms": "^7.28.6", + "@babel/helpers": "^7.28.6", + "@babel/parser": "^7.29.0", + "@babel/template": "^7.28.6", + "@babel/traverse": "^7.29.0", + "@babel/types": "^7.29.0", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.1.tgz", + "integrity": "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.0", + "@babel/types": "^7.29.0", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.28.6.tgz", + "integrity": "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.28.6", + "@babel/helper-validator-option": "^7.27.1", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", + "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.28.6.tgz", + "integrity": "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.6.tgz", + "integrity": "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.28.6", + "@babel/helper-validator-identifier": "^7.28.5", + "@babel/traverse": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", + "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.6.tgz", + "integrity": "sha512-xOBvwq86HHdB7WUDTfKfT/Vuxh7gElQ+Sfti2Cy6yIWNW05P8iUslOVcZ4/sKbE+/jQaukQAdz/gf3724kYdqw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.0.tgz", + "integrity": "sha512-IyDgFV5GeDUVX4YdF/3CPULtVGSXXMLh1xVIgdCgxApktqnQV0r7/8Nqthg+8YLGaAtdyIlo2qIdZrbCv4+7ww==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.0" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.6.tgz", + "integrity": "sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/template": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", + "integrity": "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.28.6", + "@babel/parser": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.0.tgz", + "integrity": "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.0", + "@babel/generator": "^7.29.0", + "@babel/helper-globals": "^7.28.0", + "@babel/parser": "^7.29.0", + "@babel/template": "^7.28.6", + "@babel/types": "^7.29.0", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", + "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bramus/specificity": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/@bramus/specificity/-/specificity-2.4.2.tgz", + "integrity": "sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "css-tree": "^3.0.0" + }, + "bin": { + "specificity": "bin/cli.js" + } + }, + "node_modules/@csstools/color-helpers": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.0.2.tgz", + "integrity": "sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/@csstools/css-calc": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.1.1.tgz", + "integrity": "sha512-HJ26Z/vmsZQqs/o3a6bgKslXGFAungXGbinULZO3eMsOyNJHeBBZfup5FiZInOghgoM4Hwnmw+OgbJCNg1wwUQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0" + } + }, + "node_modules/@csstools/css-color-parser": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.0.2.tgz", + "integrity": "sha512-0GEfbBLmTFf0dJlpsNU7zwxRIH0/BGEMuXLTCvFYxuL1tNhqzTbtnFICyJLTNK4a+RechKP75e7w42ClXSnJQw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/color-helpers": "^6.0.2", + "@csstools/css-calc": "^3.1.1" + }, + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0" + } + }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-4.0.0.tgz", + "integrity": "sha512-+B87qS7fIG3L5h3qwJ/IFbjoVoOe/bpOdh9hAjXbvx0o8ImEmUsGXN0inFOnk2ChCFgqkkGFQ+TpM5rbhkKe4w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "peer": true, + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-tokenizer": "^4.0.0" + } + }, + "node_modules/@csstools/css-syntax-patches-for-csstree": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.0.tgz", + "integrity": "sha512-H4tuz2nhWgNKLt1inYpoVCfbJbMwX/lQKp3g69rrrIMIYlFD9+zTykOKhNR8uGrAmbS/kT9n6hTFkmDkxLgeTA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0" + }, + "node_modules/@csstools/css-tokenizer": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-4.0.0.tgz", + "integrity": "sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "peer": true, + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", + "integrity": "sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.3.tgz", + "integrity": "sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz", + "integrity": "sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.3.tgz", + "integrity": "sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz", + "integrity": "sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz", + "integrity": "sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz", + "integrity": "sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz", + "integrity": "sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz", + "integrity": "sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz", + "integrity": "sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz", + "integrity": "sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz", + "integrity": "sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz", + "integrity": "sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==", + "cpu": [ + "mips64el" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz", + "integrity": "sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz", + "integrity": "sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz", + "integrity": "sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.3.tgz", + "integrity": "sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz", + "integrity": "sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz", + "integrity": "sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz", + "integrity": "sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz", + "integrity": "sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz", + "integrity": "sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz", + "integrity": "sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz", + "integrity": "sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz", + "integrity": "sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz", + "integrity": "sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", + "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.2.tgz", + "integrity": "sha512-nJl2KGTlrf9GjLimgIru+V/mzgSK0ABCDQRvxw5BjURL7WfH5uoWmizbH7QB6MmnMBd8cIC9uceWnezL1VZWWw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^2.1.7", + "debug": "^4.3.1", + "minimatch": "^3.1.5" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", + "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/core": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", + "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.5.tgz", + "integrity": "sha512-4IlJx0X0qftVsN5E+/vGujTRIFtwuLbNsVUe7TO6zYPDR1O6nFwvwhIKEKSrl6dZchmYBITazxKoUYOjdtjlRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.14.0", + "debug": "^4.3.2", + "espree": "^10.0.1", + "globals": "^14.0.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.1", + "minimatch": "^3.1.5", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/eslintrc/node_modules/globals": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", + "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@eslint/js": { + "version": "9.39.4", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.4.tgz", + "integrity": "sha512-nE7DEIchvtiFTwBw4Lfbu59PG+kCofhjsKaCWzxTpt4lfRjRMqG6uMBzKXuEcyXhOHoUp9riAm7/aWYGhXZ9cw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + } + }, + "node_modules/@eslint/object-schema": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", + "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", + "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0", + "levn": "^0.4.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@exodus/bytes": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.15.0.tgz", + "integrity": "sha512-UY0nlA+feH81UGSHv92sLEPLCeZFjXOuHhrIo0HQydScuQc8s0A7kL/UdgwgDq8g8ilksmuoF35YVTNphV2aBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + }, + "peerDependencies": { + "@noble/hashes": "^1.8.0 || ^2.0.0" + }, + "peerDependenciesMeta": { + "@noble/hashes": { + "optional": true + } + } + }, + "node_modules/@fontsource/roboto": { + "version": "5.2.10", + "resolved": "https://registry.npmjs.org/@fontsource/roboto/-/roboto-5.2.10.tgz", + "integrity": "sha512-8HlA5FtSfz//oFSr2eL7GFXAiE7eIkcGOtx7tjsLKq+as702x9+GU7K95iDeWFapHC4M2hv9RrpXKRTGGBI8Zg==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, + "node_modules/@hey-api/client-fetch": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/@hey-api/client-fetch/-/client-fetch-0.13.1.tgz", + "integrity": "sha512-29jBRYNdxVGlx5oewFgOrkulZckpIpBIRHth3uHFn1PrL2ucMy52FvWOY3U3dVx2go1Z3kUmMi6lr07iOpUqqA==", + "deprecated": "Starting with v0.73.0, this package is bundled directly inside @hey-api/openapi-ts.", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/hey-api" + }, + "peerDependencies": { + "@hey-api/openapi-ts": "< 2" + } + }, + "node_modules/@hey-api/codegen-core": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/@hey-api/codegen-core/-/codegen-core-0.7.1.tgz", + "integrity": "sha512-X5qG+rr/BJvr+pEGcoW6l2azoZGrVuxsviEIhuf+3VwL9bk0atfubT65Xwo+4jDxXvjbhZvlwS0Ty3I7mLE2fg==", + "license": "MIT", + "dependencies": { + "@hey-api/types": "0.1.3", + "ansi-colors": "4.1.3", + "c12": "3.3.3", + "color-support": "1.1.3" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/sponsors/hey-api" + }, + "peerDependencies": { + "typescript": ">=5.5.3" + } + }, + "node_modules/@hey-api/json-schema-ref-parser": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@hey-api/json-schema-ref-parser/-/json-schema-ref-parser-1.3.1.tgz", + "integrity": "sha512-7atnpUkT8TyUPHYPLk91j/GyaqMuwTEHanLOe50Dlx0EEvNuQqFD52Yjg8x4KU0UFL1mWlyhE+sUE/wAtQ1N2A==", + "license": "MIT", + "dependencies": { + "@jsdevtools/ono": "7.1.3", + "@types/json-schema": "7.0.15", + "js-yaml": "4.1.1" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/sponsors/hey-api" + } + }, + "node_modules/@hey-api/openapi-ts": { + "version": "0.94.0", + "resolved": "https://registry.npmjs.org/@hey-api/openapi-ts/-/openapi-ts-0.94.0.tgz", + "integrity": "sha512-dbg3GG+v7sg9/Ahb7yFzwzQIJwm151JAtsnh9KtFyqiN0rGkMGA3/VqogEUq1kJB9XWrlMQwigwzhiEQ33VCSg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@hey-api/codegen-core": "0.7.1", + "@hey-api/json-schema-ref-parser": "1.3.1", + "@hey-api/shared": "0.2.2", + "@hey-api/types": "0.1.3", + "ansi-colors": "4.1.3", + "color-support": "1.1.3", + "commander": "14.0.3" + }, + "bin": { + "openapi-ts": "bin/run.js" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/sponsors/hey-api" + }, + "peerDependencies": { + "typescript": ">=5.5.3" + } + }, + "node_modules/@hey-api/shared": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@hey-api/shared/-/shared-0.2.2.tgz", + "integrity": "sha512-vMqCS+j7F9xpWoXC7TBbqZkaelwrdeuSB+s/3elu54V5iq++S59xhkSq5rOgDIpI1trpE59zZQa6dpyUxItOgw==", + "license": "MIT", + "dependencies": { + "@hey-api/codegen-core": "0.7.1", + "@hey-api/json-schema-ref-parser": "1.3.1", + "@hey-api/types": "0.1.3", + "ansi-colors": "4.1.3", + "cross-spawn": "7.0.6", + "open": "11.0.0", + "semver": "7.7.3" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/sponsors/hey-api" + }, + "peerDependencies": { + "typescript": ">=5.5.3" + } + }, + "node_modules/@hey-api/shared/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@hey-api/types": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@hey-api/types/-/types-0.1.3.tgz", + "integrity": "sha512-mZaiPOWH761yD4GjDQvtjS2ZYLu5o5pI1TVSvV/u7cmbybv51/FVtinFBeaE1kFQCKZ8OQpn2ezjLBJrKsGATw==", + "license": "MIT", + "peerDependencies": { + "typescript": ">=5.5.3" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", + "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.7", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz", + "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.1", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@jsdevtools/ono": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", + "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==", + "license": "MIT" + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-rc.2", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.2.tgz", + "integrity": "sha512-izyXV/v+cHiRfozX62W9htOAvwMo4/bXKDrQ+vom1L1qRuexPock/7VZDAhnpHCLNejd3NJ6hiab+tO0D44Rgw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.59.0.tgz", + "integrity": "sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.59.0.tgz", + "integrity": "sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.59.0.tgz", + "integrity": "sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.59.0.tgz", + "integrity": "sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.59.0.tgz", + "integrity": "sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.59.0.tgz", + "integrity": "sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.59.0.tgz", + "integrity": "sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.59.0.tgz", + "integrity": "sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.59.0.tgz", + "integrity": "sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.59.0.tgz", + "integrity": "sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.59.0.tgz", + "integrity": "sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.59.0.tgz", + "integrity": "sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.59.0.tgz", + "integrity": "sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.59.0.tgz", + "integrity": "sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.59.0.tgz", + "integrity": "sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.59.0.tgz", + "integrity": "sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.59.0.tgz", + "integrity": "sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.59.0.tgz", + "integrity": "sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.59.0.tgz", + "integrity": "sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.59.0.tgz", + "integrity": "sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.59.0.tgz", + "integrity": "sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.59.0.tgz", + "integrity": "sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.59.0.tgz", + "integrity": "sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.59.0.tgz", + "integrity": "sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.59.0.tgz", + "integrity": "sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@swc/core": { + "version": "1.15.18", + "resolved": "https://registry.npmjs.org/@swc/core/-/core-1.15.18.tgz", + "integrity": "sha512-z87aF9GphWp//fnkRsqvtY+inMVPgYW3zSlXH1kJFvRT5H/wiAn+G32qW5l3oEk63KSF1x3Ov0BfHCObAmT8RA==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@swc/counter": "^0.1.3", + "@swc/types": "^0.1.25" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/swc" + }, + "optionalDependencies": { + "@swc/core-darwin-arm64": "1.15.18", + "@swc/core-darwin-x64": "1.15.18", + "@swc/core-linux-arm-gnueabihf": "1.15.18", + "@swc/core-linux-arm64-gnu": "1.15.18", + "@swc/core-linux-arm64-musl": "1.15.18", + "@swc/core-linux-x64-gnu": "1.15.18", + "@swc/core-linux-x64-musl": "1.15.18", + "@swc/core-win32-arm64-msvc": "1.15.18", + "@swc/core-win32-ia32-msvc": "1.15.18", + "@swc/core-win32-x64-msvc": "1.15.18" + }, + "peerDependencies": { + "@swc/helpers": ">=0.5.17" + }, + "peerDependenciesMeta": { + "@swc/helpers": { + "optional": true + } + } + }, + "node_modules/@swc/core-darwin-arm64": { + "version": "1.15.18", + "resolved": "https://registry.npmjs.org/@swc/core-darwin-arm64/-/core-darwin-arm64-1.15.18.tgz", + "integrity": "sha512-+mIv7uBuSaywN3C9LNuWaX1jJJ3SKfiJuE6Lr3bd+/1Iv8oMU7oLBjYMluX1UrEPzwN2qCdY6Io0yVicABoCwQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-darwin-x64": { + "version": "1.15.18", + "resolved": "https://registry.npmjs.org/@swc/core-darwin-x64/-/core-darwin-x64-1.15.18.tgz", + "integrity": "sha512-wZle0eaQhnzxWX5V/2kEOI6Z9vl/lTFEC6V4EWcn+5pDjhemCpQv9e/TDJ0GIoiClX8EDWRvuZwh+Z3dhL1NAg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-arm-gnueabihf": { + "version": "1.15.18", + "resolved": "https://registry.npmjs.org/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.15.18.tgz", + "integrity": "sha512-ao61HGXVqrJFHAcPtF4/DegmwEkVCo4HApnotLU8ognfmU8x589z7+tcf3hU+qBiU1WOXV5fQX6W9Nzs6hjxDw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-arm64-gnu": { + "version": "1.15.18", + "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.15.18.tgz", + "integrity": "sha512-3xnctOBLIq3kj8PxOCgPrGjBLP/kNOddr6f5gukYt/1IZxsITQaU9TDyjeX6jG+FiCIHjCuWuffsyQDL5Ew1bg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-arm64-musl": { + "version": "1.15.18", + "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.15.18.tgz", + "integrity": "sha512-0a+Lix+FSSHBSBOA0XznCcHo5/1nA6oLLjcnocvzXeqtdjnPb+SvchItHI+lfeiuj1sClYPDvPMLSLyXFaiIKw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-x64-gnu": { + "version": "1.15.18", + "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.15.18.tgz", + "integrity": "sha512-wG9J8vReUlpaHz4KOD/5UE1AUgirimU4UFT9oZmupUDEofxJKYb1mTA/DrMj0s78bkBiNI+7Fo2EgPuvOJfuAA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-x64-musl": { + "version": "1.15.18", + "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.15.18.tgz", + "integrity": "sha512-4nwbVvCphKzicwNWRmvD5iBaZj8JYsRGa4xOxJmOyHlMDpsvvJ2OR2cODlvWyGFH6BYL1MfIAK3qph3hp0Az6g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-win32-arm64-msvc": { + "version": "1.15.18", + "resolved": "https://registry.npmjs.org/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.15.18.tgz", + "integrity": "sha512-zk0RYO+LjiBCat2RTMHzAWaMky0cra9loH4oRrLKLLNuL+jarxKLFDA8xTZWEkCPLjUTwlRN7d28eDLLMgtUcQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-win32-ia32-msvc": { + "version": "1.15.18", + "resolved": "https://registry.npmjs.org/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.15.18.tgz", + "integrity": "sha512-yVuTrZ0RccD5+PEkpcLOBAuPbYBXS6rslENvIXfvJGXSdX5QGi1ehC4BjAMl5FkKLiam4kJECUI0l7Hq7T1vwg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-win32-x64-msvc": { + "version": "1.15.18", + "resolved": "https://registry.npmjs.org/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.15.18.tgz", + "integrity": "sha512-7NRmE4hmUQNCbYU3Hn9Tz57mK9Qq4c97ZS+YlamlK6qG9Fb5g/BB3gPDe0iLlJkns/sYv2VWSkm8c3NmbEGjbg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/counter": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz", + "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/@swc/types": { + "version": "0.1.25", + "resolved": "https://registry.npmjs.org/@swc/types/-/types-0.1.25.tgz", + "integrity": "sha512-iAoY/qRhNH8a/hBvm3zKj9qQ4oc2+3w1unPJa2XvTK3XjeLXtzcCingVPw/9e5mn1+0yPqxcBGp9Jf0pkfMb1g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@swc/counter": "^0.1.3" + } + }, + "node_modules/@tailwindcss/node": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.2.1.tgz", + "integrity": "sha512-jlx6sLk4EOwO6hHe1oCGm1Q4AN/s0rSrTTPBGPM0/RQ6Uylwq17FuU8IeJJKEjtc6K6O07zsvP+gDO6MMWo7pg==", + "license": "MIT", + "dependencies": { + "@jridgewell/remapping": "^2.3.5", + "enhanced-resolve": "^5.19.0", + "jiti": "^2.6.1", + "lightningcss": "1.31.1", + "magic-string": "^0.30.21", + "source-map-js": "^1.2.1", + "tailwindcss": "4.2.1" + } + }, + "node_modules/@tailwindcss/oxide": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.2.1.tgz", + "integrity": "sha512-yv9jeEFWnjKCI6/T3Oq50yQEOqmpmpfzG1hcZsAOaXFQPfzWprWrlHSdGPEF3WQTi8zu8ohC9Mh9J470nT5pUw==", + "license": "MIT", + "engines": { + "node": ">= 20" + }, + "optionalDependencies": { + "@tailwindcss/oxide-android-arm64": "4.2.1", + "@tailwindcss/oxide-darwin-arm64": "4.2.1", + "@tailwindcss/oxide-darwin-x64": "4.2.1", + "@tailwindcss/oxide-freebsd-x64": "4.2.1", + "@tailwindcss/oxide-linux-arm-gnueabihf": "4.2.1", + "@tailwindcss/oxide-linux-arm64-gnu": "4.2.1", + "@tailwindcss/oxide-linux-arm64-musl": "4.2.1", + "@tailwindcss/oxide-linux-x64-gnu": "4.2.1", + "@tailwindcss/oxide-linux-x64-musl": "4.2.1", + "@tailwindcss/oxide-wasm32-wasi": "4.2.1", + "@tailwindcss/oxide-win32-arm64-msvc": "4.2.1", + "@tailwindcss/oxide-win32-x64-msvc": "4.2.1" + } + }, + "node_modules/@tailwindcss/oxide-android-arm64": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.2.1.tgz", + "integrity": "sha512-eZ7G1Zm5EC8OOKaesIKuw77jw++QJ2lL9N+dDpdQiAB/c/B2wDh0QPFHbkBVrXnwNugvrbJFk1gK2SsVjwWReg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-darwin-arm64": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.2.1.tgz", + "integrity": "sha512-q/LHkOstoJ7pI1J0q6djesLzRvQSIfEto148ppAd+BVQK0JYjQIFSK3JgYZJa+Yzi0DDa52ZsQx2rqytBnf8Hw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-darwin-x64": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.2.1.tgz", + "integrity": "sha512-/f/ozlaXGY6QLbpvd/kFTro2l18f7dHKpB+ieXz+Cijl4Mt9AI2rTrpq7V+t04nK+j9XBQHnSMdeQRhbGyt6fw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-freebsd-x64": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.2.1.tgz", + "integrity": "sha512-5e/AkgYJT/cpbkys/OU2Ei2jdETCLlifwm7ogMC7/hksI2fC3iiq6OcXwjibcIjPung0kRtR3TxEITkqgn0TcA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm-gnueabihf": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.2.1.tgz", + "integrity": "sha512-Uny1EcVTTmerCKt/1ZuKTkb0x8ZaiuYucg2/kImO5A5Y/kBz41/+j0gxUZl+hTF3xkWpDmHX+TaWhOtba2Fyuw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-gnu": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.2.1.tgz", + "integrity": "sha512-CTrwomI+c7n6aSSQlsPL0roRiNMDQ/YzMD9EjcR+H4f0I1SQ8QqIuPnsVp7QgMkC1Qi8rtkekLkOFjo7OlEFRQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-musl": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.2.1.tgz", + "integrity": "sha512-WZA0CHRL/SP1TRbA5mp9htsppSEkWuQ4KsSUumYQnyl8ZdT39ntwqmz4IUHGN6p4XdSlYfJwM4rRzZLShHsGAQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-gnu": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.2.1.tgz", + "integrity": "sha512-qMFzxI2YlBOLW5PhblzuSWlWfwLHaneBE0xHzLrBgNtqN6mWfs+qYbhryGSXQjFYB1Dzf5w+LN5qbUTPhW7Y5g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-musl": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.2.1.tgz", + "integrity": "sha512-5r1X2FKnCMUPlXTWRYpHdPYUY6a1Ar/t7P24OuiEdEOmms5lyqjDRvVY1yy9Rmioh+AunQ0rWiOTPE8F9A3v5g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.2.1.tgz", + "integrity": "sha512-MGFB5cVPvshR85MTJkEvqDUnuNoysrsRxd6vnk1Lf2tbiqNlXpHYZqkqOQalydienEWOHHFyyuTSYRsLfxFJ2Q==", + "bundleDependencies": [ + "@napi-rs/wasm-runtime", + "@emnapi/core", + "@emnapi/runtime", + "@tybys/wasm-util", + "@emnapi/wasi-threads", + "tslib" + ], + "cpu": [ + "wasm32" + ], + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.8.1", + "@emnapi/runtime": "^1.8.1", + "@emnapi/wasi-threads": "^1.1.0", + "@napi-rs/wasm-runtime": "^1.1.1", + "@tybys/wasm-util": "^0.10.1", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.2.1.tgz", + "integrity": "sha512-YlUEHRHBGnCMh4Nj4GnqQyBtsshUPdiNroZj8VPkvTZSoHsilRCwXcVKnG9kyi0ZFAS/3u+qKHBdDc81SADTRA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-win32-x64-msvc": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.2.1.tgz", + "integrity": "sha512-rbO34G5sMWWyrN/idLeVxAZgAKWrn5LiR3/I90Q9MkA67s6T1oB0xtTe+0heoBvHSpbU9Mk7i6uwJnpo4u21XQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/vite": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/vite/-/vite-4.2.1.tgz", + "integrity": "sha512-TBf2sJjYeb28jD2U/OhwdW0bbOsxkWPwQ7SrqGf9sVcoYwZj7rkXljroBO9wKBut9XnmQLXanuDUeqQK0lGg/w==", + "license": "MIT", + "dependencies": { + "@tailwindcss/node": "4.2.1", + "@tailwindcss/oxide": "4.2.1", + "tailwindcss": "4.2.1" + }, + "peerDependencies": { + "vite": "^5.2.0 || ^6 || ^7" + } + }, + "node_modules/@tanstack/query-core": { + "version": "5.90.20", + "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.90.20.tgz", + "integrity": "sha512-OMD2HLpNouXEfZJWcKeVKUgQ5n+n3A2JFmBaScpNDUqSrQSjiveC7dKMe53uJUg1nDG16ttFPz2xfilz6i2uVg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, + "node_modules/@tanstack/query-devtools": { + "version": "5.93.0", + "resolved": "https://registry.npmjs.org/@tanstack/query-devtools/-/query-devtools-5.93.0.tgz", + "integrity": "sha512-+kpsx1NQnOFTZsw6HAFCW3HkKg0+2cepGtAWXjiiSOJJ1CtQpt72EE2nyZb+AjAbLRPoeRmPJ8MtQd8r8gsPdg==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, + "node_modules/@tanstack/react-query": { + "version": "5.90.21", + "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.90.21.tgz", + "integrity": "sha512-0Lu6y5t+tvlTJMTO7oh5NSpJfpg/5D41LlThfepTixPYkJ0sE2Jj0m0f6yYqujBwIXlId87e234+MxG3D3g7kg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@tanstack/query-core": "5.90.20" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": "^18 || ^19" + } + }, + "node_modules/@tanstack/react-query-devtools": { + "version": "5.91.3", + "resolved": "https://registry.npmjs.org/@tanstack/react-query-devtools/-/react-query-devtools-5.91.3.tgz", + "integrity": "sha512-nlahjMtd/J1h7IzOOfqeyDh5LNfG0eULwlltPEonYy0QL+nqrBB+nyzJfULV+moL7sZyxc2sHdNJki+vLA9BSA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@tanstack/query-devtools": "5.93.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "@tanstack/react-query": "^5.90.20", + "react": "^18 || ^19" + } + }, + "node_modules/@testing-library/dom": { + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.1.tgz", + "integrity": "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/code-frame": "^7.10.4", + "@babel/runtime": "^7.12.5", + "@types/aria-query": "^5.0.1", + "aria-query": "5.3.0", + "dom-accessibility-api": "^0.5.9", + "lz-string": "^1.5.0", + "picocolors": "1.1.1", + "pretty-format": "^27.0.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@testing-library/jest-dom": { + "version": "6.9.1", + "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-6.9.1.tgz", + "integrity": "sha512-zIcONa+hVtVSSep9UT3jZ5rizo2BsxgyDYU7WFD5eICBE7no3881HGeb/QkGfsJs6JTkY1aQhT7rIPC7e+0nnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@adobe/css-tools": "^4.4.0", + "aria-query": "^5.0.0", + "css.escape": "^1.5.1", + "dom-accessibility-api": "^0.6.3", + "picocolors": "^1.1.1", + "redent": "^3.0.0" + }, + "engines": { + "node": ">=14", + "npm": ">=6", + "yarn": ">=1" + } + }, + "node_modules/@testing-library/jest-dom/node_modules/dom-accessibility-api": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.6.3.tgz", + "integrity": "sha512-7ZgogeTnjuHbo+ct10G9Ffp0mif17idi0IyWNVA/wcwcm7NPOD/WEHVP3n7n3MhXqxoIYm8d6MuZohYWIZ4T3w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@testing-library/react": { + "version": "16.3.2", + "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-16.3.2.tgz", + "integrity": "sha512-XU5/SytQM+ykqMnAnvB2umaJNIOsLF3PVv//1Ew4CTcpz0/BRyy/af40qqrt7SjKpDdT1saBMc42CUok5gaw+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.12.5" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@testing-library/dom": "^10.0.0", + "@types/react": "^18.0.0 || ^19.0.0", + "@types/react-dom": "^18.0.0 || ^19.0.0", + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@testing-library/user-event": { + "version": "14.6.1", + "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.1.tgz", + "integrity": "sha512-vq7fv0rnt+QTXgPxr5Hjc210p6YKq2kmdziLgnsZGgLJ9e6VAShx1pACLuRjd/AS/sr7phAR58OIIpf0LlmQNw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12", + "npm": ">=6" + }, + "peerDependencies": { + "@testing-library/dom": ">=7.21.4" + } + }, + "node_modules/@types/aria-query": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", + "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.12.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.0.tgz", + "integrity": "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ==", + "devOptional": true, + "license": "MIT", + "peer": true, + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/@types/react": { + "version": "19.2.14", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", + "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", + "devOptional": true, + "license": "MIT", + "peer": true, + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.2.3", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", + "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", + "dev": true, + "license": "MIT", + "peer": true, + "peerDependencies": { + "@types/react": "^19.2.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.57.0.tgz", + "integrity": "sha512-qeu4rTHR3/IaFORbD16gmjq9+rEs9fGKdX0kF6BKSfi+gCuG3RCKLlSBYzn/bGsY9Tj7KE/DAQStbp8AHJGHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/regexpp": "^4.12.2", + "@typescript-eslint/scope-manager": "8.57.0", + "@typescript-eslint/type-utils": "8.57.0", + "@typescript-eslint/utils": "8.57.0", + "@typescript-eslint/visitor-keys": "8.57.0", + "ignore": "^7.0.5", + "natural-compare": "^1.4.0", + "ts-api-utils": "^2.4.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^8.57.0", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.57.0.tgz", + "integrity": "sha512-XZzOmihLIr8AD1b9hL9ccNMzEMWt/dE2u7NyTY9jJG6YNiNthaD5XtUHVF2uCXZ15ng+z2hT3MVuxnUYhq6k1g==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@typescript-eslint/scope-manager": "8.57.0", + "@typescript-eslint/types": "8.57.0", + "@typescript-eslint/typescript-estree": "8.57.0", + "@typescript-eslint/visitor-keys": "8.57.0", + "debug": "^4.4.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/project-service": { + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.57.0.tgz", + "integrity": "sha512-pR+dK0BlxCLxtWfaKQWtYr7MhKmzqZxuii+ZjuFlZlIGRZm22HnXFqa2eY+90MUz8/i80YJmzFGDUsi8dMOV5w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/tsconfig-utils": "^8.57.0", + "@typescript-eslint/types": "^8.57.0", + "debug": "^4.4.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.57.0.tgz", + "integrity": "sha512-nvExQqAHF01lUM66MskSaZulpPL5pgy5hI5RfrxviLgzZVffB5yYzw27uK/ft8QnKXI2X0LBrHJFr1TaZtAibw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.57.0", + "@typescript-eslint/visitor-keys": "8.57.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/tsconfig-utils": { + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.57.0.tgz", + "integrity": "sha512-LtXRihc5ytjJIQEH+xqjB0+YgsV4/tW35XKX3GTZHpWtcC8SPkT/d4tqdf1cKtesryHm2bgp6l555NYcT2NLvA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/type-utils": { + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.57.0.tgz", + "integrity": "sha512-yjgh7gmDcJ1+TcEg8x3uWQmn8ifvSupnPfjP21twPKrDP/pTHlEQgmKcitzF/rzPSmv7QjJ90vRpN4U+zoUjwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.57.0", + "@typescript-eslint/typescript-estree": "8.57.0", + "@typescript-eslint/utils": "8.57.0", + "debug": "^4.4.3", + "ts-api-utils": "^2.4.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/types": { + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.57.0.tgz", + "integrity": "sha512-dTLI8PEXhjUC7B9Kre+u0XznO696BhXcTlOn0/6kf1fHaQW8+VjJAVHJ3eTI14ZapTxdkOmc80HblPQLaEeJdg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.57.0.tgz", + "integrity": "sha512-m7faHcyVg0BT3VdYTlX8GdJEM7COexXxS6KqGopxdtkQRvBanK377QDHr4W/vIPAR+ah9+B/RclSW5ldVniO1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/project-service": "8.57.0", + "@typescript-eslint/tsconfig-utils": "8.57.0", + "@typescript-eslint/types": "8.57.0", + "@typescript-eslint/visitor-keys": "8.57.0", + "debug": "^4.4.3", + "minimatch": "^10.2.2", + "semver": "^7.7.3", + "tinyglobby": "^0.2.15", + "ts-api-utils": "^2.4.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz", + "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { + "version": "10.2.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", + "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.57.0.tgz", + "integrity": "sha512-5iIHvpD3CZe06riAsbNxxreP+MuYgVUsV0n4bwLH//VJmgtt54sQeY2GszntJ4BjYCpMzrfVh2SBnUQTtys2lQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.9.1", + "@typescript-eslint/scope-manager": "8.57.0", + "@typescript-eslint/types": "8.57.0", + "@typescript-eslint/typescript-estree": "8.57.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.57.0.tgz", + "integrity": "sha512-zm6xx8UT/Xy2oSr2ZXD0pZo7Jx2XsCoID2IUh9YSTFRu7z+WdwYTRk6LhUftm1crwqbuoF6I8zAFeCMw0YjwDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.57.0", + "eslint-visitor-keys": "^5.0.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@vitejs/plugin-react-swc": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react-swc/-/plugin-react-swc-4.2.3.tgz", + "integrity": "sha512-QIluDil2prhY1gdA3GGwxZzTAmLdi8cQ2CcuMW4PB/Wu4e/1pzqrwhYWVd09LInCRlDUidQjd0B70QWbjWtLxA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rolldown/pluginutils": "1.0.0-rc.2", + "@swc/core": "^1.15.11" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "vite": "^4 || ^5 || ^6 || ^7" + } + }, + "node_modules/@vitest/expect": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.0.18.tgz", + "integrity": "sha512-8sCWUyckXXYvx4opfzVY03EOiYVxyNrHS5QxX3DAIi5dpJAAkyJezHCP77VMX4HKA2LDT/Jpfo8i2r5BE3GnQQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.0.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.0.18", + "@vitest/utils": "4.0.18", + "chai": "^6.2.1", + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.0.18.tgz", + "integrity": "sha512-HhVd0MDnzzsgevnOWCBj5Otnzobjy5wLBe4EdeeFGv8luMsGcYqDuFRMcttKWZA5vVO8RFjexVovXvAM4JoJDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.0.18", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.0.18.tgz", + "integrity": "sha512-P24GK3GulZWC5tz87ux0m8OADrQIUVDPIjjj65vBXYG17ZeU3qD7r+MNZ1RNv4l8CGU2vtTRqixrOi9fYk/yKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.0.18.tgz", + "integrity": "sha512-rpk9y12PGa22Jg6g5M3UVVnTS7+zycIGk9ZNGN+m6tZHKQb7jrP7/77WfZy13Y/EUDd52NDsLRQhYKtv7XfPQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.0.18", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.0.18.tgz", + "integrity": "sha512-PCiV0rcl7jKQjbgYqjtakly6T1uwv/5BQ9SwBLekVg/EaYeQFPiXcgrC2Y7vDMA8dM1SUEAEV82kgSQIlXNMvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.0.18", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.0.18.tgz", + "integrity": "sha512-cbQt3PTSD7P2OARdVW3qWER5EGq7PHlvE+QfzSC0lbwO+xnt7+XH06ZzFjFRgzUX//JmpxrCu92VdwvEPlWSNw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.0.18.tgz", + "integrity": "sha512-msMRKLMVLWygpK3u2Hybgi4MNjcYJvwTb0Ru09+fOyCXIgT5raYP041DRRdiJiI3k/2U6SEbAETB3YtBrUkCFA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.0.18", + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/acorn": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", + "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", + "dev": true, + "license": "MIT", + "peer": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ajv": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", + "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-colors": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/aria-query": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", + "integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "dequal": "^2.0.3" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/baseline-browser-mapping": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.0.tgz", + "integrity": "sha512-lIyg0szRfYbiy67j9KN8IyeD7q7hcmqnJ1ddWmNt19ItGpNN64mnllmxUNFIOdOm6by97jlL6wfpTTJrmnjWAA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/bidi-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz", + "integrity": "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==", + "dev": true, + "license": "MIT", + "dependencies": { + "require-from-string": "^2.0.2" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/browserslist": { + "version": "4.28.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", + "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "dependencies": { + "baseline-browser-mapping": "^2.9.0", + "caniuse-lite": "^1.0.30001759", + "electron-to-chromium": "^1.5.263", + "node-releases": "^2.0.27", + "update-browserslist-db": "^1.2.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/bundle-name": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", + "integrity": "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==", + "license": "MIT", + "dependencies": { + "run-applescript": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/c12": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/c12/-/c12-3.3.3.tgz", + "integrity": "sha512-750hTRvgBy5kcMNPdh95Qo+XUBeGo8C7nsKSmedDmaQI+E0r82DwHeM6vBewDe4rGFbnxoa4V9pw+sPh5+Iz8Q==", + "license": "MIT", + "dependencies": { + "chokidar": "^5.0.0", + "confbox": "^0.2.2", + "defu": "^6.1.4", + "dotenv": "^17.2.3", + "exsolve": "^1.0.8", + "giget": "^2.0.0", + "jiti": "^2.6.1", + "ohash": "^2.0.11", + "pathe": "^2.0.3", + "perfect-debounce": "^2.0.0", + "pkg-types": "^2.3.0", + "rc9": "^2.1.2" + }, + "peerDependencies": { + "magicast": "*" + }, + "peerDependenciesMeta": { + "magicast": { + "optional": true + } + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001777", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001777.tgz", + "integrity": "sha512-tmN+fJxroPndC74efCdp12j+0rk0RHwV5Jwa1zWaFVyw2ZxAuPeG8ZgWC3Wz7uSjT3qMRQ5XHZ4COgQmsCMJAQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chokidar": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", + "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", + "license": "MIT", + "dependencies": { + "readdirp": "^5.0.0" + }, + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/citty": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/citty/-/citty-0.1.6.tgz", + "integrity": "sha512-tskPPKEs8D2KPafUypv2gxwJP8h/OaJmC82QQGGDQcHvXX43xF2VDACcJVmZ0EuSxkpO9Kc4MlrA3q0+FG58AQ==", + "license": "MIT", + "dependencies": { + "consola": "^3.2.3" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/color-support": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", + "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", + "license": "ISC", + "bin": { + "color-support": "bin.js" + } + }, + "node_modules/commander": { + "version": "14.0.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz", + "integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/confbox": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.2.4.tgz", + "integrity": "sha512-ysOGlgTFbN2/Y6Cg3Iye8YKulHw+R2fNXHrgSmXISQdMnomY6eNDprVdW9R5xBguEqI954+S6709UyiO7B+6OQ==", + "license": "MIT" + }, + "node_modules/consola": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.2.tgz", + "integrity": "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==", + "license": "MIT", + "engines": { + "node": "^14.18.0 || >=16.10.0" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/css-tree": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz", + "integrity": "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==", + "dev": true, + "license": "MIT", + "dependencies": { + "mdn-data": "2.27.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" + } + }, + "node_modules/css.escape": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/css.escape/-/css.escape-1.5.1.tgz", + "integrity": "sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cssstyle": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-6.2.0.tgz", + "integrity": "sha512-Fm5NvhYathRnXNVndkUsCCuR63DCLVVwGOOwQw782coXFi5HhkXdu289l59HlXZBawsyNccXfWRYvLzcDCdDig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@asamuzakjp/css-color": "^5.0.1", + "@csstools/css-syntax-patches-for-csstree": "^1.0.28", + "css-tree": "^3.1.0", + "lru-cache": "^11.2.6" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/cssstyle/node_modules/lru-cache": { + "version": "11.2.6", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.6.tgz", + "integrity": "sha512-ESL2CrkS/2wTPfuend7Zhkzo2u0daGJ/A2VucJOgQ/C48S/zB8MMeMHSGKYpXhIjbPxfuezITkaBH1wqv00DDQ==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "devOptional": true, + "license": "MIT" + }, + "node_modules/daisyui": { + "version": "5.5.19", + "resolved": "https://registry.npmjs.org/daisyui/-/daisyui-5.5.19.tgz", + "integrity": "sha512-pbFAkl1VCEh/MPCeclKL61I/MqRIFFhNU7yiXoDDRapXN4/qNCoMxeCCswyxEEhqL5eiTTfwHvucFtOE71C9sA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/saadeghi/daisyui?sponsor=1" + } + }, + "node_modules/data-urls": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz", + "integrity": "sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-mimetype": "^5.0.0", + "whatwg-url": "^16.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decimal.js": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", + "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", + "dev": true, + "license": "MIT" + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/default-browser": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.5.0.tgz", + "integrity": "sha512-H9LMLr5zwIbSxrmvikGuI/5KGhZ8E2zH3stkMgM5LpOWDutGM2JZaj460Udnf1a+946zc7YBgrqEWwbk7zHvGw==", + "license": "MIT", + "dependencies": { + "bundle-name": "^4.1.0", + "default-browser-id": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/default-browser-id": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.1.tgz", + "integrity": "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/define-lazy-prop": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", + "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/defu": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.4.tgz", + "integrity": "sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==", + "license": "MIT" + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/destr": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/destr/-/destr-2.0.5.tgz", + "integrity": "sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA==", + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/dom-accessibility-api": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", + "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==", + "dev": true, + "license": "MIT" + }, + "node_modules/dotenv": { + "version": "17.3.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.3.1.tgz", + "integrity": "sha512-IO8C/dzEb6O3F9/twg6ZLXz164a2fhTnEWb95H23Dm4OuN+92NmEAlTrupP9VW6Jm3sO26tQlqyvyi4CsnY9GA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/echarts": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/echarts/-/echarts-6.0.0.tgz", + "integrity": "sha512-Tte/grDQRiETQP4xz3iZWSvoHrkCQtwqd6hs+mifXcjrCuo2iKWbajFObuLJVBlDIJlOzgQPd1hsaKt/3+OMkQ==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "tslib": "2.3.0", + "zrender": "6.0.0" + } + }, + "node_modules/echarts-for-react": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/echarts-for-react/-/echarts-for-react-3.0.6.tgz", + "integrity": "sha512-4zqLgTGWS3JvkQDXjzkR1k1CHRdpd6by0988TWMJgnvDytegWLbeP/VNZmMa+0VJx2eD7Y632bi2JquXDgiGJg==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "size-sensor": "^1.0.1" + }, + "peerDependencies": { + "echarts": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0", + "react": "^15.0.0 || >=16.0.0" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.307", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.307.tgz", + "integrity": "sha512-5z3uFKBWjiNR44nFcYdkcXjKMbg5KXNdciu7mhTPo9tB7NbqSNP2sSnGR+fqknZSCwKkBN+oxiiajWs4dT6ORg==", + "dev": true, + "license": "ISC" + }, + "node_modules/enhanced-resolve": { + "version": "5.20.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.20.0.tgz", + "integrity": "sha512-/ce7+jQ1PQ6rVXwe+jKEg5hW5ciicHwIQUagZkp6IufBoY3YDgdTTY1azVs0qoRgVmvsNB+rbjLJxDAeHHtwsQ==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.3.0" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", + "integrity": "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.3", + "@esbuild/android-arm": "0.27.3", + "@esbuild/android-arm64": "0.27.3", + "@esbuild/android-x64": "0.27.3", + "@esbuild/darwin-arm64": "0.27.3", + "@esbuild/darwin-x64": "0.27.3", + "@esbuild/freebsd-arm64": "0.27.3", + "@esbuild/freebsd-x64": "0.27.3", + "@esbuild/linux-arm": "0.27.3", + "@esbuild/linux-arm64": "0.27.3", + "@esbuild/linux-ia32": "0.27.3", + "@esbuild/linux-loong64": "0.27.3", + "@esbuild/linux-mips64el": "0.27.3", + "@esbuild/linux-ppc64": "0.27.3", + "@esbuild/linux-riscv64": "0.27.3", + "@esbuild/linux-s390x": "0.27.3", + "@esbuild/linux-x64": "0.27.3", + "@esbuild/netbsd-arm64": "0.27.3", + "@esbuild/netbsd-x64": "0.27.3", + "@esbuild/openbsd-arm64": "0.27.3", + "@esbuild/openbsd-x64": "0.27.3", + "@esbuild/openharmony-arm64": "0.27.3", + "@esbuild/sunos-x64": "0.27.3", + "@esbuild/win32-arm64": "0.27.3", + "@esbuild/win32-ia32": "0.27.3", + "@esbuild/win32-x64": "0.27.3" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "9.39.4", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.4.tgz", + "integrity": "sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.1", + "@eslint/config-array": "^0.21.2", + "@eslint/config-helpers": "^0.4.2", + "@eslint/core": "^0.17.0", + "@eslint/eslintrc": "^3.3.5", + "@eslint/js": "9.39.4", + "@eslint/plugin-kit": "^0.4.1", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.14.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^8.4.0", + "eslint-visitor-keys": "^4.2.1", + "espree": "^10.4.0", + "esquery": "^1.5.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.5", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-plugin-react-hooks": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-7.0.1.tgz", + "integrity": "sha512-O0d0m04evaNzEPoSW+59Mezf8Qt0InfgGIBJnpC0h3NH/WjUAR7BIKUfysC6todmtiZ/A0oUVS8Gce0WhBrHsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.24.4", + "@babel/parser": "^7.24.4", + "hermes-parser": "^0.25.1", + "zod": "^3.25.0 || ^4.0.0", + "zod-validation-error": "^3.5.0 || ^4.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0" + } + }, + "node_modules/eslint-plugin-react-refresh": { + "version": "0.4.26", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-refresh/-/eslint-plugin-react-refresh-0.4.26.tgz", + "integrity": "sha512-1RETEylht2O6FM/MvgnyvT+8K21wLqDNg4qD51Zj3guhjt433XbnnkVttHMyaVyAFD03QSV4LPS5iE3VQmO7XQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "eslint": ">=8.40" + } + }, + "node_modules/eslint-scope": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", + "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.15.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expect-type": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", + "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/exsolve": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/exsolve/-/exsolve-1.0.8.tgz", + "integrity": "sha512-LmDxfWXwcTArk8fUEnOfSZpHOJ6zOMUJKOtFLFqJLoKJetuQG874Uc7/Kki7zFLzYybmZhp1M7+98pfMqeX8yA==", + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/flatted": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.1.tgz", + "integrity": "sha512-IxfVbRFVlV8V/yRaGzk0UVIcsKKHMSfYw66T/u4nTwlWteQePsxe//LjudR1AMX4tZW3WFCh3Zqa/sjlqpbURQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/giget": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/giget/-/giget-2.0.0.tgz", + "integrity": "sha512-L5bGsVkxJbJgdnwyuheIunkGatUF/zssUoxxjACCseZYAVbaqdh9Tsmmlkl8vYan09H7sbvKt4pS8GqKLBrEzA==", + "license": "MIT", + "dependencies": { + "citty": "^0.1.6", + "consola": "^3.4.0", + "defu": "^6.1.4", + "node-fetch-native": "^1.6.6", + "nypm": "^0.6.0", + "pathe": "^2.0.3" + }, + "bin": { + "giget": "dist/cli.mjs" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "16.5.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-16.5.0.tgz", + "integrity": "sha512-c/c15i26VrJ4IRt5Z89DnIzCGDn9EcebibhAOjw5ibqEHsE1wLUgkPn9RDmNcUKyU87GeaL633nyJ+pplFR2ZQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/hermes-estree": { + "version": "0.25.1", + "resolved": "https://registry.npmjs.org/hermes-estree/-/hermes-estree-0.25.1.tgz", + "integrity": "sha512-0wUoCcLp+5Ev5pDW2OriHC2MJCbwLwuRx+gAqMTOkGKJJiBCLjtrvy4PWUGn6MIVefecRpzoOZ/UV6iGdOr+Cw==", + "dev": true, + "license": "MIT" + }, + "node_modules/hermes-parser": { + "version": "0.25.1", + "resolved": "https://registry.npmjs.org/hermes-parser/-/hermes-parser-0.25.1.tgz", + "integrity": "sha512-6pEjquH3rqaI6cYAXYPcz9MS4rY6R4ngRgrgfDshRptUZIc3lw0MCIJIGDj9++mfySOuPTHB4nrSW99BCvOPIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hermes-estree": "0.25.1" + } + }, + "node_modules/html-encoding-sniffer": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-6.0.0.tgz", + "integrity": "sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@exodus/bytes": "^1.6.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-docker": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", + "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-in-ssh": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-in-ssh/-/is-in-ssh-1.0.0.tgz", + "integrity": "sha512-jYa6Q9rH90kR1vKB6NM7qqd1mge3Fx4Dhw5TVlK1MUBqhEOuCagrEHMevNuCcbECmXZ0ThXkRm+Ymr51HwEPAw==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-inside-container": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", + "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", + "license": "MIT", + "dependencies": { + "is-docker": "^3.0.0" + }, + "bin": { + "is-inside-container": "cli.js" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/is-wsl": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.1.tgz", + "integrity": "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==", + "license": "MIT", + "dependencies": { + "is-inside-container": "^1.0.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jiti": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz", + "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsdom": { + "version": "28.1.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-28.1.0.tgz", + "integrity": "sha512-0+MoQNYyr2rBHqO1xilltfDjV9G7ymYGlAUazgcDLQaUf8JDHbuGwsxN6U9qWaElZ4w1B2r7yEGIL3GdeW3Rug==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@acemir/cssom": "^0.9.31", + "@asamuzakjp/dom-selector": "^6.8.1", + "@bramus/specificity": "^2.4.2", + "@exodus/bytes": "^1.11.0", + "cssstyle": "^6.0.1", + "data-urls": "^7.0.0", + "decimal.js": "^10.6.0", + "html-encoding-sniffer": "^6.0.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.6", + "is-potential-custom-element-name": "^1.0.1", + "parse5": "^8.0.0", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^6.0.0", + "undici": "^7.21.0", + "w3c-xmlserializer": "^5.0.0", + "webidl-conversions": "^8.0.1", + "whatwg-mimetype": "^5.0.0", + "whatwg-url": "^16.0.0", + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + }, + "peerDependencies": { + "canvas": "^3.0.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/lightningcss": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.31.1.tgz", + "integrity": "sha512-l51N2r93WmGUye3WuFoN5k10zyvrVs0qfKBhyC5ogUQ6Ew6JUSswh78mbSO+IU3nTWsyOArqPCcShdQSadghBQ==", + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.31.1", + "lightningcss-darwin-arm64": "1.31.1", + "lightningcss-darwin-x64": "1.31.1", + "lightningcss-freebsd-x64": "1.31.1", + "lightningcss-linux-arm-gnueabihf": "1.31.1", + "lightningcss-linux-arm64-gnu": "1.31.1", + "lightningcss-linux-arm64-musl": "1.31.1", + "lightningcss-linux-x64-gnu": "1.31.1", + "lightningcss-linux-x64-musl": "1.31.1", + "lightningcss-win32-arm64-msvc": "1.31.1", + "lightningcss-win32-x64-msvc": "1.31.1" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.31.1.tgz", + "integrity": "sha512-HXJF3x8w9nQ4jbXRiNppBCqeZPIAfUo8zE/kOEGbW5NZvGc/K7nMxbhIr+YlFlHW5mpbg/YFPdbnCh1wAXCKFg==", + "cpu": [ + "arm64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.31.1.tgz", + "integrity": "sha512-02uTEqf3vIfNMq3h/z2cJfcOXnQ0GRwQrkmPafhueLb2h7mqEidiCzkE4gBMEH65abHRiQvhdcQ+aP0D0g67sg==", + "cpu": [ + "arm64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.31.1.tgz", + "integrity": "sha512-1ObhyoCY+tGxtsz1lSx5NXCj3nirk0Y0kB/g8B8DT+sSx4G9djitg9ejFnjb3gJNWo7qXH4DIy2SUHvpoFwfTA==", + "cpu": [ + "x64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.31.1.tgz", + "integrity": "sha512-1RINmQKAItO6ISxYgPwszQE1BrsVU5aB45ho6O42mu96UiZBxEXsuQ7cJW4zs4CEodPUioj/QrXW1r9pLUM74A==", + "cpu": [ + "x64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.31.1.tgz", + "integrity": "sha512-OOCm2//MZJ87CdDK62rZIu+aw9gBv4azMJuA8/KB74wmfS3lnC4yoPHm0uXZ/dvNNHmnZnB8XLAZzObeG0nS1g==", + "cpu": [ + "arm" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.31.1.tgz", + "integrity": "sha512-WKyLWztD71rTnou4xAD5kQT+982wvca7E6QoLpoawZ1gP9JM0GJj4Tp5jMUh9B3AitHbRZ2/H3W5xQmdEOUlLg==", + "cpu": [ + "arm64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.31.1.tgz", + "integrity": "sha512-mVZ7Pg2zIbe3XlNbZJdjs86YViQFoJSpc41CbVmKBPiGmC4YrfeOyz65ms2qpAobVd7WQsbW4PdsSJEMymyIMg==", + "cpu": [ + "arm64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.31.1.tgz", + "integrity": "sha512-xGlFWRMl+0KvUhgySdIaReQdB4FNudfUTARn7q0hh/V67PVGCs3ADFjw+6++kG1RNd0zdGRlEKa+T13/tQjPMA==", + "cpu": [ + "x64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.31.1.tgz", + "integrity": "sha512-eowF8PrKHw9LpoZii5tdZwnBcYDxRw2rRCyvAXLi34iyeYfqCQNA9rmUM0ce62NlPhCvof1+9ivRaTY6pSKDaA==", + "cpu": [ + "x64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.31.1.tgz", + "integrity": "sha512-aJReEbSEQzx1uBlQizAOBSjcmr9dCdL3XuC/6HLXAxmtErsj2ICo5yYggg1qOODQMtnjNQv2UHb9NpOuFtYe4w==", + "cpu": [ + "arm64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.31.1.tgz", + "integrity": "sha512-I9aiFrbd7oYHwlnQDqr1Roz+fTz61oDDJX7n9tYF9FJymH1cIN1DtKw3iYt6b8WZgEjoNwVSncwF4wx/ZedMhw==", + "cpu": [ + "x64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/lz-string": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz", + "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", + "dev": true, + "license": "MIT", + "bin": { + "lz-string": "bin/bin.js" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/mdn-data": { + "version": "2.27.1", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz", + "integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==", + "dev": true, + "license": "CC0-1.0" + }, + "node_modules/min-indent": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", + "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-fetch-native": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/node-fetch-native/-/node-fetch-native-1.6.7.tgz", + "integrity": "sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==", + "license": "MIT" + }, + "node_modules/node-releases": { + "version": "2.0.36", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.36.tgz", + "integrity": "sha512-TdC8FSgHz8Mwtw9g5L4gR/Sh9XhSP/0DEkQxfEFXOpiul5IiHgHan2VhYYb6agDSfp4KuvltmGApc8HMgUrIkA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nypm": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/nypm/-/nypm-0.6.5.tgz", + "integrity": "sha512-K6AJy1GMVyfyMXRVB88700BJqNUkByijGJM8kEHpLdcAt+vSQAVfkWWHYzuRXHSY6xA2sNc5RjTj0p9rE2izVQ==", + "license": "MIT", + "dependencies": { + "citty": "^0.2.0", + "pathe": "^2.0.3", + "tinyexec": "^1.0.2" + }, + "bin": { + "nypm": "dist/cli.mjs" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/nypm/node_modules/citty": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/citty/-/citty-0.2.1.tgz", + "integrity": "sha512-kEV95lFBhQgtogAPlQfJJ0WGVSokvLr/UEoFPiKKOXF7pl98HfUVUD0ejsuTCld/9xH9vogSywZ5KqHzXrZpqg==", + "license": "MIT" + }, + "node_modules/obug": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", + "integrity": "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT" + }, + "node_modules/ohash": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/ohash/-/ohash-2.0.11.tgz", + "integrity": "sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ==", + "license": "MIT" + }, + "node_modules/open": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/open/-/open-11.0.0.tgz", + "integrity": "sha512-smsWv2LzFjP03xmvFoJ331ss6h+jixfA4UUV/Bsiyuu4YJPfN+FIQGOIiv4w9/+MoHkfkJ22UIaQWRVFRfH6Vw==", + "license": "MIT", + "dependencies": { + "default-browser": "^5.4.0", + "define-lazy-prop": "^3.0.0", + "is-in-ssh": "^1.0.0", + "is-inside-container": "^1.0.0", + "powershell-utils": "^0.1.0", + "wsl-utils": "^0.3.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse5": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.0.tgz", + "integrity": "sha512-9m4m5GSgXjL4AjumKzq1Fgfp3Z8rsvjRNbnkVwfu2ImRqE5D0LnY2QfDen18FSY9C573YU5XxSapdHZTZ2WolA==", + "dev": true, + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "license": "MIT" + }, + "node_modules/perfect-debounce": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/perfect-debounce/-/perfect-debounce-2.1.0.tgz", + "integrity": "sha512-LjgdTytVFXeUgtHZr9WYViYSM/g8MkcTPYDlPa3cDqMirHjKiSZPYd6DoL7pK8AJQr+uWkQvCjHNdiMqsrJs+g==", + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pkg-types": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-2.3.0.tgz", + "integrity": "sha512-SIqCzDRg0s9npO5XQ3tNZioRY1uK06lA41ynBC1YmFTmnY6FjUjVt6s4LoADmwoig1qqD0oK8h1p/8mlMx8Oig==", + "license": "MIT", + "dependencies": { + "confbox": "^0.2.2", + "exsolve": "^1.0.7", + "pathe": "^2.0.3" + } + }, + "node_modules/postcss": { + "version": "8.5.8", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", + "integrity": "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/powershell-utils": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/powershell-utils/-/powershell-utils-0.1.0.tgz", + "integrity": "sha512-dM0jVuXJPsDN6DvRpea484tCUaMiXWjuCn++HGTqUWzGDjv5tZkEZldAJ/UMlqRYGFrD/etByo4/xOuC/snX2A==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/pretty-format": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", + "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1", + "ansi-styles": "^5.0.0", + "react-is": "^17.0.1" + }, + "engines": { + "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/rc9": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/rc9/-/rc9-2.1.2.tgz", + "integrity": "sha512-btXCnMmRIBINM2LDZoEmOogIZU7Qe7zn4BpomSKZ/ykbLObuBdvG+mFq11DL6fjH1DRwHhrlgtYWG96bJiC7Cg==", + "license": "MIT", + "dependencies": { + "defu": "^6.1.4", + "destr": "^2.0.3" + } + }, + "node_modules/react": { + "version": "19.2.4", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz", + "integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.2.4", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.4.tgz", + "integrity": "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "scheduler": "^0.27.0" + }, + "peerDependencies": { + "react": "^19.2.4" + } + }, + "node_modules/react-is": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", + "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", + "dev": true, + "license": "MIT" + }, + "node_modules/react-router": { + "version": "7.13.1", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.13.1.tgz", + "integrity": "sha512-td+xP4X2/6BJvZoX6xw++A2DdEi++YypA69bJUV5oVvqf6/9/9nNlD70YO1e9d3MyamJEBQFEzk6mbfDYbqrSA==", + "license": "MIT", + "dependencies": { + "cookie": "^1.0.1", + "set-cookie-parser": "^2.6.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "react": ">=18", + "react-dom": ">=18" + }, + "peerDependenciesMeta": { + "react-dom": { + "optional": true + } + } + }, + "node_modules/react-router-dom": { + "version": "7.13.1", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.13.1.tgz", + "integrity": "sha512-UJnV3Rxc5TgUPJt2KJpo1Jpy0OKQr0AjgbZzBFjaPJcFOb2Y8jA5H3LT8HUJAiRLlWrEXWHbF1Z4SCZaQjWDHw==", + "license": "MIT", + "dependencies": { + "react-router": "7.13.1" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "react": ">=18", + "react-dom": ">=18" + } + }, + "node_modules/readdirp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.0.0.tgz", + "integrity": "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==", + "license": "MIT", + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/redent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", + "integrity": "sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==", + "dev": true, + "license": "MIT", + "dependencies": { + "indent-string": "^4.0.0", + "strip-indent": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/rollup": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz", + "integrity": "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==", + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.59.0", + "@rollup/rollup-android-arm64": "4.59.0", + "@rollup/rollup-darwin-arm64": "4.59.0", + "@rollup/rollup-darwin-x64": "4.59.0", + "@rollup/rollup-freebsd-arm64": "4.59.0", + "@rollup/rollup-freebsd-x64": "4.59.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.59.0", + "@rollup/rollup-linux-arm-musleabihf": "4.59.0", + "@rollup/rollup-linux-arm64-gnu": "4.59.0", + "@rollup/rollup-linux-arm64-musl": "4.59.0", + "@rollup/rollup-linux-loong64-gnu": "4.59.0", + "@rollup/rollup-linux-loong64-musl": "4.59.0", + "@rollup/rollup-linux-ppc64-gnu": "4.59.0", + "@rollup/rollup-linux-ppc64-musl": "4.59.0", + "@rollup/rollup-linux-riscv64-gnu": "4.59.0", + "@rollup/rollup-linux-riscv64-musl": "4.59.0", + "@rollup/rollup-linux-s390x-gnu": "4.59.0", + "@rollup/rollup-linux-x64-gnu": "4.59.0", + "@rollup/rollup-linux-x64-musl": "4.59.0", + "@rollup/rollup-openbsd-x64": "4.59.0", + "@rollup/rollup-openharmony-arm64": "4.59.0", + "@rollup/rollup-win32-arm64-msvc": "4.59.0", + "@rollup/rollup-win32-ia32-msvc": "4.59.0", + "@rollup/rollup-win32-x64-gnu": "4.59.0", + "@rollup/rollup-win32-x64-msvc": "4.59.0", + "fsevents": "~2.3.2" + } + }, + "node_modules/run-applescript": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.1.0.tgz", + "integrity": "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/saxes": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", + "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", + "dev": true, + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=v12.22.7" + } + }, + "node_modules/scheduler": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", + "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/set-cookie-parser": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", + "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", + "license": "MIT" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/size-sensor": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/size-sensor/-/size-sensor-1.0.3.tgz", + "integrity": "sha512-+k9mJ2/rQMiRmQUcjn+qznch260leIXY8r4FyYKKyRBO/s5UoeMAHGkCJyE1R/4wrIhTJONfyloY55SkE7ve3A==", + "license": "ISC" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", + "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "dev": true, + "license": "MIT" + }, + "node_modules/strip-indent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-3.0.0.tgz", + "integrity": "sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "min-indent": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "dev": true, + "license": "MIT" + }, + "node_modules/tailwindcss": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.2.1.tgz", + "integrity": "sha512-/tBrSQ36vCleJkAOsy9kbNTgaxvGbyOamC30PRePTQe/o1MFwEKHQk4Cn7BNGaPtjp+PuUrByJehM1hgxfq4sw==", + "license": "MIT" + }, + "node_modules/tapable": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.0.tgz", + "integrity": "sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg==", + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.2.tgz", + "integrity": "sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.15", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", + "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyrainbow": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.0.3.tgz", + "integrity": "sha512-PSkbLUoxOFRzJYjjxHJt9xro7D+iilgMX/C9lawzVuYiIdcihh9DXmVibBe8lmcFrRi/VzlPjBxbN7rH24q8/Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tldts": { + "version": "7.0.25", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.25.tgz", + "integrity": "sha512-keinCnPbwXEUG3ilrWQZU+CqcTTzHq9m2HhoUP2l7Xmi8l1LuijAXLpAJ5zRW+ifKTNscs4NdCkfkDCBYm352w==", + "dev": true, + "license": "MIT", + "dependencies": { + "tldts-core": "^7.0.25" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "7.0.25", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.25.tgz", + "integrity": "sha512-ZjCZK0rppSBu7rjHYDYsEaMOIbbT+nWF57hKkv4IUmZWBNrBWBOjIElc0mKRgLM8bm7x/BBlof6t2gi/Oq/Asw==", + "dev": true, + "license": "MIT" + }, + "node_modules/tough-cookie": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.0.tgz", + "integrity": "sha512-kXuRi1mtaKMrsLUxz3sQYvVl37B0Ns6MzfrtV5DvJceE9bPyspOqk9xxv7XbZWcfLWbFmm997vl83qUWVJA64w==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^7.0.5" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/tr46": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-6.0.0.tgz", + "integrity": "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==", + "dev": true, + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/ts-api-utils": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.4.0.tgz", + "integrity": "sha512-3TaVTaAv2gTiMB35i3FiGJaRfwb3Pyn/j3m/bfAvGe8FB7CF6u+LMYqYlDh7reQf7UNvoTvdfAqHGmPGOSsPmA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.12" + }, + "peerDependencies": { + "typescript": ">=4.8.4" + } + }, + "node_modules/tslib": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", + "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==", + "license": "0BSD" + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "license": "Apache-2.0", + "peer": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/typescript-eslint": { + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.57.0.tgz", + "integrity": "sha512-W8GcigEMEeB07xEZol8oJ26rigm3+bfPHxHvwbYUlu1fUDsGuQ7Hiskx5xGW/xM4USc9Ephe3jtv7ZYPQntHeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/eslint-plugin": "8.57.0", + "@typescript-eslint/parser": "8.57.0", + "@typescript-eslint/typescript-estree": "8.57.0", + "@typescript-eslint/utils": "8.57.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/undici": { + "version": "7.22.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.22.0.tgz", + "integrity": "sha512-RqslV2Us5BrllB+JeiZnK4peryVTndy9Dnqq62S3yYRRTj0tFQCwEniUy2167skdGOy3vqRzEvl1Dm4sV2ReDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "devOptional": true, + "license": "MIT" + }, + "node_modules/update-browserslist-db": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/vite": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.1.tgz", + "integrity": "sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==", + "license": "MIT", + "peer": true, + "dependencies": { + "esbuild": "^0.27.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.0.18.tgz", + "integrity": "sha512-hOQuK7h0FGKgBAas7v0mSAsnvrIgAvWmRFjmzpJ7SwFHH3g1k2u37JtYwOwmEKhK6ZO3v9ggDBBm0La1LCK4uQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.0.18", + "@vitest/mocker": "4.0.18", + "@vitest/pretty-format": "4.0.18", + "@vitest/runner": "4.0.18", + "@vitest/snapshot": "4.0.18", + "@vitest/spy": "4.0.18", + "@vitest/utils": "4.0.18", + "es-module-lexer": "^1.7.0", + "expect-type": "^1.2.2", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^3.10.0", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.0.3", + "vite": "^6.0.0 || ^7.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.0.18", + "@vitest/browser-preview": "4.0.18", + "@vitest/browser-webdriverio": "4.0.18", + "@vitest/ui": "4.0.18", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "node_modules/w3c-xmlserializer": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", + "integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==", + "dev": true, + "license": "MIT", + "dependencies": { + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/webidl-conversions": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-8.0.1.tgz", + "integrity": "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=20" + } + }, + "node_modules/whatwg-mimetype": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-5.0.0.tgz", + "integrity": "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + } + }, + "node_modules/whatwg-url": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-16.0.1.tgz", + "integrity": "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@exodus/bytes": "^1.11.0", + "tr46": "^6.0.0", + "webidl-conversions": "^8.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wsl-utils": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-0.3.1.tgz", + "integrity": "sha512-g/eziiSUNBSsdDJtCLB8bdYEUMj4jR7AGeUo96p/3dTafgjHhpF4RiCFPiRILwjQoDXx5MqkBr4fwWtR3Ky4Wg==", + "license": "MIT", + "dependencies": { + "is-wsl": "^3.1.0", + "powershell-utils": "^0.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/xml-name-validator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", + "integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "dev": true, + "license": "MIT" + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", + "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", + "dev": true, + "license": "MIT", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-validation-error": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/zod-validation-error/-/zod-validation-error-4.0.2.tgz", + "integrity": "sha512-Q6/nZLe6jxuU80qb/4uJ4t5v2VEZ44lzQjPDhYJNztRQ4wyWc6VF3D3Kb/fAuPetZQnhS3hnajCf9CsWesghLQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + } + }, + "node_modules/zrender": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/zrender/-/zrender-6.0.0.tgz", + "integrity": "sha512-41dFXEEXuJpNecuUQq6JlbybmnHaqqpGlbH1yxnA5V9MMP4SbohSVZsJIwz+zdjQXSSlR1Vc34EgH1zxyTDvhg==", + "license": "BSD-3-Clause", + "dependencies": { + "tslib": "2.3.0" + } + }, + "node_modules/zustand": { + "version": "5.0.11", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-5.0.11.tgz", + "integrity": "sha512-fdZY+dk7zn/vbWNCYmzZULHRrss0jx5pPFiOuMZ/5HJN6Yv3u+1Wswy/4MpZEkEGhtNH+pwxZB8OKgUBPzYAGg==", + "license": "MIT", + "engines": { + "node": ">=12.20.0" + }, + "peerDependencies": { + "@types/react": ">=18.0.0", + "immer": ">=9.0.6", + "react": ">=18.0.0", + "use-sync-external-store": ">=1.2.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "immer": { + "optional": true + }, + "react": { + "optional": true + }, + "use-sync-external-store": { + "optional": true + } + } + } + } +} diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 00000000..db693057 --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,51 @@ +{ + "name": "frontend", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "lint": "eslint .", + "typecheck": "tsc -b --noEmit", + "test": "vitest run", + "test:watch": "vitest", + "preview": "vite preview", + "openapi-ts": "openapi-ts" + }, + "dependencies": { + "@fontsource/roboto": "^5.2.10", + "@hey-api/client-fetch": "^0.13.1", + "@tailwindcss/vite": "^4.2.1", + "@tanstack/react-query": "^5.90.21", + "echarts": "^6.0.0", + "echarts-for-react": "^3.0.6", + "react": "^19.2.0", + "react-dom": "^19.2.0", + "react-router-dom": "^7.13.1", + "tailwindcss": "^4.2.1", + "zustand": "^5.0.11" + }, + "devDependencies": { + "@eslint/js": "^9.39.1", + "@hey-api/openapi-ts": "^0.94.0", + "@tanstack/react-query-devtools": "^5.91.3", + "@testing-library/jest-dom": "^6.9.1", + "@testing-library/react": "^16.3.2", + "@testing-library/user-event": "^14.6.1", + "@types/node": "^24.10.1", + "@types/react": "^19.2.7", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react-swc": "^4.2.2", + "daisyui": "^5.5.19", + "eslint": "^9.39.1", + "eslint-plugin-react-hooks": "^7.0.1", + "eslint-plugin-react-refresh": "^0.4.24", + "globals": "^16.5.0", + "jsdom": "^28.1.0", + "typescript": "~5.9.3", + "typescript-eslint": "^8.48.0", + "vite": "^7.3.1", + "vitest": "^4.0.18" + } +} diff --git a/frontend/public/vite.svg b/frontend/public/vite.svg new file mode 100644 index 00000000..e7b8dfb1 --- /dev/null +++ b/frontend/public/vite.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx new file mode 100644 index 00000000..0e1bb62c --- /dev/null +++ b/frontend/src/App.test.tsx @@ -0,0 +1,145 @@ +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { MemoryRouter } from 'react-router-dom'; +import type { ReactNode } from 'react'; +import App from './App'; +import { useSelectionStore } from './stores/useSelectionStore'; + +// Mock the API client with controllable responses +const mockListMetrics = vi.fn(); +const mockListSeries = vi.fn(); +const mockReadiness = vi.fn(); +const mockGetSeriesData = vi.fn(); + +vi.mock('./client', () => ({ + listMetrics: (...args: unknown[]) => mockListMetrics(...args), + listSeries: (...args: unknown[]) => mockListSeries(...args), + readiness: (...args: unknown[]) => mockReadiness(...args), + getSeriesData: (...args: unknown[]) => mockGetSeriesData(...args), +})); + +const emptyCatalog = { + '@context': {}, + '@type': 'dcat:Catalog', + '@id': 'sensapp_metrics_catalog', + 'dct:title': 'SensApp Metrics Catalog', + 'dcat:dataset': [], +}; + +const catalogWithMetrics = { + ...emptyCatalog, + 'dcat:dataset': [ + { + '@type': 'dcat:Dataset', + '@id': 'temperature', + 'dct:identifier': 'metric:temperature', + 'dct:title': 'temperature', + 'dct:description': 'Temperature sensor', + 'dcat:keyword': ['metric', 'float'], + 'sensor:type': 'float', + 'sensor:seriesCount': 2, + 'sensor:labelDimensions': ['location'], + 'dcat:distribution': [], + }, + ], +}; + +const seriesCatalog = { + '@context': {}, + '@type': 'dcat:Catalog', + '@id': 'sensapp_series_catalog', + 'dcat:dataset': [ + { + '@type': 'dcat:Dataset', + '@id': 'temperature{location="office"}', + 'dct:identifier': 'uuid-office', + 'dct:title': 'temperature', + 'sensor:type': 'float', + 'sensor:labels': [{ location: 'office' }], + 'dcat:distribution': [], + }, + ], +}; + +function createWrapper() { + const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false }, + }, + }); + + return function Wrapper({ children }: { children: ReactNode }) { + return ( + + {children} + + ); + }; +} + +describe('App integration', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockListMetrics.mockResolvedValue({ data: catalogWithMetrics }); + mockListSeries.mockResolvedValue({ data: seriesCatalog }); + mockReadiness.mockResolvedValue({ data: { status: 'ready', database: 'sqlite' } }); + mockGetSeriesData.mockResolvedValue({ data: [] }); + useSelectionStore.setState({ + selectedMetric: null, + selectedSeries: [], + labelFilter: '', + }); + }); + + it('renders the header with logo, health badge, and docs link', async () => { + render(, { wrapper: createWrapper() }); + + expect(screen.getByText('SensApp')).toBeInTheDocument(); + expect(await screen.findByText('Connected')).toBeInTheDocument(); + + const docsLink = screen.getByText('API Docs').closest('a'); + expect(docsLink).toHaveAttribute('href', '/docs'); + expect(docsLink).toHaveAttribute('target', '_blank'); + }); + + it('shows metrics after loading and allows selecting one', async () => { + const user = userEvent.setup(); + render(, { wrapper: createWrapper() }); + + // Metrics should load + const tempRow = await screen.findByText('temperature'); + expect(tempRow).toBeInTheDocument(); + + // Clicking should select the metric + await user.click(tempRow); + expect(useSelectionStore.getState().selectedMetric).toBe('temperature'); + }); + + it('shows the series panel after a metric is selected', async () => { + const user = userEvent.setup(); + render(, { wrapper: createWrapper() }); + + // Select the metric + const tempRow = await screen.findByText('temperature'); + await user.click(tempRow); + + // Series panel should show the metric name + expect(await screen.findByText((_, el) => el?.tagName === 'CODE' && el?.textContent === 'temperature')).toBeInTheDocument(); + }); + + it('shows chart placeholder when no series selected', async () => { + render(, { wrapper: createWrapper() }); + await screen.findByText('temperature'); // wait for load + + expect(screen.getByText(/Select a metric, then choose series to chart/)).toBeInTheDocument(); + }); + + it('renders the application shell', () => { + render(, { wrapper: createWrapper() }); + + expect(screen.getByText('SensApp')).toBeInTheDocument(); + expect(screen.getByText('Sensor Data Explorer')).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx new file mode 100644 index 00000000..4ff0085b --- /dev/null +++ b/frontend/src/App.tsx @@ -0,0 +1,15 @@ +import { Routes, Route } from 'react-router-dom'; +import { Layout } from './components/Layout'; +import { ExplorerPage } from './pages/ExplorerPage'; + +function App() { + return ( + + }> + } /> + + + ); +} + +export default App; diff --git a/frontend/src/api/clientConfig.test.ts b/frontend/src/api/clientConfig.test.ts new file mode 100644 index 00000000..d5853b9c --- /dev/null +++ b/frontend/src/api/clientConfig.test.ts @@ -0,0 +1,37 @@ +import { describe, it, expect } from 'vitest'; +import { extractErrorMessage } from './clientConfig'; + +describe('extractErrorMessage', () => { + it('extracts message from Error objects', () => { + expect(extractErrorMessage(new Error('something broke'))).toBe('something broke'); + }); + + it('returns string errors as-is', () => { + expect(extractErrorMessage('connection refused')).toBe('connection refused'); + }); + + it('extracts from SensApp AppError union types', () => { + expect(extractErrorMessage({ BadRequest: 'Invalid query' })).toBe('Invalid query'); + expect(extractErrorMessage({ NotFound: 'Series not found' })).toBe('Series not found'); + expect(extractErrorMessage({ InternalServerError: 'DB down' })).toBe('DB down'); + expect(extractErrorMessage({ Storage: 'Connection lost' })).toBe('Connection lost'); + }); + + it('extracts from objects with message property', () => { + expect(extractErrorMessage({ message: 'fetch failed' })).toBe('fetch failed'); + }); + + it('falls back to status code when available', () => { + expect(extractErrorMessage({ status: 503 })).toBe('Request failed with status 503'); + }); + + it('extracts from nested body objects', () => { + expect(extractErrorMessage({ body: { BadRequest: 'Invalid input' } })).toBe('Invalid input'); + }); + + it('returns default message for unknown types', () => { + expect(extractErrorMessage(42)).toBe('An unexpected error occurred'); + expect(extractErrorMessage(null)).toBe('An unexpected error occurred'); + expect(extractErrorMessage(undefined)).toBe('An unexpected error occurred'); + }); +}); diff --git a/frontend/src/api/clientConfig.ts b/frontend/src/api/clientConfig.ts new file mode 100644 index 00000000..b15dbd69 --- /dev/null +++ b/frontend/src/api/clientConfig.ts @@ -0,0 +1,38 @@ +import { client } from '../client/client.gen'; + +export function configureApiClient() { + const baseUrl = import.meta.env.VITE_SENSAPP_API_URL || ''; + client.setConfig({ baseUrl }); +} + +/** + * Extract a human-readable error message from API errors. + * hey-api/client-fetch throws raw response data on throwOnError, + * which may be an object, string, or Error. + */ +export function extractErrorMessage(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + if (typeof error === 'string') { + return error; + } + if (error && typeof error === 'object') { + // hey-api error responses or AppError union types + const obj = error as Record; + for (const key of ['message', 'InternalServerError', 'BadRequest', 'NotFound', 'Storage', 'error', 'detail']) { + if (typeof obj[key] === 'string') { + return obj[key] as string; + } + } + // Nested body from fetch errors + if (obj['body'] && typeof obj['body'] === 'object') { + return extractErrorMessage(obj['body']); + } + // Fallback: try status code + if (typeof obj['status'] === 'number') { + return `Request failed with status ${obj['status']}`; + } + } + return 'An unexpected error occurred'; +} diff --git a/frontend/src/client/@tanstack/react-query.gen.ts b/frontend/src/client/@tanstack/react-query.gen.ts new file mode 100644 index 00000000..9e0d8a18 --- /dev/null +++ b/frontend/src/client/@tanstack/react-query.gen.ts @@ -0,0 +1,356 @@ +// This file is auto-generated by @hey-api/openapi-ts + +import { type DefaultError, type InfiniteData, infiniteQueryOptions, queryOptions, type UseMutationOptions } from '@tanstack/react-query'; + +import { client } from '../client.gen'; +import { frontpage, getSeriesData, listMetrics, listSeries, liveness, type Options, prometheusMetrics, prometheusRemoteRead, publishInfluxdb, publishPrometheus, publishSensorsData, readiness, simplePromqlQuery, vacuumDatabase } from '../sdk.gen'; +import type { FrontpageData, FrontpageResponse, GetSeriesDataData, ListMetricsData, ListSeriesData, LivenessData, LivenessResponse, PrometheusMetricsData, PrometheusMetricsResponse, PrometheusRemoteReadData, PrometheusRemoteReadError, PublishInfluxdbData, PublishInfluxdbError, PublishInfluxdbResponse, PublishPrometheusData, PublishPrometheusError, PublishPrometheusResponse, PublishSensorsDataData, PublishSensorsDataError, PublishSensorsDataResponse, ReadinessData, ReadinessError, ReadinessResponse2, SimplePromqlQueryData, VacuumDatabaseData, VacuumDatabaseError, VacuumDatabaseResponse } from '../types.gen'; + +export type QueryKey = [ + Pick & { + _id: string; + _infinite?: boolean; + tags?: ReadonlyArray; + } +]; + +const createQueryKey = (id: string, options?: TOptions, infinite?: boolean, tags?: ReadonlyArray): [ + QueryKey[0] +] => { + const params: QueryKey[0] = { _id: id, baseUrl: options?.baseUrl || (options?.client ?? client).getConfig().baseUrl } as QueryKey[0]; + if (infinite) { + params._infinite = infinite; + } + if (tags) { + params.tags = tags; + } + if (options?.body) { + params.body = options.body; + } + if (options?.headers) { + params.headers = options.headers; + } + if (options?.path) { + params.path = options.path; + } + if (options?.query) { + params.query = options.query; + } + return [params]; +}; + +export const frontpageQueryKey = (options?: Options) => createQueryKey('frontpage', options); + +export const frontpageOptions = (options?: Options) => queryOptions>({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await frontpage({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: frontpageQueryKey(options) +}); + +/** + * Database Vacuuming + * + * Cleans up and optimizes the database by removing unused data and reclaiming space. + * (only if supported by the underlying storage engine). + */ +export const vacuumDatabaseMutation = (options?: Partial>): UseMutationOptions> => { + const mutationOptions: UseMutationOptions> = { + mutationFn: async (fnOptions) => { + const { data } = await vacuumDatabase({ + ...options, + ...fnOptions, + throwOnError: true + }); + return data; + } + }; + return mutationOptions; +}; + +/** + * Prometheus Remote Read API. + * + * Allows you to read data from SensApp using Prometheus remote read protocol. + * + * It follows the [Prometheus Remote Read specification](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/). + */ +export const prometheusRemoteReadMutation = (options?: Partial>): UseMutationOptions> => { + const mutationOptions: UseMutationOptions> = { + mutationFn: async (fnOptions) => { + const { data } = await prometheusRemoteRead({ + ...options, + ...fnOptions, + throwOnError: true + }); + return data; + } + }; + return mutationOptions; +}; + +/** + * Prometheus Remote Write API. + * + * Allows you to write data from Prometheus to SensApp. + * + * It follows the [Prometheus Remote Write specification](https://prometheus.io/docs/concepts/remote_write_spec/). + */ +export const publishPrometheusMutation = (options?: Partial>): UseMutationOptions> => { + const mutationOptions: UseMutationOptions> = { + mutationFn: async (fnOptions) => { + const { data } = await publishPrometheus({ + ...options, + ...fnOptions, + throwOnError: true + }); + return data; + } + }; + return mutationOptions; +}; + +export const simplePromqlQueryQueryKey = (options: Options) => createQueryKey('simplePromqlQuery', options); + +/** + * Simple PromQL query endpoint. + * + * Parses a PromQL expression and returns matching time series data. + * Only simple selectors (VectorSelector and MatrixSelector) are supported. + * + * # Examples + * + * - Simple metric: `GET /api/v1/query?query=my_metric` + * - With labels: `GET /api/v1/query?query=my_metric{env="prod"}` + * - Range query: `GET /api/v1/query?query=my_metric[5m]` + * - With format: `GET /api/v1/query?query=my_metric&format=csv` + */ +export const simplePromqlQueryOptions = (options: Options) => queryOptions>({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await simplePromqlQuery({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: simplePromqlQueryQueryKey(options) +}); + +/** + * InfluxDB Compatible Write API. + * + * Allows you to write data from InfluxDB or Telegraf to SensApp. + * [More information.](https://github.com/SINTEF/sensapp/blob/main/docs/INFLUX_DB.md) + */ +export const publishInfluxdbMutation = (options?: Partial>): UseMutationOptions> => { + const mutationOptions: UseMutationOptions> = { + mutationFn: async (fnOptions) => { + const { data } = await publishInfluxdb({ + ...options, + ...fnOptions, + throwOnError: true + }); + return data; + } + }; + return mutationOptions; +}; + +export const livenessQueryKey = (options?: Options) => createQueryKey('liveness', options); + +/** + * Liveness check + * + * Checks if the service is running. + * This endpoint always returns 200 OK if the server is able to respond. + */ +export const livenessOptions = (options?: Options) => queryOptions>({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await liveness({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: livenessQueryKey(options) +}); + +export const readinessQueryKey = (options?: Options) => createQueryKey('readiness', options); + +/** + * Readiness check + * + * Checks if the service is ready to accept traffic. + * This includes checking if the database connection is working + */ +export const readinessOptions = (options?: Options) => queryOptions>({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await readiness({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: readinessQueryKey(options) +}); + +export const listMetricsQueryKey = (options?: Options) => createQueryKey('listMetrics', options); + +/** + * List unique metrics (measurement types) with aggregated information in DCAT catalog format. + */ +export const listMetricsOptions = (options?: Options) => queryOptions>({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await listMetrics({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: listMetricsQueryKey(options) +}); + +export const prometheusMetricsQueryKey = (options?: Options) => createQueryKey('prometheusMetrics', options); + +export const prometheusMetricsOptions = (options?: Options) => queryOptions>({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await prometheusMetrics({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: prometheusMetricsQueryKey(options) +}); + +/** + * SensApp native data ingestion API supporting multiple formats. + * + * Accepts sensor data in one of the following formats: + * - **SenML JSON** (RFC 8428): `Content-Type: application/json` + * - **CSV**: `Content-Type: text/csv` or `application/csv` + * - **Apache Arrow IPC**: `Content-Type: application/vnd.apache.arrow.file` + * + * If no Content-Type header is provided, defaults to CSV format. + */ +export const publishSensorsDataMutation = (options?: Partial>): UseMutationOptions> => { + const mutationOptions: UseMutationOptions> = { + mutationFn: async (fnOptions) => { + const { data } = await publishSensorsData({ + ...options, + ...fnOptions, + throwOnError: true + }); + return data; + } + }; + return mutationOptions; +}; + +export const listSeriesQueryKey = (options?: Options) => createQueryKey('listSeries', options); + +/** + * List all series (time series) in DCAT catalog format. + */ +export const listSeriesOptions = (options?: Options) => queryOptions>({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await listSeries({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: listSeriesQueryKey(options) +}); + +export const getSeriesDataQueryKey = (options: Options) => createQueryKey('getSeriesData', options); + +/** + * Get series data in various formats based on query parameter. + */ +export const getSeriesDataOptions = (options: Options) => queryOptions>({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await getSeriesData({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: getSeriesDataQueryKey(options) +}); + +const createInfiniteParams = [0], 'body' | 'headers' | 'path' | 'query'>>(queryKey: QueryKey, page: K) => { + const params = { ...queryKey[0] }; + if (page.body) { + params.body = { + ...queryKey[0].body as any, + ...page.body as any + }; + } + if (page.headers) { + params.headers = { + ...queryKey[0].headers, + ...page.headers + }; + } + if (page.path) { + params.path = { + ...queryKey[0].path as any, + ...page.path as any + }; + } + if (page.query) { + params.query = { + ...queryKey[0].query as any, + ...page.query as any + }; + } + return params as unknown as typeof page; +}; + +export const getSeriesDataInfiniteQueryKey = (options: Options): QueryKey> => createQueryKey('getSeriesData', options, true); + +/** + * Get series data in various formats based on query parameter. + */ +export const getSeriesDataInfiniteOptions = (options: Options) => infiniteQueryOptions, QueryKey>, string | Pick>[0], 'body' | 'headers' | 'path' | 'query'>>( +// @ts-ignore +{ + queryFn: async ({ pageParam, queryKey, signal }) => { + // @ts-ignore + const page: Pick>[0], 'body' | 'headers' | 'path' | 'query'> = typeof pageParam === 'object' ? pageParam : { + query: { + start: pageParam + } + }; + const params = createInfiniteParams(queryKey, page); + const { data } = await getSeriesData({ + ...options, + ...params, + signal, + throwOnError: true + }); + return data; + }, + queryKey: getSeriesDataInfiniteQueryKey(options) +}); diff --git a/frontend/src/client/client.gen.ts b/frontend/src/client/client.gen.ts new file mode 100644 index 00000000..cab3c701 --- /dev/null +++ b/frontend/src/client/client.gen.ts @@ -0,0 +1,16 @@ +// This file is auto-generated by @hey-api/openapi-ts + +import { type ClientOptions, type Config, createClient, createConfig } from './client'; +import type { ClientOptions as ClientOptions2 } from './types.gen'; + +/** + * The `createClientConfig()` function will be called on client initialization + * and the returned object will become the client's initial configuration. + * + * You may want to initialize your client this way instead of calling + * `setConfig()`. This is useful for example if you're using Next.js + * to ensure your client always has the correct values. + */ +export type CreateClientConfig = (override?: Config) => Config & T>; + +export const client = createClient(createConfig()); diff --git a/frontend/src/client/client/client.gen.ts b/frontend/src/client/client/client.gen.ts new file mode 100644 index 00000000..14dc0a0e --- /dev/null +++ b/frontend/src/client/client/client.gen.ts @@ -0,0 +1,290 @@ +// This file is auto-generated by @hey-api/openapi-ts + +import { createSseClient } from '../core/serverSentEvents.gen'; +import type { HttpMethod } from '../core/types.gen'; +import { getValidRequestBody } from '../core/utils.gen'; +import type { Client, Config, RequestOptions, ResolvedRequestOptions } from './types.gen'; +import { + buildUrl, + createConfig, + createInterceptors, + getParseAs, + mergeConfigs, + mergeHeaders, + setAuthParams, +} from './utils.gen'; + +type ReqInit = Omit & { + body?: any; + headers: ReturnType; +}; + +export const createClient = (config: Config = {}): Client => { + let _config = mergeConfigs(createConfig(), config); + + const getConfig = (): Config => ({ ..._config }); + + const setConfig = (config: Config): Config => { + _config = mergeConfigs(_config, config); + return getConfig(); + }; + + const interceptors = createInterceptors(); + + const beforeRequest = async (options: RequestOptions) => { + const opts = { + ..._config, + ...options, + fetch: options.fetch ?? _config.fetch ?? globalThis.fetch, + headers: mergeHeaders(_config.headers, options.headers), + serializedBody: undefined as string | undefined, + }; + + if (opts.security) { + await setAuthParams({ + ...opts, + security: opts.security, + }); + } + + if (opts.requestValidator) { + await opts.requestValidator(opts); + } + + if (opts.body !== undefined && opts.bodySerializer) { + opts.serializedBody = opts.bodySerializer(opts.body) as string | undefined; + } + + // remove Content-Type header if body is empty to avoid sending invalid requests + if (opts.body === undefined || opts.serializedBody === '') { + opts.headers.delete('Content-Type'); + } + + const url = buildUrl(opts); + + return { opts, url }; + }; + + const request: Client['request'] = async (options) => { + // @ts-expect-error + const { opts, url } = await beforeRequest(options); + const requestInit: ReqInit = { + redirect: 'follow', + ...opts, + body: getValidRequestBody(opts), + }; + + let request = new Request(url, requestInit); + + for (const fn of interceptors.request.fns) { + if (fn) { + request = await fn(request, opts); + } + } + + // fetch must be assigned here, otherwise it would throw the error: + // TypeError: Failed to execute 'fetch' on 'Window': Illegal invocation + const _fetch = opts.fetch!; + let response: Response; + + try { + response = await _fetch(request); + } catch (error) { + // Handle fetch exceptions (AbortError, network errors, etc.) + let finalError = error; + + for (const fn of interceptors.error.fns) { + if (fn) { + finalError = (await fn(error, undefined as any, request, opts)) as unknown; + } + } + + finalError = finalError || ({} as unknown); + + if (opts.throwOnError) { + throw finalError; + } + + // Return error response + return opts.responseStyle === 'data' + ? undefined + : { + error: finalError, + request, + response: undefined as any, + }; + } + + for (const fn of interceptors.response.fns) { + if (fn) { + response = await fn(response, request, opts); + } + } + + const result = { + request, + response, + }; + + if (response.ok) { + const parseAs = + (opts.parseAs === 'auto' + ? getParseAs(response.headers.get('Content-Type')) + : opts.parseAs) ?? 'json'; + + if (response.status === 204 || response.headers.get('Content-Length') === '0') { + let emptyData: any; + switch (parseAs) { + case 'arrayBuffer': + case 'blob': + case 'text': + emptyData = await response[parseAs](); + break; + case 'formData': + emptyData = new FormData(); + break; + case 'stream': + emptyData = response.body; + break; + case 'json': + default: + emptyData = {}; + break; + } + return opts.responseStyle === 'data' + ? emptyData + : { + data: emptyData, + ...result, + }; + } + + let data: any; + switch (parseAs) { + case 'arrayBuffer': + case 'blob': + case 'formData': + case 'text': + data = await response[parseAs](); + break; + case 'json': { + // Some servers return 200 with no Content-Length and empty body. + // response.json() would throw; read as text and parse if non-empty. + const text = await response.text(); + data = text ? JSON.parse(text) : {}; + break; + } + case 'stream': + return opts.responseStyle === 'data' + ? response.body + : { + data: response.body, + ...result, + }; + } + + if (parseAs === 'json') { + if (opts.responseValidator) { + await opts.responseValidator(data); + } + + if (opts.responseTransformer) { + data = await opts.responseTransformer(data); + } + } + + return opts.responseStyle === 'data' + ? data + : { + data, + ...result, + }; + } + + const textError = await response.text(); + let jsonError: unknown; + + try { + jsonError = JSON.parse(textError); + } catch { + // noop + } + + const error = jsonError ?? textError; + let finalError = error; + + for (const fn of interceptors.error.fns) { + if (fn) { + finalError = (await fn(error, response, request, opts)) as string; + } + } + + finalError = finalError || ({} as string); + + if (opts.throwOnError) { + throw finalError; + } + + // TODO: we probably want to return error and improve types + return opts.responseStyle === 'data' + ? undefined + : { + error: finalError, + ...result, + }; + }; + + const makeMethodFn = (method: Uppercase) => (options: RequestOptions) => + request({ ...options, method }); + + const makeSseFn = (method: Uppercase) => async (options: RequestOptions) => { + const { opts, url } = await beforeRequest(options); + return createSseClient({ + ...opts, + body: opts.body as BodyInit | null | undefined, + headers: opts.headers as unknown as Record, + method, + onRequest: async (url, init) => { + let request = new Request(url, init); + for (const fn of interceptors.request.fns) { + if (fn) { + request = await fn(request, opts); + } + } + return request; + }, + serializedBody: getValidRequestBody(opts) as BodyInit | null | undefined, + url, + }); + }; + + const _buildUrl: Client['buildUrl'] = (options) => buildUrl({ ..._config, ...options }); + + return { + buildUrl: _buildUrl, + connect: makeMethodFn('CONNECT'), + delete: makeMethodFn('DELETE'), + get: makeMethodFn('GET'), + getConfig, + head: makeMethodFn('HEAD'), + interceptors, + options: makeMethodFn('OPTIONS'), + patch: makeMethodFn('PATCH'), + post: makeMethodFn('POST'), + put: makeMethodFn('PUT'), + request, + setConfig, + sse: { + connect: makeSseFn('CONNECT'), + delete: makeSseFn('DELETE'), + get: makeSseFn('GET'), + head: makeSseFn('HEAD'), + options: makeSseFn('OPTIONS'), + patch: makeSseFn('PATCH'), + post: makeSseFn('POST'), + put: makeSseFn('PUT'), + trace: makeSseFn('TRACE'), + }, + trace: makeMethodFn('TRACE'), + } as Client; +}; diff --git a/frontend/src/client/client/index.ts b/frontend/src/client/client/index.ts new file mode 100644 index 00000000..b295edec --- /dev/null +++ b/frontend/src/client/client/index.ts @@ -0,0 +1,25 @@ +// This file is auto-generated by @hey-api/openapi-ts + +export type { Auth } from '../core/auth.gen'; +export type { QuerySerializerOptions } from '../core/bodySerializer.gen'; +export { + formDataBodySerializer, + jsonBodySerializer, + urlSearchParamsBodySerializer, +} from '../core/bodySerializer.gen'; +export { buildClientParams } from '../core/params.gen'; +export { serializeQueryKeyValue } from '../core/queryKeySerializer.gen'; +export { createClient } from './client.gen'; +export type { + Client, + ClientOptions, + Config, + CreateClientConfig, + Options, + RequestOptions, + RequestResult, + ResolvedRequestOptions, + ResponseStyle, + TDataShape, +} from './types.gen'; +export { createConfig, mergeHeaders } from './utils.gen'; diff --git a/frontend/src/client/client/types.gen.ts b/frontend/src/client/client/types.gen.ts new file mode 100644 index 00000000..a3f86165 --- /dev/null +++ b/frontend/src/client/client/types.gen.ts @@ -0,0 +1,214 @@ +// This file is auto-generated by @hey-api/openapi-ts + +import type { Auth } from '../core/auth.gen'; +import type { + ServerSentEventsOptions, + ServerSentEventsResult, +} from '../core/serverSentEvents.gen'; +import type { Client as CoreClient, Config as CoreConfig } from '../core/types.gen'; +import type { Middleware } from './utils.gen'; + +export type ResponseStyle = 'data' | 'fields'; + +export interface Config + extends Omit, CoreConfig { + /** + * Base URL for all requests made by this client. + */ + baseUrl?: T['baseUrl']; + /** + * Fetch API implementation. You can use this option to provide a custom + * fetch instance. + * + * @default globalThis.fetch + */ + fetch?: typeof fetch; + /** + * Please don't use the Fetch client for Next.js applications. The `next` + * options won't have any effect. + * + * Install {@link https://www.npmjs.com/package/@hey-api/client-next `@hey-api/client-next`} instead. + */ + next?: never; + /** + * Return the response data parsed in a specified format. By default, `auto` + * will infer the appropriate method from the `Content-Type` response header. + * You can override this behavior with any of the {@link Body} methods. + * Select `stream` if you don't want to parse response data at all. + * + * @default 'auto' + */ + parseAs?: 'arrayBuffer' | 'auto' | 'blob' | 'formData' | 'json' | 'stream' | 'text'; + /** + * Should we return only data or multiple fields (data, error, response, etc.)? + * + * @default 'fields' + */ + responseStyle?: ResponseStyle; + /** + * Throw an error instead of returning it in the response? + * + * @default false + */ + throwOnError?: T['throwOnError']; +} + +export interface RequestOptions< + TData = unknown, + TResponseStyle extends ResponseStyle = 'fields', + ThrowOnError extends boolean = boolean, + Url extends string = string, +> + extends + Config<{ + responseStyle: TResponseStyle; + throwOnError: ThrowOnError; + }>, + Pick< + ServerSentEventsOptions, + | 'onRequest' + | 'onSseError' + | 'onSseEvent' + | 'sseDefaultRetryDelay' + | 'sseMaxRetryAttempts' + | 'sseMaxRetryDelay' + > { + /** + * Any body that you want to add to your request. + * + * {@link https://developer.mozilla.org/docs/Web/API/fetch#body} + */ + body?: unknown; + path?: Record; + query?: Record; + /** + * Security mechanism(s) to use for the request. + */ + security?: ReadonlyArray; + url: Url; +} + +export interface ResolvedRequestOptions< + TResponseStyle extends ResponseStyle = 'fields', + ThrowOnError extends boolean = boolean, + Url extends string = string, +> extends RequestOptions { + serializedBody?: string; +} + +export type RequestResult< + TData = unknown, + TError = unknown, + ThrowOnError extends boolean = boolean, + TResponseStyle extends ResponseStyle = 'fields', +> = ThrowOnError extends true + ? Promise< + TResponseStyle extends 'data' + ? TData extends Record + ? TData[keyof TData] + : TData + : { + data: TData extends Record ? TData[keyof TData] : TData; + request: Request; + response: Response; + } + > + : Promise< + TResponseStyle extends 'data' + ? (TData extends Record ? TData[keyof TData] : TData) | undefined + : ( + | { + data: TData extends Record ? TData[keyof TData] : TData; + error: undefined; + } + | { + data: undefined; + error: TError extends Record ? TError[keyof TError] : TError; + } + ) & { + request: Request; + response: Response; + } + >; + +export interface ClientOptions { + baseUrl?: string; + responseStyle?: ResponseStyle; + throwOnError?: boolean; +} + +type MethodFn = < + TData = unknown, + TError = unknown, + ThrowOnError extends boolean = false, + TResponseStyle extends ResponseStyle = 'fields', +>( + options: Omit, 'method'>, +) => RequestResult; + +type SseFn = < + TData = unknown, + TError = unknown, + ThrowOnError extends boolean = false, + TResponseStyle extends ResponseStyle = 'fields', +>( + options: Omit, 'method'>, +) => Promise>; + +type RequestFn = < + TData = unknown, + TError = unknown, + ThrowOnError extends boolean = false, + TResponseStyle extends ResponseStyle = 'fields', +>( + options: Omit, 'method'> & + Pick>, 'method'>, +) => RequestResult; + +type BuildUrlFn = < + TData extends { + body?: unknown; + path?: Record; + query?: Record; + url: string; + }, +>( + options: TData & Options, +) => string; + +export type Client = CoreClient & { + interceptors: Middleware; +}; + +/** + * The `createClientConfig()` function will be called on client initialization + * and the returned object will become the client's initial configuration. + * + * You may want to initialize your client this way instead of calling + * `setConfig()`. This is useful for example if you're using Next.js + * to ensure your client always has the correct values. + */ +export type CreateClientConfig = ( + override?: Config, +) => Config & T>; + +export interface TDataShape { + body?: unknown; + headers?: unknown; + path?: unknown; + query?: unknown; + url: string; +} + +type OmitKeys = Pick>; + +export type Options< + TData extends TDataShape = TDataShape, + ThrowOnError extends boolean = boolean, + TResponse = unknown, + TResponseStyle extends ResponseStyle = 'fields', +> = OmitKeys< + RequestOptions, + 'body' | 'path' | 'query' | 'url' +> & + ([TData] extends [never] ? unknown : Omit); diff --git a/frontend/src/client/client/utils.gen.ts b/frontend/src/client/client/utils.gen.ts new file mode 100644 index 00000000..b4bd2435 --- /dev/null +++ b/frontend/src/client/client/utils.gen.ts @@ -0,0 +1,316 @@ +// This file is auto-generated by @hey-api/openapi-ts + +import { getAuthToken } from '../core/auth.gen'; +import type { QuerySerializerOptions } from '../core/bodySerializer.gen'; +import { jsonBodySerializer } from '../core/bodySerializer.gen'; +import { + serializeArrayParam, + serializeObjectParam, + serializePrimitiveParam, +} from '../core/pathSerializer.gen'; +import { getUrl } from '../core/utils.gen'; +import type { Client, ClientOptions, Config, RequestOptions } from './types.gen'; + +export const createQuerySerializer = ({ + parameters = {}, + ...args +}: QuerySerializerOptions = {}) => { + const querySerializer = (queryParams: T) => { + const search: string[] = []; + if (queryParams && typeof queryParams === 'object') { + for (const name in queryParams) { + const value = queryParams[name]; + + if (value === undefined || value === null) { + continue; + } + + const options = parameters[name] || args; + + if (Array.isArray(value)) { + const serializedArray = serializeArrayParam({ + allowReserved: options.allowReserved, + explode: true, + name, + style: 'form', + value, + ...options.array, + }); + if (serializedArray) search.push(serializedArray); + } else if (typeof value === 'object') { + const serializedObject = serializeObjectParam({ + allowReserved: options.allowReserved, + explode: true, + name, + style: 'deepObject', + value: value as Record, + ...options.object, + }); + if (serializedObject) search.push(serializedObject); + } else { + const serializedPrimitive = serializePrimitiveParam({ + allowReserved: options.allowReserved, + name, + value: value as string, + }); + if (serializedPrimitive) search.push(serializedPrimitive); + } + } + } + return search.join('&'); + }; + return querySerializer; +}; + +/** + * Infers parseAs value from provided Content-Type header. + */ +export const getParseAs = (contentType: string | null): Exclude => { + if (!contentType) { + // If no Content-Type header is provided, the best we can do is return the raw response body, + // which is effectively the same as the 'stream' option. + return 'stream'; + } + + const cleanContent = contentType.split(';')[0]?.trim(); + + if (!cleanContent) { + return; + } + + if (cleanContent.startsWith('application/json') || cleanContent.endsWith('+json')) { + return 'json'; + } + + if (cleanContent === 'multipart/form-data') { + return 'formData'; + } + + if ( + ['application/', 'audio/', 'image/', 'video/'].some((type) => cleanContent.startsWith(type)) + ) { + return 'blob'; + } + + if (cleanContent.startsWith('text/')) { + return 'text'; + } + + return; +}; + +const checkForExistence = ( + options: Pick & { + headers: Headers; + }, + name?: string, +): boolean => { + if (!name) { + return false; + } + if ( + options.headers.has(name) || + options.query?.[name] || + options.headers.get('Cookie')?.includes(`${name}=`) + ) { + return true; + } + return false; +}; + +export const setAuthParams = async ({ + security, + ...options +}: Pick, 'security'> & + Pick & { + headers: Headers; + }) => { + for (const auth of security) { + if (checkForExistence(options, auth.name)) { + continue; + } + + const token = await getAuthToken(auth, options.auth); + + if (!token) { + continue; + } + + const name = auth.name ?? 'Authorization'; + + switch (auth.in) { + case 'query': + if (!options.query) { + options.query = {}; + } + options.query[name] = token; + break; + case 'cookie': + options.headers.append('Cookie', `${name}=${token}`); + break; + case 'header': + default: + options.headers.set(name, token); + break; + } + } +}; + +export const buildUrl: Client['buildUrl'] = (options) => + getUrl({ + baseUrl: options.baseUrl as string, + path: options.path, + query: options.query, + querySerializer: + typeof options.querySerializer === 'function' + ? options.querySerializer + : createQuerySerializer(options.querySerializer), + url: options.url, + }); + +export const mergeConfigs = (a: Config, b: Config): Config => { + const config = { ...a, ...b }; + if (config.baseUrl?.endsWith('/')) { + config.baseUrl = config.baseUrl.substring(0, config.baseUrl.length - 1); + } + config.headers = mergeHeaders(a.headers, b.headers); + return config; +}; + +const headersEntries = (headers: Headers): Array<[string, string]> => { + const entries: Array<[string, string]> = []; + headers.forEach((value, key) => { + entries.push([key, value]); + }); + return entries; +}; + +export const mergeHeaders = ( + ...headers: Array['headers'] | undefined> +): Headers => { + const mergedHeaders = new Headers(); + for (const header of headers) { + if (!header) { + continue; + } + + const iterator = header instanceof Headers ? headersEntries(header) : Object.entries(header); + + for (const [key, value] of iterator) { + if (value === null) { + mergedHeaders.delete(key); + } else if (Array.isArray(value)) { + for (const v of value) { + mergedHeaders.append(key, v as string); + } + } else if (value !== undefined) { + // assume object headers are meant to be JSON stringified, i.e. their + // content value in OpenAPI specification is 'application/json' + mergedHeaders.set( + key, + typeof value === 'object' ? JSON.stringify(value) : (value as string), + ); + } + } + } + return mergedHeaders; +}; + +type ErrInterceptor = ( + error: Err, + response: Res, + request: Req, + options: Options, +) => Err | Promise; + +type ReqInterceptor = (request: Req, options: Options) => Req | Promise; + +type ResInterceptor = ( + response: Res, + request: Req, + options: Options, +) => Res | Promise; + +class Interceptors { + fns: Array = []; + + clear(): void { + this.fns = []; + } + + eject(id: number | Interceptor): void { + const index = this.getInterceptorIndex(id); + if (this.fns[index]) { + this.fns[index] = null; + } + } + + exists(id: number | Interceptor): boolean { + const index = this.getInterceptorIndex(id); + return Boolean(this.fns[index]); + } + + getInterceptorIndex(id: number | Interceptor): number { + if (typeof id === 'number') { + return this.fns[id] ? id : -1; + } + return this.fns.indexOf(id); + } + + update(id: number | Interceptor, fn: Interceptor): number | Interceptor | false { + const index = this.getInterceptorIndex(id); + if (this.fns[index]) { + this.fns[index] = fn; + return id; + } + return false; + } + + use(fn: Interceptor): number { + this.fns.push(fn); + return this.fns.length - 1; + } +} + +export interface Middleware { + error: Interceptors>; + request: Interceptors>; + response: Interceptors>; +} + +export const createInterceptors = (): Middleware< + Req, + Res, + Err, + Options +> => ({ + error: new Interceptors>(), + request: new Interceptors>(), + response: new Interceptors>(), +}); + +const defaultQuerySerializer = createQuerySerializer({ + allowReserved: false, + array: { + explode: true, + style: 'form', + }, + object: { + explode: true, + style: 'deepObject', + }, +}); + +const defaultHeaders = { + 'Content-Type': 'application/json', +}; + +export const createConfig = ( + override: Config & T> = {}, +): Config & T> => ({ + ...jsonBodySerializer, + headers: defaultHeaders, + parseAs: 'auto', + querySerializer: defaultQuerySerializer, + ...override, +}); diff --git a/frontend/src/client/core/auth.gen.ts b/frontend/src/client/core/auth.gen.ts new file mode 100644 index 00000000..3ebf9947 --- /dev/null +++ b/frontend/src/client/core/auth.gen.ts @@ -0,0 +1,41 @@ +// This file is auto-generated by @hey-api/openapi-ts + +export type AuthToken = string | undefined; + +export interface Auth { + /** + * Which part of the request do we use to send the auth? + * + * @default 'header' + */ + in?: 'header' | 'query' | 'cookie'; + /** + * Header or query parameter name. + * + * @default 'Authorization' + */ + name?: string; + scheme?: 'basic' | 'bearer'; + type: 'apiKey' | 'http'; +} + +export const getAuthToken = async ( + auth: Auth, + callback: ((auth: Auth) => Promise | AuthToken) | AuthToken, +): Promise => { + const token = typeof callback === 'function' ? await callback(auth) : callback; + + if (!token) { + return; + } + + if (auth.scheme === 'bearer') { + return `Bearer ${token}`; + } + + if (auth.scheme === 'basic') { + return `Basic ${btoa(token)}`; + } + + return token; +}; diff --git a/frontend/src/client/core/bodySerializer.gen.ts b/frontend/src/client/core/bodySerializer.gen.ts new file mode 100644 index 00000000..67daca60 --- /dev/null +++ b/frontend/src/client/core/bodySerializer.gen.ts @@ -0,0 +1,82 @@ +// This file is auto-generated by @hey-api/openapi-ts + +import type { ArrayStyle, ObjectStyle, SerializerOptions } from './pathSerializer.gen'; + +export type QuerySerializer = (query: Record) => string; + +export type BodySerializer = (body: unknown) => unknown; + +type QuerySerializerOptionsObject = { + allowReserved?: boolean; + array?: Partial>; + object?: Partial>; +}; + +export type QuerySerializerOptions = QuerySerializerOptionsObject & { + /** + * Per-parameter serialization overrides. When provided, these settings + * override the global array/object settings for specific parameter names. + */ + parameters?: Record; +}; + +const serializeFormDataPair = (data: FormData, key: string, value: unknown): void => { + if (typeof value === 'string' || value instanceof Blob) { + data.append(key, value); + } else if (value instanceof Date) { + data.append(key, value.toISOString()); + } else { + data.append(key, JSON.stringify(value)); + } +}; + +const serializeUrlSearchParamsPair = (data: URLSearchParams, key: string, value: unknown): void => { + if (typeof value === 'string') { + data.append(key, value); + } else { + data.append(key, JSON.stringify(value)); + } +}; + +export const formDataBodySerializer = { + bodySerializer: (body: unknown): FormData => { + const data = new FormData(); + + Object.entries(body as Record).forEach(([key, value]) => { + if (value === undefined || value === null) { + return; + } + if (Array.isArray(value)) { + value.forEach((v) => serializeFormDataPair(data, key, v)); + } else { + serializeFormDataPair(data, key, value); + } + }); + + return data; + }, +}; + +export const jsonBodySerializer = { + bodySerializer: (body: unknown): string => + JSON.stringify(body, (_key, value) => (typeof value === 'bigint' ? value.toString() : value)), +}; + +export const urlSearchParamsBodySerializer = { + bodySerializer: (body: unknown): string => { + const data = new URLSearchParams(); + + Object.entries(body as Record).forEach(([key, value]) => { + if (value === undefined || value === null) { + return; + } + if (Array.isArray(value)) { + value.forEach((v) => serializeUrlSearchParamsPair(data, key, v)); + } else { + serializeUrlSearchParamsPair(data, key, value); + } + }); + + return data.toString(); + }, +}; diff --git a/frontend/src/client/core/params.gen.ts b/frontend/src/client/core/params.gen.ts new file mode 100644 index 00000000..7955601a --- /dev/null +++ b/frontend/src/client/core/params.gen.ts @@ -0,0 +1,169 @@ +// This file is auto-generated by @hey-api/openapi-ts + +type Slot = 'body' | 'headers' | 'path' | 'query'; + +export type Field = + | { + in: Exclude; + /** + * Field name. This is the name we want the user to see and use. + */ + key: string; + /** + * Field mapped name. This is the name we want to use in the request. + * If omitted, we use the same value as `key`. + */ + map?: string; + } + | { + in: Extract; + /** + * Key isn't required for bodies. + */ + key?: string; + map?: string; + } + | { + /** + * Field name. This is the name we want the user to see and use. + */ + key: string; + /** + * Field mapped name. This is the name we want to use in the request. + * If `in` is omitted, `map` aliases `key` to the transport layer. + */ + map: Slot; + }; + +export interface Fields { + allowExtra?: Partial>; + args?: ReadonlyArray; +} + +export type FieldsConfig = ReadonlyArray; + +const extraPrefixesMap: Record = { + $body_: 'body', + $headers_: 'headers', + $path_: 'path', + $query_: 'query', +}; +const extraPrefixes = Object.entries(extraPrefixesMap); + +type KeyMap = Map< + string, + | { + in: Slot; + map?: string; + } + | { + in?: never; + map: Slot; + } +>; + +const buildKeyMap = (fields: FieldsConfig, map?: KeyMap): KeyMap => { + if (!map) { + map = new Map(); + } + + for (const config of fields) { + if ('in' in config) { + if (config.key) { + map.set(config.key, { + in: config.in, + map: config.map, + }); + } + } else if ('key' in config) { + map.set(config.key, { + map: config.map, + }); + } else if (config.args) { + buildKeyMap(config.args, map); + } + } + + return map; +}; + +interface Params { + body: unknown; + headers: Record; + path: Record; + query: Record; +} + +const stripEmptySlots = (params: Params) => { + for (const [slot, value] of Object.entries(params)) { + if (value && typeof value === 'object' && !Array.isArray(value) && !Object.keys(value).length) { + delete params[slot as Slot]; + } + } +}; + +export const buildClientParams = (args: ReadonlyArray, fields: FieldsConfig) => { + const params: Params = { + body: {}, + headers: {}, + path: {}, + query: {}, + }; + + const map = buildKeyMap(fields); + + let config: FieldsConfig[number] | undefined; + + for (const [index, arg] of args.entries()) { + if (fields[index]) { + config = fields[index]; + } + + if (!config) { + continue; + } + + if ('in' in config) { + if (config.key) { + const field = map.get(config.key)!; + const name = field.map || config.key; + if (field.in) { + (params[field.in] as Record)[name] = arg; + } + } else { + params.body = arg; + } + } else { + for (const [key, value] of Object.entries(arg ?? {})) { + const field = map.get(key); + + if (field) { + if (field.in) { + const name = field.map || key; + (params[field.in] as Record)[name] = value; + } else { + params[field.map] = value; + } + } else { + const extra = extraPrefixes.find(([prefix]) => key.startsWith(prefix)); + + if (extra) { + const [prefix, slot] = extra; + (params[slot] as Record)[key.slice(prefix.length)] = value; + } else if ('allowExtra' in config && config.allowExtra) { + for (const [slot, allowed] of Object.entries(config.allowExtra)) { + if (allowed) { + (params[slot as Slot] as Record)[key] = value; + break; + } + } + } + } + } + } + } + + stripEmptySlots(params); + + return params; +}; diff --git a/frontend/src/client/core/pathSerializer.gen.ts b/frontend/src/client/core/pathSerializer.gen.ts new file mode 100644 index 00000000..994b2848 --- /dev/null +++ b/frontend/src/client/core/pathSerializer.gen.ts @@ -0,0 +1,171 @@ +// This file is auto-generated by @hey-api/openapi-ts + +interface SerializeOptions extends SerializePrimitiveOptions, SerializerOptions {} + +interface SerializePrimitiveOptions { + allowReserved?: boolean; + name: string; +} + +export interface SerializerOptions { + /** + * @default true + */ + explode: boolean; + style: T; +} + +export type ArrayStyle = 'form' | 'spaceDelimited' | 'pipeDelimited'; +export type ArraySeparatorStyle = ArrayStyle | MatrixStyle; +type MatrixStyle = 'label' | 'matrix' | 'simple'; +export type ObjectStyle = 'form' | 'deepObject'; +type ObjectSeparatorStyle = ObjectStyle | MatrixStyle; + +interface SerializePrimitiveParam extends SerializePrimitiveOptions { + value: string; +} + +export const separatorArrayExplode = (style: ArraySeparatorStyle) => { + switch (style) { + case 'label': + return '.'; + case 'matrix': + return ';'; + case 'simple': + return ','; + default: + return '&'; + } +}; + +export const separatorArrayNoExplode = (style: ArraySeparatorStyle) => { + switch (style) { + case 'form': + return ','; + case 'pipeDelimited': + return '|'; + case 'spaceDelimited': + return '%20'; + default: + return ','; + } +}; + +export const separatorObjectExplode = (style: ObjectSeparatorStyle) => { + switch (style) { + case 'label': + return '.'; + case 'matrix': + return ';'; + case 'simple': + return ','; + default: + return '&'; + } +}; + +export const serializeArrayParam = ({ + allowReserved, + explode, + name, + style, + value, +}: SerializeOptions & { + value: unknown[]; +}) => { + if (!explode) { + const joinedValues = ( + allowReserved ? value : value.map((v) => encodeURIComponent(v as string)) + ).join(separatorArrayNoExplode(style)); + switch (style) { + case 'label': + return `.${joinedValues}`; + case 'matrix': + return `;${name}=${joinedValues}`; + case 'simple': + return joinedValues; + default: + return `${name}=${joinedValues}`; + } + } + + const separator = separatorArrayExplode(style); + const joinedValues = value + .map((v) => { + if (style === 'label' || style === 'simple') { + return allowReserved ? v : encodeURIComponent(v as string); + } + + return serializePrimitiveParam({ + allowReserved, + name, + value: v as string, + }); + }) + .join(separator); + return style === 'label' || style === 'matrix' ? separator + joinedValues : joinedValues; +}; + +export const serializePrimitiveParam = ({ + allowReserved, + name, + value, +}: SerializePrimitiveParam) => { + if (value === undefined || value === null) { + return ''; + } + + if (typeof value === 'object') { + throw new Error( + 'Deeply-nested arrays/objects aren’t supported. Provide your own `querySerializer()` to handle these.', + ); + } + + return `${name}=${allowReserved ? value : encodeURIComponent(value)}`; +}; + +export const serializeObjectParam = ({ + allowReserved, + explode, + name, + style, + value, + valueOnly, +}: SerializeOptions & { + value: Record | Date; + valueOnly?: boolean; +}) => { + if (value instanceof Date) { + return valueOnly ? value.toISOString() : `${name}=${value.toISOString()}`; + } + + if (style !== 'deepObject' && !explode) { + let values: string[] = []; + Object.entries(value).forEach(([key, v]) => { + values = [...values, key, allowReserved ? (v as string) : encodeURIComponent(v as string)]; + }); + const joinedValues = values.join(','); + switch (style) { + case 'form': + return `${name}=${joinedValues}`; + case 'label': + return `.${joinedValues}`; + case 'matrix': + return `;${name}=${joinedValues}`; + default: + return joinedValues; + } + } + + const separator = separatorObjectExplode(style); + const joinedValues = Object.entries(value) + .map(([key, v]) => + serializePrimitiveParam({ + allowReserved, + name: style === 'deepObject' ? `${name}[${key}]` : key, + value: v as string, + }), + ) + .join(separator); + return style === 'label' || style === 'matrix' ? separator + joinedValues : joinedValues; +}; diff --git a/frontend/src/client/core/queryKeySerializer.gen.ts b/frontend/src/client/core/queryKeySerializer.gen.ts new file mode 100644 index 00000000..5000df60 --- /dev/null +++ b/frontend/src/client/core/queryKeySerializer.gen.ts @@ -0,0 +1,117 @@ +// This file is auto-generated by @hey-api/openapi-ts + +/** + * JSON-friendly union that mirrors what Pinia Colada can hash. + */ +export type JsonValue = + | null + | string + | number + | boolean + | JsonValue[] + | { [key: string]: JsonValue }; + +/** + * Replacer that converts non-JSON values (bigint, Date, etc.) to safe substitutes. + */ +export const queryKeyJsonReplacer = (_key: string, value: unknown) => { + if (value === undefined || typeof value === 'function' || typeof value === 'symbol') { + return undefined; + } + if (typeof value === 'bigint') { + return value.toString(); + } + if (value instanceof Date) { + return value.toISOString(); + } + return value; +}; + +/** + * Safely stringifies a value and parses it back into a JsonValue. + */ +export const stringifyToJsonValue = (input: unknown): JsonValue | undefined => { + try { + const json = JSON.stringify(input, queryKeyJsonReplacer); + if (json === undefined) { + return undefined; + } + return JSON.parse(json) as JsonValue; + } catch { + return undefined; + } +}; + +/** + * Detects plain objects (including objects with a null prototype). + */ +const isPlainObject = (value: unknown): value is Record => { + if (value === null || typeof value !== 'object') { + return false; + } + const prototype = Object.getPrototypeOf(value as object); + return prototype === Object.prototype || prototype === null; +}; + +/** + * Turns URLSearchParams into a sorted JSON object for deterministic keys. + */ +const serializeSearchParams = (params: URLSearchParams): JsonValue => { + const entries = Array.from(params.entries()).sort(([a], [b]) => a.localeCompare(b)); + const result: Record = {}; + + for (const [key, value] of entries) { + const existing = result[key]; + if (existing === undefined) { + result[key] = value; + continue; + } + + if (Array.isArray(existing)) { + (existing as string[]).push(value); + } else { + result[key] = [existing, value]; + } + } + + return result; +}; + +/** + * Normalizes any accepted value into a JSON-friendly shape for query keys. + */ +export const serializeQueryKeyValue = (value: unknown): JsonValue | undefined => { + if (value === null) { + return null; + } + + if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') { + return value; + } + + if (value === undefined || typeof value === 'function' || typeof value === 'symbol') { + return undefined; + } + + if (typeof value === 'bigint') { + return value.toString(); + } + + if (value instanceof Date) { + return value.toISOString(); + } + + if (Array.isArray(value)) { + return stringifyToJsonValue(value); + } + + if (typeof URLSearchParams !== 'undefined' && value instanceof URLSearchParams) { + return serializeSearchParams(value); + } + + if (isPlainObject(value)) { + return stringifyToJsonValue(value); + } + + return undefined; +}; diff --git a/frontend/src/client/core/serverSentEvents.gen.ts b/frontend/src/client/core/serverSentEvents.gen.ts new file mode 100644 index 00000000..6aa6cf02 --- /dev/null +++ b/frontend/src/client/core/serverSentEvents.gen.ts @@ -0,0 +1,243 @@ +// This file is auto-generated by @hey-api/openapi-ts + +import type { Config } from './types.gen'; + +export type ServerSentEventsOptions = Omit & + Pick & { + /** + * Fetch API implementation. You can use this option to provide a custom + * fetch instance. + * + * @default globalThis.fetch + */ + fetch?: typeof fetch; + /** + * Implementing clients can call request interceptors inside this hook. + */ + onRequest?: (url: string, init: RequestInit) => Promise; + /** + * Callback invoked when a network or parsing error occurs during streaming. + * + * This option applies only if the endpoint returns a stream of events. + * + * @param error The error that occurred. + */ + onSseError?: (error: unknown) => void; + /** + * Callback invoked when an event is streamed from the server. + * + * This option applies only if the endpoint returns a stream of events. + * + * @param event Event streamed from the server. + * @returns Nothing (void). + */ + onSseEvent?: (event: StreamEvent) => void; + serializedBody?: RequestInit['body']; + /** + * Default retry delay in milliseconds. + * + * This option applies only if the endpoint returns a stream of events. + * + * @default 3000 + */ + sseDefaultRetryDelay?: number; + /** + * Maximum number of retry attempts before giving up. + */ + sseMaxRetryAttempts?: number; + /** + * Maximum retry delay in milliseconds. + * + * Applies only when exponential backoff is used. + * + * This option applies only if the endpoint returns a stream of events. + * + * @default 30000 + */ + sseMaxRetryDelay?: number; + /** + * Optional sleep function for retry backoff. + * + * Defaults to using `setTimeout`. + */ + sseSleepFn?: (ms: number) => Promise; + url: string; + }; + +export interface StreamEvent { + data: TData; + event?: string; + id?: string; + retry?: number; +} + +export type ServerSentEventsResult = { + stream: AsyncGenerator< + TData extends Record ? TData[keyof TData] : TData, + TReturn, + TNext + >; +}; + +export const createSseClient = ({ + onRequest, + onSseError, + onSseEvent, + responseTransformer, + responseValidator, + sseDefaultRetryDelay, + sseMaxRetryAttempts, + sseMaxRetryDelay, + sseSleepFn, + url, + ...options +}: ServerSentEventsOptions): ServerSentEventsResult => { + let lastEventId: string | undefined; + + const sleep = sseSleepFn ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + + const createStream = async function* () { + let retryDelay: number = sseDefaultRetryDelay ?? 3000; + let attempt = 0; + const signal = options.signal ?? new AbortController().signal; + + while (true) { + if (signal.aborted) break; + + attempt++; + + const headers = + options.headers instanceof Headers + ? options.headers + : new Headers(options.headers as Record | undefined); + + if (lastEventId !== undefined) { + headers.set('Last-Event-ID', lastEventId); + } + + try { + const requestInit: RequestInit = { + redirect: 'follow', + ...options, + body: options.serializedBody, + headers, + signal, + }; + let request = new Request(url, requestInit); + if (onRequest) { + request = await onRequest(url, requestInit); + } + // fetch must be assigned here, otherwise it would throw the error: + // TypeError: Failed to execute 'fetch' on 'Window': Illegal invocation + const _fetch = options.fetch ?? globalThis.fetch; + const response = await _fetch(request); + + if (!response.ok) throw new Error(`SSE failed: ${response.status} ${response.statusText}`); + + if (!response.body) throw new Error('No body in SSE response'); + + const reader = response.body.pipeThrough(new TextDecoderStream()).getReader(); + + let buffer = ''; + + const abortHandler = () => { + try { + reader.cancel(); + } catch { + // noop + } + }; + + signal.addEventListener('abort', abortHandler); + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += value; + // Normalize line endings: CRLF -> LF, then CR -> LF + buffer = buffer.replace(/\r\n/g, '\n').replace(/\r/g, '\n'); + + const chunks = buffer.split('\n\n'); + buffer = chunks.pop() ?? ''; + + for (const chunk of chunks) { + const lines = chunk.split('\n'); + const dataLines: Array = []; + let eventName: string | undefined; + + for (const line of lines) { + if (line.startsWith('data:')) { + dataLines.push(line.replace(/^data:\s*/, '')); + } else if (line.startsWith('event:')) { + eventName = line.replace(/^event:\s*/, ''); + } else if (line.startsWith('id:')) { + lastEventId = line.replace(/^id:\s*/, ''); + } else if (line.startsWith('retry:')) { + const parsed = Number.parseInt(line.replace(/^retry:\s*/, ''), 10); + if (!Number.isNaN(parsed)) { + retryDelay = parsed; + } + } + } + + let data: unknown; + let parsedJson = false; + + if (dataLines.length) { + const rawData = dataLines.join('\n'); + try { + data = JSON.parse(rawData); + parsedJson = true; + } catch { + data = rawData; + } + } + + if (parsedJson) { + if (responseValidator) { + await responseValidator(data); + } + + if (responseTransformer) { + data = await responseTransformer(data); + } + } + + onSseEvent?.({ + data, + event: eventName, + id: lastEventId, + retry: retryDelay, + }); + + if (dataLines.length) { + yield data as any; + } + } + } + } finally { + signal.removeEventListener('abort', abortHandler); + reader.releaseLock(); + } + + break; // exit loop on normal completion + } catch (error) { + // connection failed or aborted; retry after delay + onSseError?.(error); + + if (sseMaxRetryAttempts !== undefined && attempt >= sseMaxRetryAttempts) { + break; // stop after firing error + } + + // exponential backoff: double retry each attempt, cap at 30s + const backoff = Math.min(retryDelay * 2 ** (attempt - 1), sseMaxRetryDelay ?? 30000); + await sleep(backoff); + } + } + }; + + const stream = createStream(); + + return { stream }; +}; diff --git a/frontend/src/client/core/types.gen.ts b/frontend/src/client/core/types.gen.ts new file mode 100644 index 00000000..97463257 --- /dev/null +++ b/frontend/src/client/core/types.gen.ts @@ -0,0 +1,104 @@ +// This file is auto-generated by @hey-api/openapi-ts + +import type { Auth, AuthToken } from './auth.gen'; +import type { BodySerializer, QuerySerializer, QuerySerializerOptions } from './bodySerializer.gen'; + +export type HttpMethod = + | 'connect' + | 'delete' + | 'get' + | 'head' + | 'options' + | 'patch' + | 'post' + | 'put' + | 'trace'; + +export type Client< + RequestFn = never, + Config = unknown, + MethodFn = never, + BuildUrlFn = never, + SseFn = never, +> = { + /** + * Returns the final request URL. + */ + buildUrl: BuildUrlFn; + getConfig: () => Config; + request: RequestFn; + setConfig: (config: Config) => Config; +} & { + [K in HttpMethod]: MethodFn; +} & ([SseFn] extends [never] ? { sse?: never } : { sse: { [K in HttpMethod]: SseFn } }); + +export interface Config { + /** + * Auth token or a function returning auth token. The resolved value will be + * added to the request payload as defined by its `security` array. + */ + auth?: ((auth: Auth) => Promise | AuthToken) | AuthToken; + /** + * A function for serializing request body parameter. By default, + * {@link JSON.stringify()} will be used. + */ + bodySerializer?: BodySerializer | null; + /** + * An object containing any HTTP headers that you want to pre-populate your + * `Headers` object with. + * + * {@link https://developer.mozilla.org/docs/Web/API/Headers/Headers#init See more} + */ + headers?: + | RequestInit['headers'] + | Record< + string, + string | number | boolean | (string | number | boolean)[] | null | undefined | unknown + >; + /** + * The request method. + * + * {@link https://developer.mozilla.org/docs/Web/API/fetch#method See more} + */ + method?: Uppercase; + /** + * A function for serializing request query parameters. By default, arrays + * will be exploded in form style, objects will be exploded in deepObject + * style, and reserved characters are percent-encoded. + * + * This method will have no effect if the native `paramsSerializer()` Axios + * API function is used. + * + * {@link https://swagger.io/docs/specification/serialization/#query View examples} + */ + querySerializer?: QuerySerializer | QuerySerializerOptions; + /** + * A function validating request data. This is useful if you want to ensure + * the request conforms to the desired shape, so it can be safely sent to + * the server. + */ + requestValidator?: (data: unknown) => Promise; + /** + * A function transforming response data before it's returned. This is useful + * for post-processing data, e.g. converting ISO strings into Date objects. + */ + responseTransformer?: (data: unknown) => Promise; + /** + * A function validating response data. This is useful if you want to ensure + * the response conforms to the desired shape, so it can be safely passed to + * the transformers and returned to the user. + */ + responseValidator?: (data: unknown) => Promise; +} + +type IsExactlyNeverOrNeverUndefined = [T] extends [never] + ? true + : [T] extends [never | undefined] + ? [undefined] extends [T] + ? false + : true + : false; + +export type OmitNever> = { + [K in keyof T as IsExactlyNeverOrNeverUndefined extends true ? never : K]: T[K]; +}; diff --git a/frontend/src/client/core/utils.gen.ts b/frontend/src/client/core/utils.gen.ts new file mode 100644 index 00000000..e7ddbe35 --- /dev/null +++ b/frontend/src/client/core/utils.gen.ts @@ -0,0 +1,140 @@ +// This file is auto-generated by @hey-api/openapi-ts + +import type { BodySerializer, QuerySerializer } from './bodySerializer.gen'; +import { + type ArraySeparatorStyle, + serializeArrayParam, + serializeObjectParam, + serializePrimitiveParam, +} from './pathSerializer.gen'; + +export interface PathSerializer { + path: Record; + url: string; +} + +export const PATH_PARAM_RE = /\{[^{}]+\}/g; + +export const defaultPathSerializer = ({ path, url: _url }: PathSerializer) => { + let url = _url; + const matches = _url.match(PATH_PARAM_RE); + if (matches) { + for (const match of matches) { + let explode = false; + let name = match.substring(1, match.length - 1); + let style: ArraySeparatorStyle = 'simple'; + + if (name.endsWith('*')) { + explode = true; + name = name.substring(0, name.length - 1); + } + + if (name.startsWith('.')) { + name = name.substring(1); + style = 'label'; + } else if (name.startsWith(';')) { + name = name.substring(1); + style = 'matrix'; + } + + const value = path[name]; + + if (value === undefined || value === null) { + continue; + } + + if (Array.isArray(value)) { + url = url.replace(match, serializeArrayParam({ explode, name, style, value })); + continue; + } + + if (typeof value === 'object') { + url = url.replace( + match, + serializeObjectParam({ + explode, + name, + style, + value: value as Record, + valueOnly: true, + }), + ); + continue; + } + + if (style === 'matrix') { + url = url.replace( + match, + `;${serializePrimitiveParam({ + name, + value: value as string, + })}`, + ); + continue; + } + + const replaceValue = encodeURIComponent( + style === 'label' ? `.${value as string}` : (value as string), + ); + url = url.replace(match, replaceValue); + } + } + return url; +}; + +export const getUrl = ({ + baseUrl, + path, + query, + querySerializer, + url: _url, +}: { + baseUrl?: string; + path?: Record; + query?: Record; + querySerializer: QuerySerializer; + url: string; +}) => { + const pathUrl = _url.startsWith('/') ? _url : `/${_url}`; + let url = (baseUrl ?? '') + pathUrl; + if (path) { + url = defaultPathSerializer({ path, url }); + } + let search = query ? querySerializer(query) : ''; + if (search.startsWith('?')) { + search = search.substring(1); + } + if (search) { + url += `?${search}`; + } + return url; +}; + +export function getValidRequestBody(options: { + body?: unknown; + bodySerializer?: BodySerializer | null; + serializedBody?: unknown; +}) { + const hasBody = options.body !== undefined; + const isSerializedBody = hasBody && options.bodySerializer; + + if (isSerializedBody) { + if ('serializedBody' in options) { + const hasSerializedBody = + options.serializedBody !== undefined && options.serializedBody !== ''; + + return hasSerializedBody ? options.serializedBody : null; + } + + // not all clients implement a serializedBody property (i.e. client-axios) + return options.body !== '' ? options.body : null; + } + + // plain/text body + if (hasBody) { + return options.body; + } + + // no body was provided + return undefined; +} diff --git a/frontend/src/client/index.ts b/frontend/src/client/index.ts new file mode 100644 index 00000000..ac5804b9 --- /dev/null +++ b/frontend/src/client/index.ts @@ -0,0 +1,4 @@ +// This file is auto-generated by @hey-api/openapi-ts + +export { frontpage, getSeriesData, listMetrics, listSeries, liveness, type Options, prometheusMetrics, prometheusRemoteRead, publishInfluxdb, publishPrometheus, publishSensorsData, readiness, simplePromqlQuery, vacuumDatabase } from './sdk.gen'; +export type { AppError, ClientOptions, FrontpageData, FrontpageResponse, FrontpageResponses, GetSeriesDataData, GetSeriesDataErrors, GetSeriesDataResponses, HealthResponse, ListMetricsData, ListMetricsResponses, ListSeriesData, ListSeriesResponses, LivenessData, LivenessResponse, LivenessResponses, PrometheusMetricsData, PrometheusMetricsResponse, PrometheusMetricsResponses, PrometheusRemoteReadData, PrometheusRemoteReadError, PrometheusRemoteReadErrors, PrometheusRemoteReadResponses, PublishInfluxdbData, PublishInfluxdbError, PublishInfluxdbErrors, PublishInfluxdbResponse, PublishInfluxdbResponses, PublishPrometheusData, PublishPrometheusError, PublishPrometheusErrors, PublishPrometheusResponse, PublishPrometheusResponses, PublishSensorsDataData, PublishSensorsDataError, PublishSensorsDataErrors, PublishSensorsDataResponse, PublishSensorsDataResponses, ReadinessData, ReadinessError, ReadinessErrors, ReadinessResponse, ReadinessResponse2, ReadinessResponses, SimplePromqlQueryData, SimplePromqlQueryErrors, SimplePromqlQueryResponses, VacuumDatabaseData, VacuumDatabaseError, VacuumDatabaseErrors, VacuumDatabaseResponse, VacuumDatabaseResponses } from './types.gen'; diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts new file mode 100644 index 00000000..aac43772 --- /dev/null +++ b/frontend/src/client/sdk.gen.ts @@ -0,0 +1,131 @@ +// This file is auto-generated by @hey-api/openapi-ts + +import type { Client, Options as Options2, TDataShape } from './client'; +import { client } from './client.gen'; +import type { FrontpageData, FrontpageResponses, GetSeriesDataData, GetSeriesDataErrors, GetSeriesDataResponses, ListMetricsData, ListMetricsResponses, ListSeriesData, ListSeriesResponses, LivenessData, LivenessResponses, PrometheusMetricsData, PrometheusMetricsResponses, PrometheusRemoteReadData, PrometheusRemoteReadErrors, PrometheusRemoteReadResponses, PublishInfluxdbData, PublishInfluxdbErrors, PublishInfluxdbResponses, PublishPrometheusData, PublishPrometheusErrors, PublishPrometheusResponses, PublishSensorsDataData, PublishSensorsDataErrors, PublishSensorsDataResponses, ReadinessData, ReadinessErrors, ReadinessResponses, SimplePromqlQueryData, SimplePromqlQueryErrors, SimplePromqlQueryResponses, VacuumDatabaseData, VacuumDatabaseErrors, VacuumDatabaseResponses } from './types.gen'; + +export type Options = Options2 & { + /** + * You can provide a client instance returned by `createClient()` instead of + * individual options. This might be also useful if you want to implement a + * custom client. + */ + client?: Client; + /** + * You can pass arbitrary values through the `meta` object. This can be + * used to access values that aren't defined as part of the SDK function. + */ + meta?: Record; +}; + +export const frontpage = (options?: Options) => (options?.client ?? client).get({ url: '/', ...options }); + +/** + * Database Vacuuming + * + * Cleans up and optimizes the database by removing unused data and reclaiming space. + * (only if supported by the underlying storage engine). + */ +export const vacuumDatabase = (options?: Options) => (options?.client ?? client).post({ url: '/api/v1/admin/vacuum', ...options }); + +/** + * Prometheus Remote Read API. + * + * Allows you to read data from SensApp using Prometheus remote read protocol. + * + * It follows the [Prometheus Remote Read specification](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/). + */ +export const prometheusRemoteRead = (options: Options) => (options.client ?? client).post({ url: '/api/v1/prometheus_remote_read', ...options }); + +/** + * Prometheus Remote Write API. + * + * Allows you to write data from Prometheus to SensApp. + * + * It follows the [Prometheus Remote Write specification](https://prometheus.io/docs/concepts/remote_write_spec/). + */ +export const publishPrometheus = (options: Options) => (options.client ?? client).post({ url: '/api/v1/prometheus_remote_write', ...options }); + +/** + * Simple PromQL query endpoint. + * + * Parses a PromQL expression and returns matching time series data. + * Only simple selectors (VectorSelector and MatrixSelector) are supported. + * + * # Examples + * + * - Simple metric: `GET /api/v1/query?query=my_metric` + * - With labels: `GET /api/v1/query?query=my_metric{env="prod"}` + * - Range query: `GET /api/v1/query?query=my_metric[5m]` + * - With format: `GET /api/v1/query?query=my_metric&format=csv` + */ +export const simplePromqlQuery = (options: Options) => (options.client ?? client).get({ url: '/api/v1/query', ...options }); + +/** + * InfluxDB Compatible Write API. + * + * Allows you to write data from InfluxDB or Telegraf to SensApp. + * [More information.](https://github.com/SINTEF/sensapp/blob/main/docs/INFLUX_DB.md) + */ +export const publishInfluxdb = (options: Options) => (options.client ?? client).post({ + bodySerializer: null, + url: '/api/v2/write', + ...options, + headers: { + 'Content-Type': 'text/plain', + ...options.headers + } +}); + +/** + * Liveness check + * + * Checks if the service is running. + * This endpoint always returns 200 OK if the server is able to respond. + */ +export const liveness = (options?: Options) => (options?.client ?? client).get({ url: '/health/live', ...options }); + +/** + * Readiness check + * + * Checks if the service is ready to accept traffic. + * This includes checking if the database connection is working + */ +export const readiness = (options?: Options) => (options?.client ?? client).get({ url: '/health/ready', ...options }); + +/** + * List unique metrics (measurement types) with aggregated information in DCAT catalog format. + */ +export const listMetrics = (options?: Options) => (options?.client ?? client).get({ url: '/metrics', ...options }); + +export const prometheusMetrics = (options?: Options) => (options?.client ?? client).get({ url: '/prometheus/metrics', ...options }); + +/** + * SensApp native data ingestion API supporting multiple formats. + * + * Accepts sensor data in one of the following formats: + * - **SenML JSON** (RFC 8428): `Content-Type: application/json` + * - **CSV**: `Content-Type: text/csv` or `application/csv` + * - **Apache Arrow IPC**: `Content-Type: application/vnd.apache.arrow.file` + * + * If no Content-Type header is provided, defaults to CSV format. + */ +export const publishSensorsData = (options: Options) => (options.client ?? client).post({ + bodySerializer: null, + url: '/publish', + ...options, + headers: { + 'Content-Type': 'text/plain', + ...options.headers + } +}); + +/** + * List all series (time series) in DCAT catalog format. + */ +export const listSeries = (options?: Options) => (options?.client ?? client).get({ url: '/series', ...options }); + +/** + * Get series data in various formats based on query parameter. + */ +export const getSeriesData = (options: Options) => (options.client ?? client).get({ url: '/series/{series_uuid}', ...options }); diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts new file mode 100644 index 00000000..0ad6570e --- /dev/null +++ b/frontend/src/client/types.gen.ts @@ -0,0 +1,429 @@ +// This file is auto-generated by @hey-api/openapi-ts + +export type ClientOptions = { + baseUrl: `${string}://${string}` | (string & {}); +}; + +export type AppError = { + InternalServerError: string; +} | { + BadRequest: string; +} | { + NotFound: string; +} | { + Unauthorized: string; +} | { + Forbidden: string; +} | { + Storage: string; +}; + +export type HealthResponse = { + status: string; +}; + +export type ReadinessResponse = { + database: string; + error?: string | null; + status: string; +}; + +export type FrontpageData = { + body?: never; + path?: never; + query?: never; + url: '/'; +}; + +export type FrontpageResponses = { + /** + * SensApp Frontpage + */ + 200: string; +}; + +export type FrontpageResponse = FrontpageResponses[keyof FrontpageResponses]; + +export type VacuumDatabaseData = { + body?: never; + path?: never; + query?: never; + url: '/api/v1/admin/vacuum'; +}; + +export type VacuumDatabaseErrors = { + /** + * Failed to vacuum database + */ + 500: string; +}; + +export type VacuumDatabaseError = VacuumDatabaseErrors[keyof VacuumDatabaseErrors]; + +export type VacuumDatabaseResponses = { + /** + * Database vacuum completed successfully + */ + 200: string; +}; + +export type VacuumDatabaseResponse = VacuumDatabaseResponses[keyof VacuumDatabaseResponses]; + +export type PrometheusRemoteReadData = { + /** + * Prometheus Remote Read endpoint. [Reference](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/) + */ + body?: unknown; + headers: { + /** + * Content encoding, must be snappy + */ + 'content-encoding': string; + /** + * Content type, must be application/x-protobuf + */ + 'content-type': string; + /** + * Prometheus Remote Read version, must be 0.1.0 + */ + 'x-prometheus-remote-read-version': string; + }; + path?: never; + query?: never; + url: '/api/v1/prometheus_remote_read'; +}; + +export type PrometheusRemoteReadErrors = { + /** + * Bad Request + */ + 400: AppError; + /** + * Internal Server Error + */ + 500: AppError; +}; + +export type PrometheusRemoteReadError = PrometheusRemoteReadErrors[keyof PrometheusRemoteReadErrors]; + +export type PrometheusRemoteReadResponses = { + /** + * Read Response + */ + 200: unknown; +}; + +export type PublishPrometheusData = { + /** + * Prometheus Remote Write endpoint. [Reference](https://prometheus.io/docs/concepts/remote_write_spec/) + */ + body?: unknown; + headers: { + /** + * Content encoding, must be snappy + */ + 'content-encoding': string; + /** + * Content type, must be application/x-protobuf + */ + 'content-type': string; + /** + * Prometheus Remote Write version, must be 0.1.0 + */ + 'x-prometheus-remote-write-version': string; + }; + path?: never; + query?: never; + url: '/api/v1/prometheus_remote_write'; +}; + +export type PublishPrometheusErrors = { + /** + * Bad Request + */ + 400: AppError; + /** + * Internal Server Error + */ + 500: AppError; +}; + +export type PublishPrometheusError = PublishPrometheusErrors[keyof PublishPrometheusErrors]; + +export type PublishPrometheusResponses = { + /** + * No Content + */ + 204: void; +}; + +export type PublishPrometheusResponse = PublishPrometheusResponses[keyof PublishPrometheusResponses]; + +export type SimplePromqlQueryData = { + body?: never; + path?: never; + query: { + /** + * PromQL query string (e.g., 'my_metric{label="value"}' or 'my_metric[5m]') + */ + query: string; + /** + * Output format: senml (default), csv, jsonl, or arrow + */ + format?: string; + }; + url: '/api/v1/query'; +}; + +export type SimplePromqlQueryErrors = { + /** + * Invalid or unsupported PromQL query + */ + 400: unknown; + /** + * Internal server error + */ + 500: unknown; +}; + +export type SimplePromqlQueryResponses = { + /** + * Query results in requested format + */ + 200: unknown; +}; + +export type PublishInfluxdbData = { + /** + * InfluxDB Line Protocol endpoint. [Reference](https://docs.influxdata.com/influxdb/v2/reference/syntax/line-protocol/). + */ + body: string; + path?: never; + query: { + /** + * Bucket name + */ + bucket: string; + /** + * Organization name + */ + org?: string; + /** + * Organization ID + */ + org_id?: string; + /** + * Precision of the timestamps. One of ns, us, ms, s + */ + precision?: string; + }; + url: '/api/v2/write'; +}; + +export type PublishInfluxdbErrors = { + /** + * Bad Request + */ + 400: AppError; + /** + * Internal Server Error + */ + 500: AppError; +}; + +export type PublishInfluxdbError = PublishInfluxdbErrors[keyof PublishInfluxdbErrors]; + +export type PublishInfluxdbResponses = { + /** + * No Content + */ + 204: void; +}; + +export type PublishInfluxdbResponse = PublishInfluxdbResponses[keyof PublishInfluxdbResponses]; + +export type LivenessData = { + body?: never; + path?: never; + query?: never; + url: '/health/live'; +}; + +export type LivenessResponses = { + /** + * Service is alive + */ + 200: HealthResponse; +}; + +export type LivenessResponse = LivenessResponses[keyof LivenessResponses]; + +export type ReadinessData = { + body?: never; + path?: never; + query?: never; + url: '/health/ready'; +}; + +export type ReadinessErrors = { + /** + * Service is not ready + */ + 503: ReadinessResponse; +}; + +export type ReadinessError = ReadinessErrors[keyof ReadinessErrors]; + +export type ReadinessResponses = { + /** + * Service is ready + */ + 200: ReadinessResponse; +}; + +export type ReadinessResponse2 = ReadinessResponses[keyof ReadinessResponses]; + +export type ListMetricsData = { + body?: never; + path?: never; + query?: { + /** + * Filter metrics by name (substring match) + */ + name?: string; + /** + * Filter metrics by name using regex pattern + */ + name_regex?: string; + /** + * Filter metrics by sensor type (float, integer, string, boolean, location, json, blob, numeric) + */ + type?: string; + }; + url: '/metrics'; +}; + +export type ListMetricsResponses = { + /** + * Metrics catalog in DCAT format + */ + 200: unknown; +}; + +export type PrometheusMetricsData = { + body?: never; + path?: never; + query?: never; + url: '/prometheus/metrics'; +}; + +export type PrometheusMetricsResponses = { + /** + * Prometheus-compatible metrics + */ + 200: string; +}; + +export type PrometheusMetricsResponse = PrometheusMetricsResponses[keyof PrometheusMetricsResponses]; + +export type PublishSensorsDataData = { + /** + * Sensor data in SenML JSON, CSV, or Apache Arrow format + */ + body: string; + path?: never; + query?: never; + url: '/publish'; +}; + +export type PublishSensorsDataErrors = { + /** + * Bad Request - invalid data format + */ + 400: AppError; + /** + * Internal Server Error + */ + 500: AppError; +}; + +export type PublishSensorsDataError = PublishSensorsDataErrors[keyof PublishSensorsDataErrors]; + +export type PublishSensorsDataResponses = { + /** + * Data ingested successfully + */ + 200: string; +}; + +export type PublishSensorsDataResponse = PublishSensorsDataResponses[keyof PublishSensorsDataResponses]; + +export type ListSeriesData = { + body?: never; + path?: never; + query?: { + /** + * Filter series by metric name + */ + metric?: string; + /** + * PromQL-style label selector (e.g., '{env="prod",region=~"us.*"}') + */ + selector?: string; + }; + url: '/series'; +}; + +export type ListSeriesResponses = { + /** + * Time series catalog in DCAT format + */ + 200: unknown; +}; + +export type GetSeriesDataData = { + body?: never; + path: { + /** + * UUID of the series + */ + series_uuid: string; + }; + query?: { + /** + * Output format: senml, csv, or jsonl (default: senml) + */ + format?: string; + /** + * Start datetime in ISO 8601 format (e.g., '2024-01-15T10:30:00Z') + */ + start?: string; + /** + * End datetime in ISO 8601 format (e.g., '2024-01-15T11:00:00Z') + */ + end?: string; + /** + * Maximum number of samples (default: 10,000,000) + */ + limit?: number; + }; + url: '/series/{series_uuid}'; +}; + +export type GetSeriesDataErrors = { + /** + * Invalid format + */ + 400: unknown; + /** + * Series not found + */ + 404: unknown; +}; + +export type GetSeriesDataResponses = { + /** + * Series data in requested format + */ + 200: unknown; +}; diff --git a/frontend/src/components/HealthBadge.test.tsx b/frontend/src/components/HealthBadge.test.tsx new file mode 100644 index 00000000..e2e7a9b3 --- /dev/null +++ b/frontend/src/components/HealthBadge.test.tsx @@ -0,0 +1,53 @@ +import { render, screen, waitFor } from '@testing-library/react'; +import { describe, it, expect, vi } from 'vitest'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import type { ReactNode } from 'react'; +import { HealthBadge } from '../components/HealthBadge'; + +const mockReadiness = vi.fn(); +vi.mock('../client', () => ({ + readiness: (...args: unknown[]) => mockReadiness(...args), +})); + +function createWrapper() { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + return function Wrapper({ children }: { children: ReactNode }) { + return ( + {children} + ); + }; +} + +describe('HealthBadge', () => { + it('shows "Connected" when backend is healthy', async () => { + mockReadiness.mockResolvedValue({ + data: { status: 'ready', database: 'sqlite' }, + }); + + render(, { wrapper: createWrapper() }); + expect(await screen.findByText('Connected')).toBeInTheDocument(); + }); + + it('shows "Disconnected" when backend returns error', async () => { + mockReadiness.mockResolvedValue({ + data: undefined, + error: { status: 'error' }, + }); + + render(, { wrapper: createWrapper() }); + await waitFor(() => { + expect(screen.getByText('Disconnected')).toBeInTheDocument(); + }); + }); + + it('shows "Disconnected" on network failure', async () => { + mockReadiness.mockRejectedValue(new Error('Network error')); + + render(, { wrapper: createWrapper() }); + await waitFor(() => { + expect(screen.getByText('Disconnected')).toBeInTheDocument(); + }); + }); +}); diff --git a/frontend/src/components/HealthBadge.tsx b/frontend/src/components/HealthBadge.tsx new file mode 100644 index 00000000..e13fdb8d --- /dev/null +++ b/frontend/src/components/HealthBadge.tsx @@ -0,0 +1,29 @@ +import { useQuery } from '@tanstack/react-query'; +import { readiness } from '../client'; +import type { ReadinessResponse } from '../client'; + +export function HealthBadge() { + const { data, isError } = useQuery({ + queryKey: ['health'], + queryFn: async () => { + const result = await readiness(); + if (result.error) { + throw new Error('Backend unreachable'); + } + return result.data as ReadinessResponse; + }, + refetchInterval: 30000, + retry: false, + }); + + const isHealthy = data?.status === 'ready' && !isError; + + return ( +
+ + + {isHealthy ? 'Connected' : 'Disconnected'} + +
+ ); +} diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx new file mode 100644 index 00000000..c808846f --- /dev/null +++ b/frontend/src/components/Layout.tsx @@ -0,0 +1,44 @@ +import { Outlet } from 'react-router-dom'; +import { HealthBadge } from './HealthBadge'; + +export function Layout() { + return ( +
+
+
+
+
+ + + + + SensApp + +
+ Sensor Data Explorer +
+
+ +
+
+
+ +
+ +
+
+ ); +} diff --git a/frontend/src/components/MetricsTable.test.tsx b/frontend/src/components/MetricsTable.test.tsx new file mode 100644 index 00000000..d16c8216 --- /dev/null +++ b/frontend/src/components/MetricsTable.test.tsx @@ -0,0 +1,167 @@ +import { render, screen, within } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import type { ReactNode } from 'react'; +import { MetricsTable } from '../components/MetricsTable'; +import { useSelectionStore } from '../stores/useSelectionStore'; + +// Mock the generated API client at the module level +const mockListMetrics = vi.fn(); +vi.mock('../client', () => ({ + listMetrics: (...args: unknown[]) => mockListMetrics(...args), +})); + +function createWrapper() { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + return function Wrapper({ children }: { children: ReactNode }) { + return ( + {children} + ); + }; +} + +const sampleMetrics = { + '@context': {}, + '@type': 'dcat:Catalog', + '@id': 'test', + 'dct:title': 'Test', + 'dct:description': 'Test catalog', + 'dct:publisher': { '@type': 'foaf:Organization', 'foaf:name': 'SensApp' }, + 'dcat:dataset': [ + { + '@type': 'dcat:Dataset', + '@id': 'cpu_usage', + 'dct:identifier': 'metric:cpu_usage', + 'dct:title': 'cpu_usage', + 'dct:description': 'CPU usage metric', + 'dcat:keyword': ['metric', 'float', 'host'], + 'sensor:type': 'float', + 'sensor:seriesCount': 3, + 'sensor:labelDimensions': ['host', 'region'], + 'dcat:distribution': [], + }, + { + '@type': 'dcat:Dataset', + '@id': 'memory_total', + 'dct:identifier': 'metric:memory_total', + 'dct:title': 'memory_total', + 'dct:description': 'Memory total', + 'dcat:keyword': ['metric', 'integer'], + 'sensor:type': 'integer', + 'sensor:seriesCount': 1, + 'sensor:labelDimensions': [], + 'dcat:distribution': [], + }, + ], +}; + +describe('MetricsTable', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockListMetrics.mockResolvedValue({ data: sampleMetrics }); + // Reset store state + useSelectionStore.setState({ + selectedMetric: null, + selectedSeries: [], + labelFilter: '', + }); + }); + + it('shows loading state while fetching metrics', () => { + mockListMetrics.mockReturnValue(new Promise(() => {})); // never resolves + render(, { wrapper: createWrapper() }); + expect(screen.getByText('Loading...')).toBeInTheDocument(); + }); + + it('renders metrics with type badges and series counts', async () => { + render(, { wrapper: createWrapper() }); + + // Wait for metrics to load + expect(await screen.findByText('cpu_usage')).toBeInTheDocument(); + expect(screen.getByText('memory_total')).toBeInTheDocument(); + + // Check type badges are in the table rows + const table = screen.getByRole('table'); + expect(within(table).getByText('float')).toBeInTheDocument(); + expect(within(table).getByText('integer')).toBeInTheDocument(); + + // Check series counts + expect(screen.getByText('3')).toBeInTheDocument(); + expect(screen.getByText('1')).toBeInTheDocument(); + + // Check label dimensions + expect(screen.getByText('host')).toBeInTheDocument(); + expect(screen.getByText('region')).toBeInTheDocument(); + }); + + it('selects a metric when clicking a row and updates store', async () => { + const user = userEvent.setup(); + render(, { wrapper: createWrapper() }); + + const cpuRow = await screen.findByText('cpu_usage'); + await user.click(cpuRow); + + expect(useSelectionStore.getState().selectedMetric).toBe('cpu_usage'); + }); + + it('deselects a metric when clicking the already-selected row', async () => { + const user = userEvent.setup(); + useSelectionStore.setState({ selectedMetric: 'cpu_usage' }); + + render(, { wrapper: createWrapper() }); + + const cpuRow = await screen.findByText('cpu_usage'); + await user.click(cpuRow); + + expect(useSelectionStore.getState().selectedMetric).toBeNull(); + }); + + it('shows error message with readable text on failure', async () => { + mockListMetrics.mockResolvedValue({ + data: undefined, + error: { BadRequest: 'Invalid type filter' }, + }); + + render(, { wrapper: createWrapper() }); + + expect(await screen.findByText(/Failed to load metrics/)).toBeInTheDocument(); + expect(screen.getByText(/Invalid type filter/)).toBeInTheDocument(); + }); + + it('shows empty state when no metrics match', async () => { + mockListMetrics.mockResolvedValue({ + data: { ...sampleMetrics, 'dcat:dataset': [] }, + }); + + render(, { wrapper: createWrapper() }); + expect(await screen.findByText('No metrics found')).toBeInTheDocument(); + }); + + it('highlights the selected metric row', async () => { + useSelectionStore.setState({ selectedMetric: 'cpu_usage' }); + + render(, { wrapper: createWrapper() }); + const cpuRow = (await screen.findByText('cpu_usage')).closest('tr')!; + expect(cpuRow.className).toContain('bg-primary'); + }); + + it('passes filter values to the API call', async () => { + const user = userEvent.setup(); + render(, { wrapper: createWrapper() }); + + // Wait for initial load + await screen.findByText('cpu_usage'); + + // Type in the search box + const searchInput = screen.getByPlaceholderText('Filter by name...'); + await user.type(searchInput, 'cpu'); + + // The API should be called with the name filter + // (debounce means we check the latest call) + const lastCall = mockListMetrics.mock.calls.at(-1); + expect(lastCall?.[0]?.query?.name).toBe('cpu'); + }); +}); diff --git a/frontend/src/components/MetricsTable.tsx b/frontend/src/components/MetricsTable.tsx new file mode 100644 index 00000000..a2a53b25 --- /dev/null +++ b/frontend/src/components/MetricsTable.tsx @@ -0,0 +1,179 @@ +import { useState } from 'react'; +import { useMetrics } from '../hooks/useMetrics'; +import type { DcatDataset } from '../hooks/useMetrics'; +import { useSelectionStore } from '../stores/useSelectionStore'; + +const SENSOR_TYPES = [ + '', + 'float', + 'integer', + 'string', + 'boolean', + 'location', + 'json', + 'blob', + 'numeric', +]; + +export function MetricsTable() { + const [nameFilter, setNameFilter] = useState(''); + const [typeFilter, setTypeFilter] = useState(''); + const { selectedMetric, setSelectedMetric } = useSelectionStore(); + + const { data, isLoading, error } = useMetrics({ + name: nameFilter || undefined, + type: typeFilter || undefined, + }); + + const metrics = data?.['dcat:dataset'] ?? []; + + function handleSelectMetric(metric: DcatDataset) { + const metricName = metric['dct:title']; + if (selectedMetric === metricName) { + setSelectedMetric(null); + } else { + setSelectedMetric(metricName); + } + } + + function sensorTypeBadgeClass(type: string): string { + switch (type) { + case 'float': + case 'numeric': + return 'badge-primary'; + case 'integer': + return 'badge-secondary'; + case 'string': + return 'badge-accent'; + case 'boolean': + return 'badge-info'; + case 'location': + return 'badge-warning'; + default: + return 'badge-ghost'; + } + } + + return ( +
+
+ setNameFilter(e.target.value)} + /> + + {!isLoading && !error && ( + + {metrics.length} metric{metrics.length !== 1 ? 's' : ''} + + )} +
+ +
+ {isLoading && ( +
+ + Loading... +
+ )} + + {error && ( +
+ + + + Failed to load metrics: {error instanceof Error ? error.message : 'Unknown error'} +
+ )} + + {!isLoading && !error && metrics.length === 0 && ( +
+

No metrics found

+ {(nameFilter || typeFilter) && ( + + )} +
+ )} + + {!isLoading && !error && metrics.length > 0 && ( +
+ + + + + + + + + + + {metrics.map((metric) => { + const name = metric['dct:title']; + const isSelected = selectedMetric === name; + const seriesCount = metric['sensor:seriesCount']; + const dimensions = metric['sensor:labelDimensions'] ?? []; + return ( + handleSelectMetric(metric)} + > + + + + + + ); + })} + +
Metric NameTypeSeriesDimensions
+ {name} + {metric['sensor:unit'] && ( + ({metric['sensor:unit']}) + )} + + + {metric['sensor:type']} + + + {seriesCount ?? '—'} + +
+ {dimensions.slice(0, 5).map((dim) => ( + + {dim} + + ))} + {dimensions.length === 0 && ( + none + )} +
+
+
+ )} +
+
+ ); +} diff --git a/frontend/src/components/SeriesTable.tsx b/frontend/src/components/SeriesTable.tsx new file mode 100644 index 00000000..af311959 --- /dev/null +++ b/frontend/src/components/SeriesTable.tsx @@ -0,0 +1,190 @@ +import { useState } from 'react'; +import { useSeries } from '../hooks/useSeries'; +import type { SeriesDataset } from '../hooks/useSeries'; +import { useSelectionStore } from '../stores/useSelectionStore'; + +function labelsToRecord( + labels?: Array> +): Record { + if (!labels || labels.length === 0) return {}; + return labels.reduce>( + (acc, l) => ({ ...acc, ...l }), + {} + ); +} + +export function SeriesTable() { + const { + selectedMetric, + selectedSeries, + toggleSeries, + labelFilter, + setLabelFilter, + } = useSelectionStore(); + const [selectorInput, setSelectorInput] = useState(''); + + const selector = selectorInput || undefined; + + const { data, isLoading, error } = useSeries({ + metric: selectedMetric || undefined, + selector, + }); + + const series = data?.['dcat:dataset'] ?? []; + + function isSelected(dataset: SeriesDataset): boolean { + return selectedSeries.some( + (s) => s.uuid === dataset['dct:identifier'] + ); + } + + function handleToggle(dataset: SeriesDataset) { + toggleSeries({ + uuid: dataset['dct:identifier'], + name: dataset['dct:title'], + labels: labelsToRecord(dataset['sensor:labels']), + type: dataset['sensor:type'], + }); + } + + // Client-side label text filtering + const filteredSeries = labelFilter + ? series.filter((s) => { + const labelPairs = (s['sensor:labels'] ?? []) + .flatMap((obj) => Object.entries(obj)) + .map(([k, v]) => `${k}=${v}`) + .join(' '); + const searchStr = `${s['dct:title']} ${labelPairs}`.toLowerCase(); + return searchStr.includes(labelFilter.toLowerCase()); + }) + : series; + + if (!selectedMetric) { + return null; + } + + return ( +
+
+ setSelectorInput(e.target.value)} + /> + setLabelFilter(e.target.value)} + /> + {selectedSeries.length > 0 && ( + + {selectedSeries.length} selected + + )} +
+ +
+ {isLoading && ( +
+ + Loading... +
+ )} + + {error && ( +
+ + + + Failed to load series: {error instanceof Error ? error.message : 'Unknown error'} +
+ )} + + {!isLoading && !error && filteredSeries.length === 0 && ( +
+

+ {series.length === 0 ? 'No series found for this metric' : 'No series match the current filter'} +

+
+ )} + + {!isLoading && !error && filteredSeries.length > 0 && ( +
+ + + + + + + + + + + {filteredSeries.map((s) => { + const checked = isSelected(s); + return ( + handleToggle(s)} + > + + + + + + ); + })} + +
+ Select + Series IDTypeLabels
+ handleToggle(s)} + onClick={(e) => e.stopPropagation()} + /> + + {s['dct:identifier']} + + + {s['sensor:type']} + + +
+ {(s['sensor:labels'] ?? []).map((labelObj, i) => + Object.entries(labelObj).map(([k, v]) => ( + + {k}= + {v} + + )) + )} + {(!s['sensor:labels'] || s['sensor:labels'].length === 0) && ( + no labels + )} +
+
+
+ {filteredSeries.length} series + {filteredSeries.length !== series.length && ` (${series.length} total)`} +
+
+ )} +
+
+ ); +} diff --git a/frontend/src/components/TimeRangeSelector.test.tsx b/frontend/src/components/TimeRangeSelector.test.tsx new file mode 100644 index 00000000..294e6f1a --- /dev/null +++ b/frontend/src/components/TimeRangeSelector.test.tsx @@ -0,0 +1,52 @@ +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { describe, it, expect, beforeEach } from 'vitest'; +import { TimeRangeSelector } from '../components/TimeRangeSelector'; +import { useSelectionStore } from '../stores/useSelectionStore'; + +describe('TimeRangeSelector', () => { + beforeEach(() => { + // Reset store to a known time range + useSelectionStore.setState({ + timeRange: { + start: '2025-01-01T00:00:00.000Z', + end: '2025-01-01T01:00:00.000Z', + }, + }); + }); + + it('renders all preset buttons', () => { + render(); + for (const label of ['15m', '1h', '6h', '24h', '7d', '30d']) { + expect(screen.getByText(label)).toBeInTheDocument(); + } + }); + + it('updates time range in store when clicking a preset', async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByText('1h')); + + const { timeRange } = useSelectionStore.getState(); + const diffMs = new Date(timeRange.end).getTime() - new Date(timeRange.start).getTime(); + expect(diffMs).toBeCloseTo(60 * 60 * 1000, -3); // ~1 hour + }); + + it('clicking "24h" sets a 24-hour range', async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByText('24h')); + + const { timeRange } = useSelectionStore.getState(); + const diffMs = new Date(timeRange.end).getTime() - new Date(timeRange.start).getTime(); + expect(diffMs).toBeCloseTo(24 * 60 * 60 * 1000, -3); + }); + + it('renders from and to datetime inputs', () => { + render(); + expect(screen.getByText('from')).toBeInTheDocument(); + expect(screen.getByText('to')).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/components/TimeRangeSelector.tsx b/frontend/src/components/TimeRangeSelector.tsx new file mode 100644 index 00000000..62f1e2d8 --- /dev/null +++ b/frontend/src/components/TimeRangeSelector.tsx @@ -0,0 +1,72 @@ +import { useSelectionStore } from '../stores/useSelectionStore'; + +const PRESETS = [ + { label: '15m', minutes: 15 }, + { label: '1h', minutes: 60 }, + { label: '6h', minutes: 360 }, + { label: '24h', minutes: 1440 }, + { label: '7d', minutes: 10080 }, + { label: '30d', minutes: 43200 }, +]; + +export function TimeRangeSelector() { + const { timeRange, setTimeRange } = useSelectionStore(); + + function handlePreset(minutes: number) { + const end = new Date(); + const start = new Date(end.getTime() - minutes * 60 * 1000); + setTimeRange(start.toISOString(), end.toISOString()); + } + + function handleStartChange(value: string) { + if (value) { + setTimeRange(new Date(value).toISOString(), timeRange.end); + } + } + + function handleEndChange(value: string) { + if (value) { + setTimeRange(timeRange.start, new Date(value).toISOString()); + } + } + + function toLocalDatetime(iso: string): string { + const d = new Date(iso); + const offset = d.getTimezoneOffset(); + const local = new Date(d.getTime() - offset * 60 * 1000); + return local.toISOString().slice(0, 16); + } + + return ( +
+
+ {PRESETS.map((preset) => ( + + ))} +
+ +
+ from + handleStartChange(e.target.value)} + /> + to + handleEndChange(e.target.value)} + /> +
+
+ ); +} diff --git a/frontend/src/components/TimeSeriesChart.tsx b/frontend/src/components/TimeSeriesChart.tsx new file mode 100644 index 00000000..d393279b --- /dev/null +++ b/frontend/src/components/TimeSeriesChart.tsx @@ -0,0 +1,177 @@ +import { useMemo } from 'react'; +import ReactECharts from 'echarts-for-react'; +import { useQueries } from '@tanstack/react-query'; +import { getSeriesData } from '../client'; +import { useSelectionStore } from '../stores/useSelectionStore'; +import { extractErrorMessage } from '../api/clientConfig'; +import type { SenMLRecord } from '../hooks/useSeriesData'; + +// Color palette for multiple series +const COLORS = [ + '#3b82f6', '#ef4444', '#10b981', '#f59e0b', '#8b5cf6', + '#ec4899', '#06b6d4', '#84cc16', '#f97316', '#6366f1', +]; + +function buildSeriesLabel(name: string, labels: Record): string { + const labelStr = Object.entries(labels) + .map(([k, v]) => `${k}="${v}"`) + .join(', '); + return labelStr ? `${name}{${labelStr}}` : name; +} + +function parseSenMLToTimeSeries( + records: SenMLRecord[] +): Array<[number, number]> { + let baseName = ''; + let baseTime = 0; + + const points: Array<[number, number]> = []; + + for (const rec of records) { + if (rec.bn !== undefined) baseName = rec.bn; + if (rec.bt !== undefined) baseTime = rec.bt; + + // We only care about re-assigning baseName for multi-record packs + void baseName; + + const time = (baseTime + (rec.t ?? 0)) * 1000; // Convert to ms + const value = rec.v ?? (rec.vs ? parseFloat(rec.vs) : NaN); + + if (!isNaN(value) && isFinite(time)) { + points.push([time, value]); + } + } + + return points.sort((a, b) => a[0] - b[0]); +} + +export function TimeSeriesChart() { + const { selectedSeries, timeRange } = useSelectionStore(); + + const queries = useQueries({ + queries: selectedSeries.map((s, index) => ({ + queryKey: ['seriesData', s.uuid, timeRange] as const, + queryFn: async () => { + const result = await getSeriesData({ + path: { series_uuid: s.uuid }, + query: { + format: 'senml', + start: timeRange.start, + end: timeRange.end, + }, + }); + if (result.error) { + throw new Error(extractErrorMessage(result.error)); + } + return { + records: result.data as unknown as SenMLRecord[], + series: s, + colorIndex: index, + }; + }, + enabled: !!s.uuid, + })), + }); + + const isLoading = queries.some((q) => q.isLoading); + const errors = queries.filter((q) => q.error); + + const option = useMemo(() => { + const seriesData = queries + .filter((q) => q.data) + .map((q) => { + const { records, series, colorIndex } = q.data!; + const points = parseSenMLToTimeSeries(records); + return { + name: buildSeriesLabel(series.name, series.labels), + type: 'line' as const, + data: points, + smooth: false, + showSymbol: points.length < 100, + symbol: 'circle', + symbolSize: 3, + lineStyle: { width: 1.5 }, + color: COLORS[colorIndex % COLORS.length], + }; + }); + + return { + tooltip: { + trigger: 'axis' as const, + axisPointer: { + type: 'cross' as const, + }, + }, + legend: { + data: seriesData.map((s) => s.name), + type: 'scroll' as const, + bottom: 0, + }, + grid: { + top: 40, + right: 40, + bottom: 60, + left: 60, + }, + xAxis: { + type: 'time' as const, + min: new Date(timeRange.start).getTime(), + max: new Date(timeRange.end).getTime(), + }, + yAxis: { + type: 'value' as const, + scale: true, + }, + dataZoom: [ + { + type: 'inside' as const, + start: 0, + end: 100, + }, + { + type: 'slider' as const, + start: 0, + end: 100, + bottom: 30, + height: 20, + }, + ], + toolbox: { + feature: { + saveAsImage: {}, + dataZoom: {}, + restore: {}, + }, + }, + series: seriesData, + }; + }, [queries, timeRange]); + + if (selectedSeries.length === 0) { + return null; + } + + return ( +
+ {isLoading && ( +
+ + Loading chart data... +
+ )} + + {errors.length > 0 && ( +
+ Some series failed to load: {errors.map(q => q.error instanceof Error ? q.error.message : 'Unknown error').join(', ')} +
+ )} + + +
+ ); +} diff --git a/frontend/src/hooks/useMetrics.ts b/frontend/src/hooks/useMetrics.ts new file mode 100644 index 00000000..f49d22e7 --- /dev/null +++ b/frontend/src/hooks/useMetrics.ts @@ -0,0 +1,58 @@ +import { useQuery } from '@tanstack/react-query'; +import { listMetrics } from '../client'; +import { extractErrorMessage } from '../api/clientConfig'; + +export interface DcatDataset { + '@type': string; + '@id': string; + 'dct:identifier': string; + 'dct:title': string; + 'dct:description': string; + 'dcat:keyword': string[]; + 'sensor:type': string; + 'sensor:seriesCount'?: number; + 'sensor:labelDimensions'?: string[]; + 'sensor:unit'?: string; + 'dcat:distribution': Array<{ + '@type': string; + 'dcat:accessURL'?: string; + 'dcat:downloadURL'?: string; + 'dcat:mediaType': string; + }>; +} + +export interface DcatCatalog { + '@context': Record; + '@type': string; + '@id': string; + 'dct:title': string; + 'dct:description': string; + 'dct:publisher': { + '@type': string; + 'foaf:name': string; + }; + 'dcat:dataset': DcatDataset[]; +} + +export function useMetrics(filters?: { + name?: string; + nameRegex?: string; + type?: string; +}) { + return useQuery({ + queryKey: ['metrics', filters], + queryFn: async () => { + const result = await listMetrics({ + query: { + name: filters?.name, + name_regex: filters?.nameRegex, + type: filters?.type, + }, + }); + if (result.error) { + throw new Error(extractErrorMessage(result.error)); + } + return result.data as unknown as DcatCatalog; + }, + }); +} diff --git a/frontend/src/hooks/useSeries.ts b/frontend/src/hooks/useSeries.ts new file mode 100644 index 00000000..00ef9e97 --- /dev/null +++ b/frontend/src/hooks/useSeries.ts @@ -0,0 +1,59 @@ +import { useQuery } from '@tanstack/react-query'; +import { listSeries } from '../client'; +import { extractErrorMessage } from '../api/clientConfig'; + +export interface SeriesLabel { + [key: string]: string; +} + +export interface SeriesDataset { + '@type': string; + '@id': string; + 'dct:identifier': string; + 'dct:title': string; + 'dct:description': string; + 'sensor:type': string; + 'sensor:unit'?: string; + 'sensor:labels'?: SeriesLabel[]; + 'dcat:distribution': Array<{ + '@type'?: string; + 'dcat:downloadURL': string; + 'dcat:mediaType': string; + }>; +} + +export interface SeriesCatalog { + '@context': Record; + '@type': string; + '@id': string; + 'dct:title': string; + 'dct:description': string; + 'dcat:dataset': SeriesDataset[]; + 'hydra:view'?: { + '@type': string; + 'hydra:next': string; + 'hydra:itemsPerPage': number; + }; +} + +export function useSeries(filters?: { + metric?: string; + selector?: string; +}) { + return useQuery({ + queryKey: ['series', filters], + queryFn: async () => { + const result = await listSeries({ + query: { + metric: filters?.metric, + selector: filters?.selector, + }, + }); + if (result.error) { + throw new Error(extractErrorMessage(result.error)); + } + return result.data as unknown as SeriesCatalog; + }, + enabled: !!filters?.metric || !!filters?.selector, + }); +} diff --git a/frontend/src/hooks/useSeriesData.ts b/frontend/src/hooks/useSeriesData.ts new file mode 100644 index 00000000..5c889260 --- /dev/null +++ b/frontend/src/hooks/useSeriesData.ts @@ -0,0 +1,46 @@ +import { useQuery } from '@tanstack/react-query'; +import { getSeriesData } from '../client'; +import { extractErrorMessage } from '../api/clientConfig'; + +export interface SenMLRecord { + bn?: string; + bt?: number; + bu?: string; + n?: string; + t?: number; + u?: string; + v?: number; + vs?: string; + vb?: boolean; + vd?: string; +} + +export function useSeriesData( + seriesUuid: string | undefined, + options?: { + start?: string; + end?: string; + format?: string; + limit?: number; + } +) { + return useQuery({ + queryKey: ['seriesData', seriesUuid, options], + queryFn: async () => { + const result = await getSeriesData({ + path: { series_uuid: seriesUuid! }, + query: { + format: options?.format || 'senml', + start: options?.start, + end: options?.end, + limit: options?.limit, + }, + }); + if (result.error) { + throw new Error(extractErrorMessage(result.error)); + } + return result.data as unknown as SenMLRecord[]; + }, + enabled: !!seriesUuid, + }); +} diff --git a/frontend/src/index.css b/frontend/src/index.css new file mode 100644 index 00000000..49cd2852 --- /dev/null +++ b/frontend/src/index.css @@ -0,0 +1,44 @@ +@import "tailwindcss"; +@import "@fontsource/roboto"; +@import "@fontsource/roboto/500.css"; +@import "@fontsource/roboto/700.css"; + +@plugin "daisyui" { + themes: light --default, dark; + logs: false; +} + +@plugin "daisyui/theme" { + name: "light"; + default: true; + color-scheme: light; + --color-primary: #2563eb; + --color-primary-content: #ffffff; + --color-secondary: #7c3aed; + --color-secondary-content: #ffffff; + --color-accent: #059669; + --color-accent-content: #ffffff; + --color-neutral: #1f2937; + --color-base-100: #ffffff; + --color-base-200: #f8fafc; + --color-base-300: #e2e8f0; +} + +@theme { + --font-sans: Roboto, ui-sans-serif, system-ui, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji"; +} + +html, body, #root { + height: 100%; + overflow: hidden; +} + +/* Smooth transitions for interactive elements */ +.transition-card { + transition: box-shadow 0.2s ease, border-color 0.2s ease; +} + +/* Better table row hover */ +tr.cursor-pointer:hover td { + background-color: oklch(var(--color-primary) / 0.04); +} diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx new file mode 100644 index 00000000..b9af088b --- /dev/null +++ b/frontend/src/main.tsx @@ -0,0 +1,29 @@ +import { StrictMode } from 'react' +import { createRoot } from 'react-dom/client' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { BrowserRouter } from 'react-router-dom' +import './index.css' +import App from './App.tsx' +import { configureApiClient } from './api/clientConfig' + +// Configure API client +configureApiClient() + +const queryClient = new QueryClient({ + defaultOptions: { + queries: { + staleTime: 30_000, + retry: 1, + }, + }, +}) + +createRoot(document.getElementById('root')!).render( + + + + + + + , +) diff --git a/frontend/src/pages/ExplorerPage.tsx b/frontend/src/pages/ExplorerPage.tsx new file mode 100644 index 00000000..9142b151 --- /dev/null +++ b/frontend/src/pages/ExplorerPage.tsx @@ -0,0 +1,90 @@ +import { MetricsTable } from '../components/MetricsTable'; +import { SeriesTable } from '../components/SeriesTable'; +import { TimeSeriesChart } from '../components/TimeSeriesChart'; +import { TimeRangeSelector } from '../components/TimeRangeSelector'; +import { useSelectionStore } from '../stores/useSelectionStore'; + +export function ExplorerPage() { + const { selectedMetric, selectedSeries, clearSelectedSeries } = + useSelectionStore(); + + return ( +
+ {/* Chart + Time Range — always on top, à la InfluxDB */} +
+
+

Chart

+ {selectedSeries.length > 0 && ( + + {selectedSeries.length} series + + )} +
+ +
+
+
+ {selectedSeries.length > 0 ? ( + + ) : ( +
+ + + +

+ {selectedMetric + ? 'Select series below to visualize data' + : 'Select a metric, then choose series to chart'} +

+
+ )} +
+
+ + {/* Metrics + Series side-by-side below the chart */} +
+ {/* Metrics panel */} +
+
+

Metrics

+ +
+ +
+
+ + {/* Series panel */} +
+
+

+ {selectedMetric ? ( + <> + Series{' '} + {selectedMetric} + + ) : ( + 'Series' + )} +

+ +
+
+ {selectedMetric ? ( + + ) : ( +
+

Select a metric to browse series

+
+ )} +
+
+
+
+ ); +} diff --git a/frontend/src/stores/useSelectionStore.test.ts b/frontend/src/stores/useSelectionStore.test.ts new file mode 100644 index 00000000..22b96406 --- /dev/null +++ b/frontend/src/stores/useSelectionStore.test.ts @@ -0,0 +1,108 @@ +import { renderHook, act } from '@testing-library/react'; +import { describe, it, expect } from 'vitest'; +import { useSelectionStore } from '../stores/useSelectionStore'; + +describe('useSelectionStore', () => { + beforeEach(() => { + const { setSelectedMetric, clearSelectedSeries, setLabelFilter } = + useSelectionStore.getState(); + setSelectedMetric(null); + clearSelectedSeries(); + setLabelFilter(''); + }); + + it('should have sensible default state', () => { + const { result } = renderHook(() => useSelectionStore()); + expect(result.current.selectedMetric).toBeNull(); + expect(result.current.selectedSeries).toEqual([]); + expect(result.current.labelFilter).toBe(''); + // Default time range should be approximately 1 hour ending at "now" + const start = new Date(result.current.timeRange.start).getTime(); + const end = new Date(result.current.timeRange.end).getTime(); + expect(end - start).toBeCloseTo(60 * 60 * 1000, -3); + }); + + it('should clear series selection when switching metrics', () => { + const { result } = renderHook(() => useSelectionStore()); + + // Select a series first + act(() => { + result.current.toggleSeries({ + uuid: 'uuid-1', + name: 'cpu', + labels: { host: 'a' }, + type: 'float', + }); + }); + expect(result.current.selectedSeries).toHaveLength(1); + + // Switching metric should clear selected series + act(() => { + result.current.setSelectedMetric('memory'); + }); + expect(result.current.selectedMetric).toBe('memory'); + expect(result.current.selectedSeries).toEqual([]); + }); + + it('should toggle series on and off by uuid', () => { + const { result } = renderHook(() => useSelectionStore()); + + const series1 = { uuid: 'uuid-1', name: 'cpu', labels: { host: 'a' }, type: 'float' }; + const series2 = { uuid: 'uuid-2', name: 'cpu', labels: { host: 'b' }, type: 'float' }; + + // Add two series + act(() => { + result.current.toggleSeries(series1); + }); + act(() => { + result.current.toggleSeries(series2); + }); + expect(result.current.selectedSeries).toHaveLength(2); + + // Remove the first by toggling again + act(() => { + result.current.toggleSeries(series1); + }); + expect(result.current.selectedSeries).toHaveLength(1); + expect(result.current.selectedSeries[0].uuid).toBe('uuid-2'); + }); + + it('should deselect metric when clicking the same one', () => { + const { result } = renderHook(() => useSelectionStore()); + + act(() => { + result.current.setSelectedMetric('cpu'); + }); + expect(result.current.selectedMetric).toBe('cpu'); + + act(() => { + result.current.setSelectedMetric(null); + }); + expect(result.current.selectedMetric).toBeNull(); + }); + + it('should update time range independently', () => { + const { result } = renderHook(() => useSelectionStore()); + + act(() => { + result.current.setTimeRange('2025-01-01T00:00:00Z', '2025-01-02T00:00:00Z'); + }); + expect(result.current.timeRange.start).toBe('2025-01-01T00:00:00Z'); + expect(result.current.timeRange.end).toBe('2025-01-02T00:00:00Z'); + }); + + it('clearSelectedSeries should empty the array', () => { + const { result } = renderHook(() => useSelectionStore()); + + act(() => { + result.current.toggleSeries({ uuid: '1', name: 'a', labels: {}, type: 'float' }); + result.current.toggleSeries({ uuid: '2', name: 'b', labels: {}, type: 'float' }); + }); + expect(result.current.selectedSeries).toHaveLength(2); + + act(() => { + result.current.clearSelectedSeries(); + }); + expect(result.current.selectedSeries).toHaveLength(0); + }); +}); diff --git a/frontend/src/stores/useSelectionStore.ts b/frontend/src/stores/useSelectionStore.ts new file mode 100644 index 00000000..ddc186c8 --- /dev/null +++ b/frontend/src/stores/useSelectionStore.ts @@ -0,0 +1,63 @@ +import { create } from 'zustand'; + +interface SelectedSeries { + uuid: string; + name: string; + labels: Record; + type: string; +} + +interface SelectionState { + selectedMetric: string | null; + selectedSeries: SelectedSeries[]; + timeRange: { + start: string; + end: string; + }; + labelFilter: string; + setSelectedMetric: (metric: string | null) => void; + toggleSeries: (series: SelectedSeries) => void; + clearSelectedSeries: () => void; + setTimeRange: (start: string, end: string) => void; + setLabelFilter: (filter: string) => void; +} + +function defaultTimeRange() { + const end = new Date(); + const start = new Date(end.getTime() - 60 * 60 * 1000); // 1 hour ago + return { + start: start.toISOString(), + end: end.toISOString(), + }; +} + +export const useSelectionStore = create((set) => ({ + selectedMetric: null, + selectedSeries: [], + timeRange: defaultTimeRange(), + labelFilter: '', + + setSelectedMetric: (metric) => + set({ selectedMetric: metric, selectedSeries: [] }), + + toggleSeries: (series) => + set((state) => { + const exists = state.selectedSeries.find((s) => s.uuid === series.uuid); + if (exists) { + return { + selectedSeries: state.selectedSeries.filter( + (s) => s.uuid !== series.uuid + ), + }; + } + return { + selectedSeries: [...state.selectedSeries, series], + }; + }), + + clearSelectedSeries: () => set({ selectedSeries: [] }), + + setTimeRange: (start, end) => set({ timeRange: { start, end } }), + + setLabelFilter: (filter) => set({ labelFilter: filter }), +})); diff --git a/frontend/src/test/setup.tsx b/frontend/src/test/setup.tsx new file mode 100644 index 00000000..a3baa90c --- /dev/null +++ b/frontend/src/test/setup.tsx @@ -0,0 +1,15 @@ +import '@testing-library/jest-dom/vitest'; + +// Mock echarts-for-react globally since it requires canvas in jsdom +vi.mock('echarts-for-react', () => ({ + default: (props: { option: unknown }) => { + const seriesCount = Array.isArray((props.option as Record)?.series) + ? ((props.option as Record).series as unknown[]).length + : 0; + return ( +
+ Chart +
+ ); + }, +})); diff --git a/frontend/tsconfig.app.json b/frontend/tsconfig.app.json new file mode 100644 index 00000000..2e154d31 --- /dev/null +++ b/frontend/tsconfig.app.json @@ -0,0 +1,29 @@ +{ + "compilerOptions": { + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo", + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "types": ["vite/client"], + "skipLibCheck": true, + + /* Bundler mode */ + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + + /* Linting */ + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "erasableSyntaxOnly": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true + }, + "include": ["src"], + "exclude": ["src/**/*.test.ts", "src/**/*.test.tsx", "src/test"] +} diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json new file mode 100644 index 00000000..01490aab --- /dev/null +++ b/frontend/tsconfig.json @@ -0,0 +1,8 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.app.json" }, + { "path": "./tsconfig.node.json" }, + { "path": "./tsconfig.test.json" } + ] +} diff --git a/frontend/tsconfig.node.json b/frontend/tsconfig.node.json new file mode 100644 index 00000000..8a67f62f --- /dev/null +++ b/frontend/tsconfig.node.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo", + "target": "ES2023", + "lib": ["ES2023"], + "module": "ESNext", + "types": ["node"], + "skipLibCheck": true, + + /* Bundler mode */ + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "moduleDetection": "force", + "noEmit": true, + + /* Linting */ + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "erasableSyntaxOnly": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true + }, + "include": ["vite.config.ts"] +} diff --git a/frontend/tsconfig.test.json b/frontend/tsconfig.test.json new file mode 100644 index 00000000..4e6a8edd --- /dev/null +++ b/frontend/tsconfig.test.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.test.tsbuildinfo", + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "types": ["vitest/globals", "vite/client"], + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "erasableSyntaxOnly": true, + "noFallthroughCasesInSwitch": true + }, + "include": ["src/**/*.test.ts", "src/**/*.test.tsx", "src/test"] +} diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts new file mode 100644 index 00000000..fa211c79 --- /dev/null +++ b/frontend/vite.config.ts @@ -0,0 +1,43 @@ +import { defineConfig } from 'vite' +import react from '@vitejs/plugin-react-swc' +import tailwindcss from '@tailwindcss/vite' + +// https://vite.dev/config/ +export default defineConfig({ + plugins: [react(), tailwindcss()], + build: { + rollupOptions: { + output: { + manualChunks: { + echarts: ['echarts', 'echarts-for-react'], + react: ['react', 'react-dom', 'react-router-dom'], + query: ['@tanstack/react-query'], + }, + }, + }, + }, + server: { + proxy: { + '/api': { + target: 'http://localhost:3000', + changeOrigin: true, + }, + '/metrics': { + target: 'http://localhost:3000', + changeOrigin: true, + }, + '/series': { + target: 'http://localhost:3000', + changeOrigin: true, + }, + '/health': { + target: 'http://localhost:3000', + changeOrigin: true, + }, + '/docs': { + target: 'http://localhost:3000', + changeOrigin: true, + }, + }, + }, +}) diff --git a/frontend/vitest.config.ts b/frontend/vitest.config.ts new file mode 100644 index 00000000..67bc81a7 --- /dev/null +++ b/frontend/vitest.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from 'vitest/config' +import react from '@vitejs/plugin-react-swc' + +export default defineConfig({ + plugins: [react()], + test: { + globals: true, + environment: 'jsdom', + setupFiles: './src/test/setup.tsx', + css: true, + include: ['src/**/*.{test,spec}.{ts,tsx}'], + }, +}) diff --git a/ideas/.keep b/ideas/.keep new file mode 100644 index 00000000..e69de29b diff --git a/ideas/rrdcached-follow-ups.md b/ideas/rrdcached-follow-ups.md new file mode 100644 index 00000000..4e0cc041 --- /dev/null +++ b/ideas/rrdcached-follow-ups.md @@ -0,0 +1,9 @@ +# RRDCached Follow-Ups + +These are optional follow-ups for RRDCached. They are not part of the active roadmap unless the backend is promoted beyond its current experimental role. + +## Ideas + +- Support additional RRD consolidation functions where query semantics justify it +- Revisit cleanup and isolation semantics if RRDCached is ever expected to participate in broader shared backend tests +- Consider a separate metadata sidecar if richer sensor metadata becomes important for RRDCached use cases diff --git a/memories/repo/testing.md b/memories/repo/testing.md new file mode 100644 index 00000000..3c807c56 --- /dev/null +++ b/memories/repo/testing.md @@ -0,0 +1,3 @@ +- ClickHouse integration tests and the ClickHouse Rust client use the HTTP endpoint on port 8123; using port 9000 in `TEST_DATABASE_URL` causes migrations/tests to fail. +- This workspace currently has a broken `target` symlink; use `CARGO_TARGET_DIR=/tmp/sensapp-target` for local Cargo validation unless the symlink is fixed. +- RRDCached cannot run the full generic backend test matrix because cleanup and metadata assumptions in shared tests do not hold; use backend-specific RRDCached build/tests instead. diff --git a/python/sensapp/README.md b/python/sensapp/README.md new file mode 100644 index 00000000..38194c9d --- /dev/null +++ b/python/sensapp/README.md @@ -0,0 +1,146 @@ +# sensapp + +Async Python client for the [SensApp](https://github.com/SINTEF/sensapp) HTTP API. + +Arrow-first data exchange, typed DCAT catalog models, and Polars-first query results, powered by [niquests](https://niquests.readthedocs.io/). + +## Why this client + +- async-only API built around `niquests` +- Arrow over the wire, Polars in memory +- typed DCAT catalog models instead of loose dictionaries +- small public surface aimed at common SensApp workflows + +## Install + +Local editable install: + +```bash +cd python/sensapp +uv pip install -e '.[dev]' +``` + +Install directly from GitHub before PyPI publication: + +```bash +uv pip install 'git+https://github.com/SINTEF/sensapp.git@main#subdirectory=python/sensapp' +``` + +Set a server URL once and let the examples use it: + +```bash +export SENSAPP_URL=http://127.0.0.1:3000 +``` + +## Quick example + +```python +import asyncio + +from sensapp import SensAppClient + + +async def main() -> None: + async with SensAppClient.from_env() as client: + await client.publish("temperature", 21.5) + series = await client.query_one("temperature[1h]") + print(series.frame) + + +asyncio.run(main()) +``` + +Runnable examples live in `examples/quickstart.py` and `examples/catalog_overview.py`. + +## Publish with explicit timestamps + +```python +from datetime import UTC, datetime +from sensapp import SensAppClient, SamplePoint + +async with SensAppClient() as client: + await client.publish("temperature", [ + SamplePoint(datetime(2026, 4, 13, 12, 0, tzinfo=UTC), 21.5), + SamplePoint(datetime(2026, 4, 13, 12, 1, tzinfo=UTC), 21.7), + ]) +``` + +## Publish from Polars + +```python +from datetime import UTC, datetime + +import polars as pl +from sensapp import SensAppClient, build_upload_table_from_polars + +frame = pl.DataFrame( + { + "timestamp": [ + datetime(2026, 4, 13, 12, 0, tzinfo=UTC), + datetime(2026, 4, 13, 12, 1, tzinfo=UTC), + ], + "value": [21.5, 21.7], + } +) + +async with SensAppClient() as client: + await client.publish("temperature", frame) + +table = build_upload_table_from_polars( + frame.with_columns( + sensor_name=pl.lit("temperature"), + sensor_id=pl.lit("sensor-123"), + ) +) +``` + +For `publish()`, the sensor name still comes from the method argument. If you want the Polars frame itself to carry a uniform `sensor_name` and optional `sensor_id`, use `build_upload_table_from_polars()` directly. + +## Query results + +`query()` returns a list of `TimeSeries` objects. Each series stores metadata once and exposes a Polars DataFrame with only `timestamp` and `value`. + +If you expect exactly one result, use `query_one()` for a simpler and stricter happy path. + +```python +async with SensAppClient() as client: + results = await client.query('{__name__="temperature",room="lab"}[6h]') + + for item in results: + print(item.name, item.sensor_type, item.labels) + print(item.frame) + print(item.to_pandas()) +``` + +## Typed DCAT catalogs + +```python +async with SensAppClient() as client: + catalog = await client.list_metrics() + for metric in catalog.metrics: + print(metric.name, metric.sensor_type, metric.series_count) + + series_catalog = await client.list_series(metric="temperature") + for series in series_catalog.series: + data = await client.get_series(series.uuid) + print(data.frame) +``` + +## Public API + +- `SensAppClient` +- `SamplePoint` +- `TimeSeries` +- `build_upload_table()` +- `build_upload_table_from_polars()` +- `serialize_arrow_table()` and `read_arrow_table()` +- `__version__` + +## Quality checks + +```bash +cd python/sensapp +uv run ruff check . +uv run ruff format --check . +uv run pytest +``` diff --git a/python/sensapp/examples/catalog_overview.py b/python/sensapp/examples/catalog_overview.py new file mode 100644 index 00000000..f0c4f78c --- /dev/null +++ b/python/sensapp/examples/catalog_overview.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import asyncio + +from sensapp import SensAppClient + + +async def main() -> None: + async with SensAppClient.from_env() as client: + metrics = await client.list_metrics() + print("metrics:") + for metric in metrics.metrics: + print(f"- {metric.name} ({metric.sensor_type}) x{metric.series_count}") + + series_catalog = await client.list_series(limit=5) + print("\nseries:") + for series in series_catalog.series: + print(f"- {series.uuid}: {series.name} {series.labels}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/sensapp/examples/quickstart.py b/python/sensapp/examples/quickstart.py new file mode 100644 index 00000000..5a4dca38 --- /dev/null +++ b/python/sensapp/examples/quickstart.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +import asyncio + +from sensapp import SensAppClient + + +async def main() -> None: + async with SensAppClient.from_env() as client: + await client.publish("temperature", 21.5) + series = await client.query_one("temperature[1h]") + print(series.frame) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/sensapp/pyproject.toml b/python/sensapp/pyproject.toml new file mode 100644 index 00000000..344fddc8 --- /dev/null +++ b/python/sensapp/pyproject.toml @@ -0,0 +1,70 @@ +[build-system] +requires = ["hatchling>=1.29.0"] +build-backend = "hatchling.build" + +[project] +name = "sensapp" +version = "0.1.0" +description = "Async Python client for the SensApp HTTP API — Arrow-first data exchange" +readme = "README.md" +license = { text = "Apache-2.0" } +requires-python = ">=3.11" +authors = [ + { name = "SINTEF" } +] +keywords = ["sensapp", "timeseries", "sensors", "arrow", "pyarrow", "async"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Framework :: AsyncIO", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Scientific/Engineering :: Information Analysis", +] +dependencies = [ + "niquests>=3,<4", + "polars>=1,<2", + "pyarrow>=18,<25", +] + +[project.optional-dependencies] +pandas = [ + "pandas>=2.2,<3", +] +dev = [ + "pandas>=2.2,<3", + "pytest>=8,<10", + "pytest-asyncio>=0.24,<1", + "ruff>=0.9,<1", +] + +[project.urls] +Homepage = "https://github.com/SINTEF/sensapp" +Repository = "https://github.com/SINTEF/sensapp" +Documentation = "https://github.com/SINTEF/sensapp/tree/main/python/sensapp" +Issues = "https://github.com/SINTEF/sensapp/issues" + +[tool.hatch.build.targets.wheel] +packages = ["src/sensapp"] + +[tool.ruff] +line-length = 88 +target-version = "py311" +src = ["src", "tests"] + +[tool.ruff.format] +docstring-code-format = true + +[tool.ruff.lint] +select = ["B", "E", "F", "I", "UP", "RUF"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +asyncio_mode = "auto" +markers = [ + "integration: tests that require a live SensApp server", +] diff --git a/python/sensapp/src/sensapp/__init__.py b/python/sensapp/src/sensapp/__init__.py new file mode 100644 index 00000000..f31b21ee --- /dev/null +++ b/python/sensapp/src/sensapp/__init__.py @@ -0,0 +1,53 @@ +"""Async Python client for the SensApp HTTP API.""" + +from importlib.metadata import PackageNotFoundError, version + +from ._arrow import ( + build_upload_table, + build_upload_table_from_polars, + read_arrow_table, + serialize_arrow_table, +) +from ._exceptions import SensAppError, SensAppHTTPError, SensAppValidationError +from ._models import ( + Distribution, + HealthStatus, + MetricInfo, + MetricsCatalog, + ReadinessStatus, + SamplePoint, + SensorType, + SeriesCatalog, + SeriesInfo, + TimeSeries, + UploadValue, +) +from .client import SensAppClient + +try: + __version__ = version("sensapp") +except PackageNotFoundError: + __version__ = "0.0.0" + +__all__ = [ + "Distribution", + "HealthStatus", + "MetricInfo", + "MetricsCatalog", + "ReadinessStatus", + "SamplePoint", + "SensAppClient", + "SensAppError", + "SensAppHTTPError", + "SensAppValidationError", + "SensorType", + "SeriesCatalog", + "SeriesInfo", + "TimeSeries", + "UploadValue", + "__version__", + "build_upload_table", + "build_upload_table_from_polars", + "read_arrow_table", + "serialize_arrow_table", +] diff --git a/python/sensapp/src/sensapp/_arrow.py b/python/sensapp/src/sensapp/_arrow.py new file mode 100644 index 00000000..653de821 --- /dev/null +++ b/python/sensapp/src/sensapp/_arrow.py @@ -0,0 +1,427 @@ +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping, Sequence +from datetime import UTC, datetime +from decimal import Decimal +from typing import Any +from uuid import UUID + +import polars as pl +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.ipc as ipc + +from ._exceptions import SensAppValidationError +from ._models import ( + SamplePoint, + SensorType, + TimeSeries, + TimestampLike, + UploadSensorType, +) + + +def serialize_arrow_table(table: pa.Table | pa.RecordBatch) -> bytes: + if isinstance(table, pa.RecordBatch): + table = pa.Table.from_batches([table]) + + sink = pa.BufferOutputStream() + options = _stream_write_options() + with ipc.new_stream(sink, table.schema, options=options) as writer: + writer.write_table(table) + return sink.getvalue().to_pybytes() + + +def read_arrow_table(data: bytes) -> pa.Table: + source = pa.py_buffer(data) + try: + return ipc.open_stream(source).read_all() + except (pa.ArrowInvalid, OSError): + return ipc.open_file(source).read_all() + + +def decode_query_time_series(data: bytes) -> list[TimeSeries]: + table = read_arrow_table(data) + if table.num_rows == 0: + return [] + + if "sensor_id" not in table.column_names: + return [decode_single_time_series(data)] + + ordered_sensor_ids: list[str] = [] + seen: set[str] = set() + for sensor_id in table.column("sensor_id").to_pylist(): + if sensor_id not in seen: + seen.add(sensor_id) + ordered_sensor_ids.append(sensor_id) + + return [_extract_multi_series(table, sensor_id) for sensor_id in ordered_sensor_ids] + + +def decode_single_time_series(data: bytes) -> TimeSeries: + table = read_arrow_table(data) + compact_table = _compact_single_series_table(table) + metadata = _decode_schema_metadata(table.schema.metadata) + + sensor_id = metadata.get("sensapp.sensor.uuid") or _first_scalar(table, "sensor_id") + sensor_name = metadata.get("sensapp.sensor.name") or _first_scalar( + table, "sensor_name" + ) + sensor_type = metadata.get("sensapp.sensor.type") or _first_scalar( + table, "sensor_type" + ) + if sensor_type is None: + sensor_type = _infer_sensor_type_from_arrow( + compact_table.schema.field("value").type + ) + + return TimeSeries( + sensor_id=sensor_id or "", + name=sensor_name or sensor_id or "", + sensor_type=sensor_type, + unit=metadata.get("sensapp.sensor.unit") or _first_scalar(table, "unit"), + labels=_parse_labels_json( + metadata.get("sensapp.sensor.labels") or _first_scalar(table, "labels") + ), + frame=pl.from_arrow(compact_table), + ) + + +def build_upload_table( + sensor_name: str, + samples: Sequence[SamplePoint | tuple[TimestampLike, Any] | Mapping[str, Any]], + *, + sensor_id: UUID | str | None = None, + sensor_type: UploadSensorType | None = None, +) -> pa.Table: + normalized = [_normalize_sample(s) for s in samples] + if not normalized: + raise SensAppValidationError("samples must not be empty") + + inferred_type = sensor_type or _infer_sensor_type(normalized[0][1]) + timestamps = [_coerce_timestamp(ts) for ts, _ in normalized] + values = [v for _, v in normalized] + + value_array = _build_value_array(values, inferred_type) + columns: dict[str, pa.Array] = { + "timestamp": pa.array(timestamps, type=pa.timestamp("us", tz="UTC")), + "value": value_array, + "sensor_name": pa.array([sensor_name] * len(values), type=pa.string()), + } + + if sensor_id is not None: + columns["sensor_id"] = pa.array( + [str(sensor_id)] * len(values), type=pa.string() + ) + + return pa.table(columns) + + +def build_upload_table_from_polars( + frame: pl.DataFrame, + *, + sensor_name: str | None = None, + sensor_id: UUID | str | None = None, + sensor_type: UploadSensorType | None = None, +) -> pa.Table: + if frame.is_empty(): + raise SensAppValidationError("frame must not be empty") + + if "timestamp" not in frame.columns or "value" not in frame.columns: + raise SensAppValidationError( + "frame must contain 'timestamp' and 'value' columns" + ) + + resolved_sensor_name = sensor_name or _resolve_uniform_string_column( + frame, + "sensor_name", + ) + if resolved_sensor_name is None: + raise SensAppValidationError( + "pass sensor_name or include a uniform 'sensor_name' column in the frame" + ) + + resolved_sensor_id = sensor_id or _resolve_uniform_string_column( + frame, + "sensor_id", + ) + + samples = [ + SamplePoint(timestamp=row["timestamp"], value=row["value"]) + for row in frame.select(["timestamp", "value"]).to_dicts() + ] + return build_upload_table( + resolved_sensor_name, + samples, + sensor_id=resolved_sensor_id, + sensor_type=sensor_type, + ) + + +# ── internal helpers ──────────────────────────────────────────── + + +def _normalize_sample( + sample: SamplePoint | tuple[TimestampLike, Any] | Mapping[str, Any], +) -> tuple[TimestampLike, Any]: + if isinstance(sample, SamplePoint): + return sample.timestamp, sample.value + + if isinstance(sample, tuple) and len(sample) == 2: + return sample[0], sample[1] + + if isinstance(sample, Mapping): + if "timestamp" not in sample or "value" not in sample: + raise SensAppValidationError( + "mapping samples must contain 'timestamp' and 'value' keys" + ) + return sample["timestamp"], sample["value"] + + raise SensAppValidationError( + "samples must be SamplePoint, (timestamp, value) tuples, or mappings" + ) + + +def _coerce_timestamp(value: TimestampLike) -> datetime: + if isinstance(value, datetime): + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + + if isinstance(value, (int, float)): + return datetime.fromtimestamp(value, tz=UTC) + + if isinstance(value, str): + candidate = value.replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(candidate) + except ValueError as exc: + raise SensAppValidationError(f"invalid timestamp string: {value}") from exc + if parsed.tzinfo is None: + return parsed.replace(tzinfo=UTC) + return parsed.astimezone(UTC) + + raise SensAppValidationError(f"unsupported timestamp type: {type(value)!r}") + + +def _infer_sensor_type(value: Any) -> UploadSensorType: + if isinstance(value, bool): + return "boolean" + if isinstance(value, int): + return "integer" + if isinstance(value, float): + return "float" + if isinstance(value, Decimal): + return "numeric" + if isinstance(value, str): + return "string" + if isinstance(value, (bytes, bytearray, memoryview)): + return "blob" + if _is_location_value(value): + return "location" + raise SensAppValidationError(f"cannot infer sensor type for {type(value)!r}") + + +def _build_value_array( + values: Iterable[Any], + sensor_type: UploadSensorType, +) -> pa.Array: + materialized = list(values) + + if sensor_type == "boolean": + return pa.array([bool(v) for v in materialized], type=pa.bool_()) + if sensor_type == "integer": + return pa.array([int(v) for v in materialized], type=pa.int64()) + if sensor_type == "float": + return pa.array([float(v) for v in materialized], type=pa.float64()) + if sensor_type == "numeric": + return pa.array(materialized, type=pa.decimal128(38, 18)) + if sensor_type == "string": + return pa.array([str(v) for v in materialized], type=pa.string()) + if sensor_type == "blob": + return pa.array([bytes(v) for v in materialized], type=pa.binary()) + if sensor_type == "location": + latitudes: list[float] = [] + longitudes: list[float] = [] + for v in materialized: + lat, lon = _coerce_location(v) + latitudes.append(lat) + longitudes.append(lon) + return pa.StructArray.from_arrays( + [ + pa.array(latitudes, type=pa.float64()), + pa.array(longitudes, type=pa.float64()), + ], + fields=[ + pa.field("latitude", pa.float64()), + pa.field("longitude", pa.float64()), + ], + ) + + raise SensAppValidationError(f"unsupported sensor type: {sensor_type}") + + +def _is_location_value(value: Any) -> bool: + return ( + isinstance(value, tuple) + and len(value) == 2 + and all(isinstance(item, (int, float)) for item in value) + ) or (isinstance(value, Mapping) and "latitude" in value and "longitude" in value) + + +def _coerce_location(value: Any) -> tuple[float, float]: + if isinstance(value, tuple) and len(value) == 2: + return float(value[0]), float(value[1]) + if isinstance(value, Mapping) and "latitude" in value and "longitude" in value: + return float(value["latitude"]), float(value["longitude"]) + raise SensAppValidationError( + "location values must be (latitude, longitude) tuples or mappings" + ) + + +def _stream_write_options() -> ipc.IpcWriteOptions: + try: + return ipc.IpcWriteOptions(compression="lz4") + except (TypeError, ValueError): + return ipc.IpcWriteOptions() + + +def _extract_multi_series(table: pa.Table, sensor_id: str) -> TimeSeries: + mask = pc.equal(table.column("sensor_id"), sensor_id) + group = table.filter(mask) + + sensor_name = _first_scalar(group, "sensor_name") or sensor_id + sensor_type = _first_scalar(group, "sensor_type") or _first_scalar(group, "type") + if sensor_type is None: + raise SensAppValidationError("query response is missing 'sensor_type'") + + compact_table = _compact_multi_series_table(group, sensor_type) + return TimeSeries( + sensor_id=sensor_id, + name=sensor_name, + sensor_type=sensor_type, + unit=_first_scalar(group, "unit"), + labels=_parse_labels_json(_first_scalar(group, "labels")), + frame=pl.from_arrow(compact_table), + ) + + +def _compact_single_series_table(table: pa.Table) -> pa.Table: + if "timestamp" not in table.column_names or "value" not in table.column_names: + raise SensAppValidationError( + "single-series Arrow response must contain 'timestamp' and 'value' columns" + ) + return table.select(["timestamp", "value"]) + + +def _compact_multi_series_table(table: pa.Table, sensor_type: SensorType) -> pa.Table: + timestamp_column = table.column("timestamp").combine_chunks() + + if "value" in table.column_names: + value_column = table.column("value").combine_chunks() + elif sensor_type == "integer": + value_column = _require_column(table, "integer_value").combine_chunks() + elif sensor_type == "numeric": + value_column = _require_column(table, "numeric_value").combine_chunks() + elif sensor_type == "float": + value_column = _require_column(table, "float_value").combine_chunks() + elif sensor_type == "string": + value_column = _require_column(table, "string_value").combine_chunks() + elif sensor_type == "boolean": + value_column = _require_column(table, "boolean_value").combine_chunks() + elif sensor_type == "blob": + value_column = _require_column(table, "blob_value").combine_chunks() + elif sensor_type == "json": + value_column = _require_column(table, "json_value").combine_chunks() + elif sensor_type == "location": + latitude = _require_column(table, "latitude").combine_chunks() + longitude = _require_column(table, "longitude").combine_chunks() + value_column = pa.StructArray.from_arrays( + [latitude, longitude], + fields=[ + pa.field("latitude", pa.float64(), nullable=True), + pa.field("longitude", pa.float64(), nullable=True), + ], + ) + else: + raise SensAppValidationError( + f"unsupported sensor type in query response: {sensor_type}" + ) + + return pa.table([timestamp_column, value_column], names=["timestamp", "value"]) + + +def _require_column(table: pa.Table, column_name: str) -> pa.ChunkedArray: + if column_name not in table.column_names: + raise SensAppValidationError( + f"query response is missing required column '{column_name}'" + ) + return table.column(column_name) + + +def _decode_schema_metadata( + metadata: Mapping[bytes, bytes] | None, +) -> dict[str, str]: + if metadata is None: + return {} + return { + key.decode("utf-8"): value.decode("utf-8") for key, value in metadata.items() + } + + +def _parse_labels_json(payload: str | None) -> dict[str, str]: + if not payload: + return {} + try: + parsed = json.loads(payload) + except json.JSONDecodeError as exc: + raise SensAppValidationError(f"invalid labels JSON: {payload}") from exc + if not isinstance(parsed, dict): + raise SensAppValidationError("labels metadata must decode to an object") + return {str(key): str(value) for key, value in parsed.items()} + + +def _first_scalar(table: pa.Table, column_name: str) -> str | None: + if column_name not in table.column_names or table.num_rows == 0: + return None + scalar = table.column(column_name)[0] + if not scalar.is_valid: + return None + value = scalar.as_py() + return None if value is None else str(value) + + +def _infer_sensor_type_from_arrow(data_type: pa.DataType) -> SensorType: + if pa.types.is_int64(data_type): + return "integer" + if pa.types.is_decimal(data_type): + return "numeric" + if pa.types.is_float64(data_type): + return "float" + if pa.types.is_string(data_type): + return "string" + if pa.types.is_boolean(data_type): + return "boolean" + if pa.types.is_struct(data_type): + return "location" + if pa.types.is_binary(data_type): + return "blob" + raise SensAppValidationError(f"unsupported Arrow value type: {data_type}") + + +def _resolve_uniform_string_column( + frame: pl.DataFrame, + column_name: str, +) -> str | None: + if column_name not in frame.columns: + return None + + values = frame.get_column(column_name).drop_nulls().unique().to_list() + if not values: + return None + if len(values) != 1: + raise SensAppValidationError( + f"frame column '{column_name}' must contain a single value" + ) + return str(values[0]) diff --git a/python/sensapp/src/sensapp/_exceptions.py b/python/sensapp/src/sensapp/_exceptions.py new file mode 100644 index 00000000..352b74e5 --- /dev/null +++ b/python/sensapp/src/sensapp/_exceptions.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +class SensAppError(Exception): + """Base class for sensapp errors.""" + + +class SensAppValidationError(SensAppError): + """Raised when client-side input validation fails.""" + + +@dataclass(slots=True) +class SensAppHTTPError(SensAppError): + status_code: int + message: str + details: dict[str, Any] | None = None + + def __str__(self) -> str: + return f"SensApp HTTP {self.status_code}: {self.message}" diff --git a/python/sensapp/src/sensapp/_models.py b/python/sensapp/src/sensapp/_models.py new file mode 100644 index 00000000..194ffa1f --- /dev/null +++ b/python/sensapp/src/sensapp/_models.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from decimal import Decimal +from typing import TYPE_CHECKING, Any, Literal, TypeAlias +from urllib.parse import parse_qs, urlparse + +import polars as pl +import pyarrow as pa + +if TYPE_CHECKING: + import pandas as pd + +SensorType: TypeAlias = Literal[ + "boolean", + "integer", + "float", + "numeric", + "string", + "blob", + "location", + "json", +] + +UploadSensorType: TypeAlias = Literal[ + "boolean", + "integer", + "float", + "numeric", + "string", + "blob", + "location", +] + +UploadValue: TypeAlias = ( + bool | int | float | Decimal | str | bytes | tuple[float, float] +) + +TimestampLike: TypeAlias = datetime | str | int | float + + +@dataclass(slots=True) +class SamplePoint: + timestamp: TimestampLike + value: UploadValue + + +@dataclass(slots=True, frozen=True) +class HealthStatus: + status: str + + +@dataclass(slots=True, frozen=True) +class ReadinessStatus: + status: str + database: str + error: str | None = None + + +@dataclass(slots=True) +class TimeSeries: + sensor_id: str + name: str + sensor_type: SensorType + frame: pl.DataFrame + labels: dict[str, str] = field(default_factory=dict) + unit: str | None = None + + @property + def rows(self) -> list[dict[str, Any]]: + return self.frame.to_dicts() + + def __len__(self) -> int: + return self.frame.height + + def to_arrow(self) -> pa.Table: + return self.frame.to_arrow() + + def to_polars(self) -> pl.DataFrame: + return self.frame.clone() + + def to_pandas(self) -> pd.DataFrame: + try: + import pandas # noqa: F401 + except ImportError as exc: + raise ImportError( + "Install `sensapp[pandas]` to use TimeSeries.to_pandas()." + ) from exc + + return self.frame.to_pandas(use_pyarrow_extension_array=True) + + +@dataclass(slots=True, frozen=True) +class Distribution: + url: str + media_type: str + format: str + description: str | None = None + + +@dataclass(slots=True, frozen=True) +class MetricInfo: + name: str + identifier: str + description: str + sensor_type: SensorType + series_count: int + label_dimensions: list[str] + unit: str | None = None + keywords: list[str] = field(default_factory=list) + distributions: list[Distribution] = field(default_factory=list) + + +@dataclass(slots=True, frozen=True) +class SeriesInfo: + uuid: str + name: str + description: str + sensor_type: SensorType + labels: dict[str, str] = field(default_factory=dict) + unit: str | None = None + keywords: list[str] = field(default_factory=list) + distributions: list[Distribution] = field(default_factory=list) + + +@dataclass(slots=True, frozen=True) +class MetricsCatalog: + metrics: list[MetricInfo] + + +@dataclass(slots=True, frozen=True) +class SeriesCatalog: + series: list[SeriesInfo] + next_bookmark: str | None = None + + +def parse_metrics_catalog(data: dict[str, Any]) -> MetricsCatalog: + metrics: list[MetricInfo] = [] + for ds in data.get("dcat:dataset", []): + metrics.append( + MetricInfo( + name=ds["dct:title"], + identifier=ds["dct:identifier"], + description=ds.get("dct:description", ""), + sensor_type=ds["sensor:type"], + series_count=ds.get("sensor:seriesCount", 0), + label_dimensions=ds.get("sensor:labelDimensions", []), + unit=ds.get("sensor:unit"), + keywords=ds.get("dcat:keyword", []), + distributions=_parse_distributions(ds.get("dcat:distribution", [])), + ) + ) + return MetricsCatalog(metrics=metrics) + + +def parse_series_catalog(data: dict[str, Any]) -> SeriesCatalog: + series: list[SeriesInfo] = [] + for ds in data.get("dcat:dataset", []): + labels: dict[str, str] = {} + for label_obj in ds.get("sensor:labels", []): + if isinstance(label_obj, dict): + labels.update(label_obj) + + series.append( + SeriesInfo( + uuid=ds["dct:identifier"], + name=ds["dct:title"], + description=ds.get("dct:description", ""), + sensor_type=ds["sensor:type"], + labels=labels, + unit=ds.get("sensor:unit"), + keywords=ds.get("dcat:keyword", []), + distributions=_parse_distributions(ds.get("dcat:distribution", [])), + ) + ) + + next_bookmark = None + view = data.get("hydra:view") + if isinstance(view, dict): + next_url = view.get("hydra:next", "") + next_bookmark = _extract_bookmark(next_url) + + return SeriesCatalog(series=series, next_bookmark=next_bookmark) + + +def _parse_distributions(raw: list[dict[str, Any]]) -> list[Distribution]: + result: list[Distribution] = [] + for d in raw: + url = d.get("dcat:downloadURL") or d.get("dcat:accessURL", "") + result.append( + Distribution( + url=url, + media_type=d.get("dcat:mediaType", ""), + format=d.get("dct:format", ""), + description=d.get("dct:description"), + ) + ) + return result + + +def _extract_bookmark(url: str) -> str | None: + parsed = urlparse(url) + params = parse_qs(parsed.query) + bookmarks = params.get("bookmark", []) + return bookmarks[0] if bookmarks else None diff --git a/python/sensapp/src/sensapp/client.py b/python/sensapp/src/sensapp/client.py new file mode 100644 index 00000000..147eb45c --- /dev/null +++ b/python/sensapp/src/sensapp/client.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +import json +import os +from datetime import UTC, datetime +from typing import Any +from uuid import UUID + +import niquests +import polars as pl + +from ._arrow import ( + build_upload_table, + build_upload_table_from_polars, + decode_query_time_series, + decode_single_time_series, + serialize_arrow_table, +) +from ._exceptions import SensAppHTTPError +from ._models import ( + HealthStatus, + MetricsCatalog, + ReadinessStatus, + SamplePoint, + SeriesCatalog, + TimeSeries, + UploadSensorType, + UploadValue, + parse_metrics_catalog, + parse_series_catalog, +) + +_ARROW_CONTENT_TYPE = "application/vnd.apache.arrow.stream" + + +class SensAppClient: + """Async client for the SensApp HTTP API. + + Usage:: + + async with SensAppClient("http://127.0.0.1:3000") as client: + await client.publish("temperature", 21.5) + series = await client.query_one("temperature[1h]") + print(series.frame) + """ + + def __init__( + self, + base_url: str = "http://127.0.0.1:3000", + *, + token: str | None = None, + timeout: float = 10.0, + ) -> None: + self._base_url = base_url.rstrip("/") + self._token = token + self._session = niquests.AsyncSession(timeout=timeout) + + @classmethod + def from_env( + cls, + *, + url_var: str = "SENSAPP_URL", + token_var: str = "SENSAPP_TOKEN", + default_url: str = "http://127.0.0.1:3000", + ) -> SensAppClient: + """Build a client from environment variables.""" + return cls( + os.environ.get(url_var, default_url), + token=os.environ.get(token_var), + ) + + async def close(self) -> None: + await self._session.close() + + async def __aenter__(self) -> SensAppClient: + return self + + async def __aexit__(self, exc_type: object, exc: object, tb: object) -> None: + await self.close() + + async def health_live(self) -> HealthStatus: + data = await self._get_json("/health/live") + return HealthStatus(status=str(data["status"])) + + async def health_ready(self) -> ReadinessStatus: + data = await self._get_json("/health/ready") + return ReadinessStatus( + status=str(data["status"]), + database=str(data["database"]), + error=data.get("error"), + ) + + async def publish( + self, + sensor: str, + value: UploadValue | list[SamplePoint] | pl.DataFrame, + *, + sensor_id: str | UUID | None = None, + sensor_type: UploadSensorType | None = None, + ) -> str: + """Publish one sensor's samples as Arrow IPC.""" + if isinstance(value, pl.DataFrame): + table = build_upload_table_from_polars( + value, + sensor_name=sensor, + sensor_id=sensor_id, + sensor_type=sensor_type, + ) + elif isinstance(value, list): + samples = value + table = build_upload_table( + sensor, + samples, + sensor_id=sensor_id, + sensor_type=sensor_type, + ) + else: + samples = [SamplePoint(datetime.now(UTC), value)] + table = build_upload_table( + sensor, + samples, + sensor_id=sensor_id, + sensor_type=sensor_type, + ) + body = serialize_arrow_table(table) + response = await self._post( + "/publish", + data=body, + headers={"content-type": _ARROW_CONTENT_TYPE}, + ) + return response.text + + async def query(self, query: str) -> list[TimeSeries]: + """Run a query and return all matching series.""" + response = await self._get( + "/api/v1/query", + params={"query": query, "format": "arrow"}, + ) + return decode_query_time_series(response.content) + + async def query_one(self, query: str) -> TimeSeries: + """Run a query that must resolve to exactly one series.""" + series_list = await self.query(query) + if not series_list: + raise ValueError(f"query returned no series: {query}") + if len(series_list) != 1: + raise ValueError( + f"query expected exactly one series, got {len(series_list)}: {query}" + ) + return series_list[0] + + async def list_metrics( + self, + *, + name: str | None = None, + name_regex: str | None = None, + sensor_type: str | None = None, + ) -> MetricsCatalog: + """Fetch the typed DCAT metrics catalog.""" + data = await self._get_json( + "/metrics", + params={ + "name": name, + "name_regex": name_regex, + "type": sensor_type, + }, + ) + return parse_metrics_catalog(data) + + async def list_series( + self, + *, + metric: str | None = None, + selector: str | None = None, + limit: int | None = None, + bookmark: str | None = None, + ) -> SeriesCatalog: + """Fetch the typed DCAT series catalog.""" + data = await self._get_json( + "/series", + params={ + "metric": metric, + "selector": selector, + "limit": limit, + "bookmark": bookmark, + }, + ) + return parse_series_catalog(data) + + async def get_series( + self, + series_uuid: str | UUID, + *, + start: str | None = None, + end: str | None = None, + limit: int | None = None, + step: str | None = None, + aggregation: str | None = None, + simplify: bool | None = None, + simplify_tolerance: float | None = None, + simplify_high_quality: bool | None = None, + ) -> TimeSeries: + """Fetch one series by UUID and decode it into a compact TimeSeries.""" + params: dict[str, Any] = { + "format": "arrow", + "start": start, + "end": end, + "limit": limit, + "step": step, + "aggregation": aggregation, + "simplify": simplify, + "simplify_tolerance": simplify_tolerance, + "simplify_high_quality": simplify_high_quality, + } + response = await self._get(f"/series/{series_uuid}", params=params) + return decode_single_time_series(response.content) + + async def _get_json( + self, + path: str, + *, + params: dict[str, Any] | None = None, + ) -> Any: + response = await self._get(path, params=params) + return response.json() + + async def _get( + self, + path: str, + *, + params: dict[str, Any] | None = None, + ) -> niquests.Response: + url = f"{self._base_url}{path}" + clean_params = ( + {key: value for key, value in params.items() if value is not None} + if params + else None + ) + response = await self._session.get( + url, + params=clean_params, + headers=self._auth_headers(), + ) + _check_response(response) + return response # type: ignore[return-value] + + async def _post( + self, + path: str, + *, + data: bytes | str | None = None, + json_body: Any = None, + headers: dict[str, str] | None = None, + ) -> niquests.Response: + url = f"{self._base_url}{path}" + merged_headers = {**self._auth_headers(), **(headers or {})} + response = await self._session.post( + url, + data=data, + json=json_body, + headers=merged_headers, + ) + _check_response(response) + return response # type: ignore[return-value] + + def _auth_headers(self) -> dict[str, str]: + if self._token: + return {"authorization": f"Bearer {self._token}"} + return {} + + @property + def base_url(self) -> str: + """Return the normalized SensApp base URL for this client.""" + return self._base_url + + +def _check_response(resp: niquests.Response) -> None: + if resp.status_code < 400: + return + + details: dict[str, Any] | None = None + message = resp.text or f"HTTP {resp.status_code}" + try: + payload = resp.json() + except (ValueError, json.JSONDecodeError): + payload = None + + if isinstance(payload, dict): + details = payload + if len(payload) == 1: + key, value = next(iter(payload.items())) + message = f"{key}: {value}" + else: + message = str(payload) + + raise SensAppHTTPError( + status_code=resp.status_code, + message=message, + details=details, + ) diff --git a/python/sensapp/src/sensapp/py.typed b/python/sensapp/src/sensapp/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/python/sensapp/tests/__init__.py b/python/sensapp/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/sensapp/tests/conftest.py b/python/sensapp/tests/conftest.py new file mode 100644 index 00000000..2c55bea5 --- /dev/null +++ b/python/sensapp/tests/conftest.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import json +import os +from dataclasses import dataclass, field +from typing import Any + +import niquests +import pytest + + +@dataclass +class MockResponse: + """Minimal stand-in for ``niquests.Response`` in unit tests.""" + + status_code: int = 200 + _content: bytes = b"" + headers: dict[str, str] = field(default_factory=dict) + + @property + def content(self) -> bytes: + return self._content + + @property + def text(self) -> str: + return self._content.decode("utf-8") + + def json(self) -> Any: + return json.loads(self._content) + + # ── convenience constructors ──────────────────────────────── + + @classmethod + def json_ok(cls, data: Any) -> MockResponse: + return cls(_content=json.dumps(data).encode()) + + @classmethod + def text_ok(cls, text: str) -> MockResponse: + return cls(_content=text.encode()) + + @classmethod + def bytes_ok(cls, data: bytes) -> MockResponse: + return cls(_content=data) + + @classmethod + def error(cls, status_code: int, body: str = "") -> MockResponse: + return cls(status_code=status_code, _content=body.encode()) + + +@pytest.fixture(scope="session") +def live_server_url() -> str: + base_url = os.environ.get("SENSAPP_LIVE_URL", "http://127.0.0.1:3000") + + try: + response = niquests.get(f"{base_url}/health/ready", timeout=1) + except Exception as exc: + pytest.skip(f"live SensApp server not available at {base_url}: {exc}") + + if response.status_code not in {200, 503}: + pytest.skip( + f"live SensApp server at {base_url} returned unexpected status " + f"{response.status_code}" + ) + + return base_url diff --git a/python/sensapp/tests/test_arrow.py b/python/sensapp/tests/test_arrow.py new file mode 100644 index 00000000..4f403fa9 --- /dev/null +++ b/python/sensapp/tests/test_arrow.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from decimal import Decimal +from typing import Any + +import polars as pl +import pyarrow as pa +import pytest + +from sensapp import ( + SamplePoint, + build_upload_table, + build_upload_table_from_polars, + read_arrow_table, + serialize_arrow_table, +) +from sensapp._exceptions import SensAppValidationError + + +def test_build_upload_table_for_float_samples() -> None: + table = build_upload_table( + "temperature", + [ + SamplePoint(datetime(2026, 3, 16, 12, 0, tzinfo=UTC), 21.5), + SamplePoint("2026-03-16T12:01:00Z", 21.7), + ], + sensor_id="9d87123d-9b47-466d-9eda-001c2ecf9c54", + ) + + assert table.column_names == [ + "timestamp", + "value", + "sensor_name", + "sensor_id", + ] + assert table.schema.field("value").type == pa.float64() + assert table["sensor_name"].to_pylist() == ["temperature", "temperature"] + + +def test_build_upload_table_for_numeric_samples() -> None: + table = build_upload_table( + "power_cost", + [ + SamplePoint( + datetime(2026, 3, 16, 12, 0, tzinfo=UTC), + Decimal("12.34"), + ), + SamplePoint( + datetime(2026, 3, 16, 12, 1, tzinfo=UTC), + Decimal("12.56"), + ), + ], + ) + + assert table.schema.field("value").type == pa.decimal128(38, 18) + + +def test_build_upload_table_for_location_samples() -> None: + table = build_upload_table( + "vehicle_position", + [ + SamplePoint( + datetime(2026, 3, 16, 12, 0, tzinfo=UTC), + (63.4305, 10.3951), + ), + ], + ) + + value_field = table.schema.field("value") + assert pa.types.is_struct(value_field.type) + assert table["value"].to_pylist() == [{"latitude": 63.4305, "longitude": 10.3951}] + + +@pytest.mark.parametrize( + ("sample_value", "expected_type"), + [ + (True, pa.bool_()), + (7, pa.int64()), + ("ok", pa.string()), + (b"abc", pa.binary()), + ], +) +def test_build_upload_table_supports_multiple_scalar_types( + sample_value: Any, + expected_type: pa.DataType, +) -> None: + table = build_upload_table( + "generic_sensor", + [ + SamplePoint( + datetime(2026, 3, 16, 12, 0, tzinfo=UTC), + sample_value, + ), + ], + ) + + assert table.schema.field("value").type == expected_type + + +def test_build_upload_table_accepts_mapping_samples() -> None: + table = build_upload_table( + "temperature", + [{"timestamp": "2026-03-16T12:00:00Z", "value": 21.5}], + ) + + assert table["value"].to_pylist() == [21.5] + + +def test_build_upload_table_from_polars_uses_explicit_sensor_name() -> None: + frame = pl.DataFrame( + { + "timestamp": [ + datetime(2026, 3, 16, 12, 0, tzinfo=UTC), + datetime(2026, 3, 16, 12, 1, tzinfo=UTC), + ], + "value": [21.5, 21.7], + } + ) + + table = build_upload_table_from_polars( + frame, + sensor_name="temperature", + sensor_id="sensor-123", + ) + + assert table.column_names == ["timestamp", "value", "sensor_name", "sensor_id"] + assert table["sensor_name"].to_pylist() == ["temperature", "temperature"] + assert table["sensor_id"].to_pylist() == ["sensor-123", "sensor-123"] + + +def test_build_upload_table_from_polars_uses_uniform_sensor_name_column() -> None: + frame = pl.DataFrame( + { + "timestamp": [datetime(2026, 3, 16, 12, 0, tzinfo=UTC)], + "value": [21.5], + "sensor_name": ["temperature"], + } + ) + + table = build_upload_table_from_polars(frame) + + assert table["sensor_name"].to_pylist() == ["temperature"] + + +def test_build_upload_table_from_polars_uses_uniform_sensor_id_column() -> None: + frame = pl.DataFrame( + { + "timestamp": [datetime(2026, 3, 16, 12, 0, tzinfo=UTC)], + "value": [21.5], + "sensor_name": ["temperature"], + "sensor_id": ["sensor-123"], + } + ) + + table = build_upload_table_from_polars(frame) + + assert table["sensor_id"].to_pylist() == ["sensor-123"] + + +def test_build_upload_table_from_polars_rejects_missing_required_columns() -> None: + frame = pl.DataFrame({"value": [21.5]}) + + with pytest.raises(SensAppValidationError, match="timestamp"): + build_upload_table_from_polars(frame, sensor_name="temperature") + + +def test_build_upload_table_from_polars_rejects_multiple_sensor_names() -> None: + frame = pl.DataFrame( + { + "timestamp": [ + datetime(2026, 3, 16, 12, 0, tzinfo=UTC), + datetime(2026, 3, 16, 12, 1, tzinfo=UTC), + ], + "value": [21.5, 21.7], + "sensor_name": ["temperature", "humidity"], + } + ) + + with pytest.raises(SensAppValidationError, match="single value"): + build_upload_table_from_polars(frame) + + +def test_build_upload_table_from_polars_rejects_multiple_sensor_ids() -> None: + frame = pl.DataFrame( + { + "timestamp": [ + datetime(2026, 3, 16, 12, 0, tzinfo=UTC), + datetime(2026, 3, 16, 12, 1, tzinfo=UTC), + ], + "value": [21.5, 21.7], + "sensor_name": ["temperature", "temperature"], + "sensor_id": ["sensor-1", "sensor-2"], + } + ) + + with pytest.raises(SensAppValidationError, match="single value"): + build_upload_table_from_polars(frame) + + +def test_build_upload_table_rejects_mapping_without_required_keys() -> None: + with pytest.raises(SensAppValidationError, match="timestamp"): + build_upload_table("temperature", [{"value": 21.5}]) + + +def test_build_upload_table_rejects_invalid_timestamp_string() -> None: + with pytest.raises(SensAppValidationError, match="invalid timestamp"): + build_upload_table( + "temperature", + [SamplePoint("not-a-timestamp", 21.5)], + ) + + +def test_build_upload_table_rejects_unsupported_value_type() -> None: + with pytest.raises(SensAppValidationError, match="cannot infer"): + build_upload_table( + "temperature", + [ + SamplePoint( + datetime(2026, 3, 16, 12, 0, tzinfo=UTC), + object(), + ) + ], + ) + + +def test_build_upload_table_rejects_empty_samples() -> None: + with pytest.raises(SensAppValidationError): + build_upload_table("temperature", []) + + +def test_arrow_round_trip() -> None: + source = build_upload_table( + "temperature", + [SamplePoint(datetime(2026, 3, 16, 12, 0, tzinfo=UTC), 21.5)], + sensor_id="abc", + ) + + encoded = serialize_arrow_table(source) + decoded = read_arrow_table(encoded) + + assert decoded.to_pylist() == source.to_pylist() + assert encoded != b"" + + +def test_read_arrow_table_rejects_invalid_bytes() -> None: + with pytest.raises((pa.ArrowInvalid, pa.ArrowTypeError, OSError)): + read_arrow_table(b"not-arrow") diff --git a/python/sensapp/tests/test_client.py b/python/sensapp/tests/test_client.py new file mode 100644 index 00000000..98b8755c --- /dev/null +++ b/python/sensapp/tests/test_client.py @@ -0,0 +1,453 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from unittest.mock import AsyncMock + +import polars as pl +import pyarrow as pa +import pytest + +from sensapp import ( + SamplePoint, + SensAppClient, + __version__, + serialize_arrow_table, +) +from sensapp._exceptions import SensAppHTTPError + +from .conftest import MockResponse + +# ── helpers ───────────────────────────────────────────────────── + + +def _mock_client( + token: str | None = None, +) -> tuple[SensAppClient, AsyncMock]: + """Return a client with a fully-mocked session.""" + client = SensAppClient("http://sensapp.test", token=token) + mock_session = AsyncMock() + client._session = mock_session + return client, mock_session + + +def _arrow_response_table() -> pa.Table: + return pa.table( + { + "timestamp": pa.array( + [datetime(2026, 3, 16, 12, 0, tzinfo=UTC)], + type=pa.timestamp("us", tz="UTC"), + ), + "sensor_id": ["sensor-1"], + "sensor_name": ["temperature"], + "sensor_type": ["float"], + "unit": ["degC"], + "labels": ['{"room":"lab"}'], + "float_value": [21.5], + } + ) + + +def _single_series_response_table() -> pa.Table: + schema = pa.schema( + [ + pa.field("timestamp", pa.timestamp("us"), nullable=False), + pa.field("value", pa.float64(), nullable=False), + ], + metadata={ + b"sensapp.sensor.uuid": b"series-uuid", + b"sensapp.sensor.name": b"temperature", + b"sensapp.sensor.type": b"float", + b"sensapp.sensor.unit": b"degC", + b"sensapp.sensor.labels": b'{"room":"lab"}', + }, + ) + return pa.Table.from_arrays( + [ + pa.array( + [datetime(2026, 3, 16, 12, 0, tzinfo=UTC)], + type=pa.timestamp("us"), + ), + pa.array([21.5], type=pa.float64()), + ], + schema=schema, + ) + + +# ── health ────────────────────────────────────────────────────── + + +async def test_health_live_parses_response() -> None: + client, session = _mock_client() + session.get.return_value = MockResponse.json_ok({"status": "ok"}) + + result = await client.health_live() + + assert result.status == "ok" + session.get.assert_called_once() + url = session.get.call_args[0][0] + assert url.endswith("/health/live") + + +async def test_health_ready_parses_response() -> None: + client, session = _mock_client() + session.get.return_value = MockResponse.json_ok( + {"status": "ready", "database": "ok", "error": None} + ) + + result = await client.health_ready() + + assert result.status == "ready" + assert result.database == "ok" + assert result.error is None + + +# ── publish ───────────────────────────────────────────────────── + + +async def test_publish_single_value_sends_arrow() -> None: + client, session = _mock_client() + session.post.return_value = MockResponse.text_ok("ok") + + result = await client.publish("temperature", 21.5) + + assert result == "ok" + session.post.assert_called_once() + call_kwargs = session.post.call_args + assert call_kwargs[1]["headers"]["content-type"] == ( + "application/vnd.apache.arrow.stream" + ) + body = call_kwargs[1]["data"] + payload = pa.ipc.open_stream(pa.py_buffer(body)).read_all() + assert payload.column_names == ["timestamp", "value", "sensor_name"] + + +async def test_publish_sample_list_sends_arrow() -> None: + client, session = _mock_client() + session.post.return_value = MockResponse.text_ok("ok") + + await client.publish( + "temperature", + [ + SamplePoint(datetime(2026, 3, 16, 12, 0, tzinfo=UTC), 21.5), + SamplePoint(datetime(2026, 3, 16, 12, 1, tzinfo=UTC), 21.7), + ], + ) + + body = session.post.call_args[1]["data"] + payload = pa.ipc.open_stream(pa.py_buffer(body)).read_all() + assert payload.num_rows == 2 + + +async def test_publish_polars_frame_sends_arrow() -> None: + client, session = _mock_client() + session.post.return_value = MockResponse.text_ok("ok") + + frame = pl.DataFrame( + { + "timestamp": [ + datetime(2026, 3, 16, 12, 0, tzinfo=UTC), + datetime(2026, 3, 16, 12, 1, tzinfo=UTC), + ], + "value": [21.5, 21.7], + } + ) + + result = await client.publish("temperature", frame, sensor_id="sensor-1") + + assert result == "ok" + body = session.post.call_args[1]["data"] + payload = pa.ipc.open_stream(pa.py_buffer(body)).read_all() + assert payload["sensor_name"].to_pylist() == ["temperature", "temperature"] + assert payload["sensor_id"].to_pylist() == ["sensor-1", "sensor-1"] + + +async def test_publish_with_token_sends_auth_header() -> None: + client, session = _mock_client(token="secret-token") + session.post.return_value = MockResponse.text_ok("ok") + + await client.publish("temperature", 21.5) + + headers = session.post.call_args[1]["headers"] + assert headers["authorization"] == "Bearer secret-token" + + +# ── query ─────────────────────────────────────────────────────── + + +async def test_query_returns_compact_time_series() -> None: + table = _arrow_response_table() + client, session = _mock_client() + session.get.return_value = MockResponse.bytes_ok(serialize_arrow_table(table)) + + result = await client.query("temperature[1h]") + + assert len(result) == 1 + series = result[0] + assert series.sensor_id == "sensor-1" + assert series.name == "temperature" + assert series.sensor_type == "float" + assert series.unit == "degC" + assert series.labels == {"room": "lab"} + assert series.rows == [ + {"timestamp": datetime(2026, 3, 16, 12, 0, tzinfo=UTC), "value": 21.5} + ] + url = session.get.call_args[0][0] + assert "/api/v1/query" in url + + +async def test_query_returns_multiple_series() -> None: + table = _arrow_response_table() + extra = pa.table( + { + "timestamp": pa.array( + [datetime(2026, 3, 16, 12, 1, tzinfo=UTC)], + type=pa.timestamp("us", tz="UTC"), + ), + "sensor_id": ["sensor-2"], + "sensor_name": ["humidity"], + "sensor_type": ["integer"], + "unit": [None], + "labels": ['{"room":"lab"}'], + "float_value": [None], + "integer_value": [45], + } + ) + merged = pa.concat_tables( + [ + table.append_column("integer_value", pa.array([None], type=pa.int64())), + extra, + ], + promote_options="default", + ) + client, session = _mock_client() + session.get.return_value = MockResponse.bytes_ok(serialize_arrow_table(merged)) + + result = await client.query('{__name__=~"temperature|humidity"}[1h]') + + assert [series.name for series in result] == ["temperature", "humidity"] + assert result[1].rows == [ + {"timestamp": datetime(2026, 3, 16, 12, 1, tzinfo=UTC), "value": 45} + ] + + +async def test_query_returns_polars_and_pandas() -> None: + table = _arrow_response_table() + client, session = _mock_client() + session.get.return_value = MockResponse.bytes_ok(serialize_arrow_table(table)) + + [series] = await client.query("temperature[5m]") + + assert isinstance(series.to_polars(), pl.DataFrame) + assert series.to_polars().columns == ["timestamp", "value"] + pandas_frame = series.to_pandas() + assert list(pandas_frame.columns) == ["timestamp", "value"] + + +async def test_query_one_returns_single_series() -> None: + table = _arrow_response_table() + client, session = _mock_client() + session.get.return_value = MockResponse.bytes_ok(serialize_arrow_table(table)) + + series = await client.query_one("temperature[5m]") + + assert series.name == "temperature" + assert series.rows == [ + {"timestamp": datetime(2026, 3, 16, 12, 0, tzinfo=UTC), "value": 21.5} + ] + + +async def test_query_one_rejects_no_series() -> None: + empty = pa.table( + { + "timestamp": pa.array([], type=pa.timestamp("us", tz="UTC")), + "sensor_id": pa.array([], type=pa.string()), + "sensor_name": pa.array([], type=pa.string()), + "sensor_type": pa.array([], type=pa.string()), + "unit": pa.array([], type=pa.string()), + "labels": pa.array([], type=pa.string()), + "float_value": pa.array([], type=pa.float64()), + } + ) + client, session = _mock_client() + session.get.return_value = MockResponse.bytes_ok(serialize_arrow_table(empty)) + + with pytest.raises(ValueError, match="no series"): + await client.query_one("temperature[5m]") + + +async def test_query_one_rejects_multiple_series() -> None: + table = _arrow_response_table() + extra = pa.table( + { + "timestamp": pa.array( + [datetime(2026, 3, 16, 12, 1, tzinfo=UTC)], + type=pa.timestamp("us", tz="UTC"), + ), + "sensor_id": ["sensor-2"], + "sensor_name": ["humidity"], + "sensor_type": ["integer"], + "unit": [None], + "labels": ['{"room":"lab"}'], + "float_value": [None], + "integer_value": [45], + } + ) + merged = pa.concat_tables( + [ + table.append_column("integer_value", pa.array([None], type=pa.int64())), + extra, + ], + promote_options="default", + ) + client, session = _mock_client() + session.get.return_value = MockResponse.bytes_ok(serialize_arrow_table(merged)) + + with pytest.raises(ValueError, match="expected exactly one series"): + await client.query_one('{__name__=~"temperature|humidity"}[1h]') + + +# ── catalog ───────────────────────────────────────────────────── + + +async def test_list_metrics_returns_typed_catalog() -> None: + client, session = _mock_client() + session.get.return_value = MockResponse.json_ok( + { + "@type": "dcat:Catalog", + "dcat:dataset": [ + { + "dct:title": "temperature", + "dct:identifier": "metric:temperature", + "dct:description": "Aggregated metric", + "sensor:type": "float", + "sensor:seriesCount": 2, + "sensor:labelDimensions": ["room"], + "dcat:keyword": ["metric", "float"], + "dcat:distribution": [], + } + ], + } + ) + + catalog = await client.list_metrics() + + assert len(catalog.metrics) == 1 + assert catalog.metrics[0].name == "temperature" + assert catalog.metrics[0].series_count == 2 + + +async def test_list_series_returns_typed_catalog_with_pagination() -> None: + client, session = _mock_client() + session.get.return_value = MockResponse.json_ok( + { + "dcat:dataset": [ + { + "dct:identifier": "uuid-1", + "dct:title": "temperature", + "dct:description": "Sensor data", + "sensor:type": "float", + "sensor:labels": [{"room": "lab"}], + "dcat:keyword": [], + "dcat:distribution": [], + } + ], + "hydra:view": { + "hydra:next": "/series?limit=5&bookmark=page2", + }, + } + ) + + catalog = await client.list_series(metric="temperature", limit=5) + + assert len(catalog.series) == 1 + assert catalog.series[0].uuid == "uuid-1" + assert catalog.series[0].labels == {"room": "lab"} + assert catalog.next_bookmark == "page2" + + +# ── series data ───────────────────────────────────────────────── + + +async def test_get_series_returns_arrow_table() -> None: + table = _single_series_response_table() + client, session = _mock_client() + session.get.return_value = MockResponse.bytes_ok(serialize_arrow_table(table)) + + result = await client.get_series("series-uuid", step="5m") + + assert result.sensor_id == "series-uuid" + assert result.name == "temperature" + assert result.sensor_type == "float" + assert result.unit == "degC" + assert result.labels == {"room": "lab"} + assert result.rows == [{"timestamp": datetime(2026, 3, 16, 12, 0), "value": 21.5}] + params = session.get.call_args[1]["params"] + assert params["format"] == "arrow" + assert params["step"] == "5m" + + +# ── errors ────────────────────────────────────────────────────── + + +async def test_http_error_raised_with_json_body() -> None: + client, session = _mock_client() + session.get.return_value = MockResponse.error( + 400, '{"BadRequest": "Unsupported format"}' + ) + + with pytest.raises(SensAppHTTPError) as exc: + await client.query("temperature") + + assert exc.value.status_code == 400 + assert "BadRequest" in str(exc.value) + + +async def test_http_error_raised_with_plain_text() -> None: + client, session = _mock_client() + session.get.return_value = MockResponse.error(500, "server error") + + with pytest.raises(SensAppHTTPError) as exc: + await client.health_live() + + assert exc.value.status_code == 500 + assert exc.value.details is None + + +# ── from_env ──────────────────────────────────────────────────── + + +async def test_from_env_reads_environment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SENSAPP_URL", "http://from-env.test") + monkeypatch.setenv("SENSAPP_TOKEN", "env-token") + + client = SensAppClient.from_env() + + assert client._base_url == "http://from-env.test" + assert client._token == "env-token" + await client.close() + + +async def test_base_url_property_returns_normalized_value() -> None: + client, _session = _mock_client() + + assert client.base_url == "http://sensapp.test" + await client.close() + + +def test_public_version_is_exposed() -> None: + assert isinstance(__version__, str) + assert __version__ + + +# ── context manager ───────────────────────────────────────────── + + +async def test_async_context_manager_closes_session() -> None: + client, session = _mock_client() + + async with client: + pass + + session.close.assert_awaited_once() diff --git a/python/sensapp/tests/test_integration_live.py b/python/sensapp/tests/test_integration_live.py new file mode 100644 index 00000000..6620a8b8 --- /dev/null +++ b/python/sensapp/tests/test_integration_live.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from uuid import uuid4 + +import polars as pl +import pytest + +from sensapp import ( + SensAppClient, + build_upload_table_from_polars, + serialize_arrow_table, +) + + +@pytest.mark.integration +async def test_live_publish_polars_and_query_timeseries( + live_server_url: str, +) -> None: + sensor_name = f"sdk-polars-{uuid4().hex}" + now = datetime.now(UTC).replace(microsecond=0) + frame = pl.DataFrame( + { + "timestamp": [now - timedelta(seconds=1), now], + "value": [21.5, 21.7], + } + ) + + async with SensAppClient(live_server_url) as client: + await client.publish(sensor_name, frame) + result = await client.query(f'{{__name__="{sensor_name}"}}[1h]') + + assert len(result) == 1 + series = result[0] + assert series.name == sensor_name + assert series.sensor_type == "float" + assert series.to_polars().columns == ["timestamp", "value"] + assert series.to_polars().height == 2 + assert set(series.to_polars().get_column("value").to_list()) == {21.5, 21.7} + + +@pytest.mark.integration +async def test_live_polars_upload_helper_and_get_series( + live_server_url: str, +) -> None: + sensor_name = f"sdk-polars-helper-{uuid4().hex}" + sensor_id = str(uuid4()) + now = datetime.now(UTC).replace(microsecond=0) + frame = pl.DataFrame( + { + "timestamp": [now - timedelta(seconds=1), now], + "value": [45, 46], + "sensor_name": [sensor_name, sensor_name], + "sensor_id": [sensor_id, sensor_id], + } + ) + upload_table = build_upload_table_from_polars(frame) + + async with SensAppClient(live_server_url) as client: + response = await client._post( + "/publish", + data=serialize_arrow_table(upload_table), + headers={"content-type": "application/vnd.apache.arrow.stream"}, + ) + assert response.text == "ok" + + catalog = await client.list_series(metric=sensor_name) + assert catalog.series, "expected at least one series in the catalog" + + series = await client.get_series(catalog.series[0].uuid) + + assert series.sensor_id == sensor_id + assert series.name == sensor_name + assert series.to_polars().columns == ["timestamp", "value"] + assert set(series.to_polars().get_column("value").to_list()) == {45, 46} diff --git a/python/sensapp/tests/test_models.py b/python/sensapp/tests/test_models.py new file mode 100644 index 00000000..20a0e1b7 --- /dev/null +++ b/python/sensapp/tests/test_models.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +from typing import Any + +from sensapp._models import ( + MetricsCatalog, + SeriesCatalog, + parse_metrics_catalog, + parse_series_catalog, +) + + +def _make_metric_dataset(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "@type": "dcat:Dataset", + "@id": "temperature", + "dct:identifier": "metric:temperature", + "dct:title": "temperature", + "dct:description": "Aggregated metric 'temperature'", + "dcat:keyword": ["metric", "aggregated", "time-series", "float"], + "sensor:type": "float", + "sensor:seriesCount": 3, + "sensor:labelDimensions": ["room", "floor"], + "dct:temporal": {"@type": "dct:PeriodOfTime"}, + "dcat:distribution": [ + { + "@type": "dcat:Distribution", + "dcat:accessURL": "/series?metric=temperature", + "dcat:mediaType": "application/json", + "dct:format": "DCAT Series Catalog", + } + ], + } + base.update(overrides) + return base + + +def _make_series_dataset(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "@type": "dcat:Dataset", + "@id": 'temperature{room="lab"}', + "dct:identifier": "abc-123", + "dct:title": "temperature", + "dct:description": "Sensor data from temperature (float)", + "dcat:keyword": ["sensor", "IoT", "time-series", "float", "room"], + "sensor:type": "float", + "sensor:labels": [{"room": "lab"}], + "dct:temporal": {"@type": "dct:PeriodOfTime"}, + "dcat:distribution": [ + { + "@type": "dcat:Distribution", + "dcat:downloadURL": "/series/abc-123?format=senml", + "dcat:mediaType": "application/senml+json", + "dct:format": "SenML JSON", + }, + ], + } + base.update(overrides) + return base + + +def test_parse_metrics_catalog_basic() -> None: + raw = { + "@type": "dcat:Catalog", + "dcat:dataset": [_make_metric_dataset()], + } + catalog = parse_metrics_catalog(raw) + + assert isinstance(catalog, MetricsCatalog) + assert len(catalog.metrics) == 1 + + m = catalog.metrics[0] + assert m.name == "temperature" + assert m.identifier == "metric:temperature" + assert m.sensor_type == "float" + assert m.series_count == 3 + assert m.label_dimensions == ["room", "floor"] + assert m.unit is None + assert len(m.distributions) == 1 + + +def test_parse_metrics_catalog_with_unit() -> None: + raw = { + "dcat:dataset": [_make_metric_dataset(**{"sensor:unit": "°C"})], + } + catalog = parse_metrics_catalog(raw) + + assert catalog.metrics[0].unit == "°C" + + +def test_parse_metrics_catalog_empty() -> None: + catalog = parse_metrics_catalog({"dcat:dataset": []}) + assert catalog.metrics == [] + + +def test_parse_series_catalog_basic() -> None: + raw = { + "@type": "dcat:Catalog", + "dcat:dataset": [_make_series_dataset()], + } + catalog = parse_series_catalog(raw) + + assert isinstance(catalog, SeriesCatalog) + assert len(catalog.series) == 1 + + s = catalog.series[0] + assert s.uuid == "abc-123" + assert s.name == "temperature" + assert s.sensor_type == "float" + assert s.labels == {"room": "lab"} + assert catalog.next_bookmark is None + + +def test_parse_series_catalog_pagination() -> None: + raw = { + "dcat:dataset": [_make_series_dataset()], + "hydra:view": { + "@type": "hydra:PartialCollectionView", + "hydra:next": "/series?limit=10&bookmark=page2&metric=temperature", + "hydra:itemsPerPage": 10, + }, + } + catalog = parse_series_catalog(raw) + + assert catalog.next_bookmark == "page2" + + +def test_parse_series_catalog_no_pagination() -> None: + raw = {"dcat:dataset": [_make_series_dataset()]} + catalog = parse_series_catalog(raw) + + assert catalog.next_bookmark is None + + +def test_parse_series_catalog_multiple_labels() -> None: + ds = _make_series_dataset(**{"sensor:labels": [{"room": "lab"}, {"floor": "2"}]}) + raw = {"dcat:dataset": [ds]} + catalog = parse_series_catalog(raw) + + assert catalog.series[0].labels == {"room": "lab", "floor": "2"} + + +def test_parse_series_catalog_with_unit() -> None: + ds = _make_series_dataset(**{"sensor:unit": "hPa"}) + raw = {"dcat:dataset": [ds]} + catalog = parse_series_catalog(raw) + + assert catalog.series[0].unit == "hPa" diff --git a/settings.toml b/settings.toml index 36162a74..6e3aed48 100644 --- a/settings.toml +++ b/settings.toml @@ -19,6 +19,11 @@ storage_connection_string = "postgres://postgres:postgres@localhost:5432/sensapp # Storage sync timeout in seconds (default: 15) # storage_sync_timeout_seconds = 30 +# JWT Authentication (optional) +# Set to a random string of at least 32 characters to enable JWT auth. +# When unset, all endpoints are open (default — like Prometheus). +# jwt_secret = "change-me-to-a-real-secret-at-least-32-chars" + # InfluxDB Configuration # Use Numeric/Decimal (high precision) for InfluxDB numeric values instead of Float (default: false) # Numeric (Decimal) is precise but slower; Float (f64) is faster but has limited precision diff --git a/src/config/mod.rs b/src/config/mod.rs index 0f8f084b..e9144301 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -5,6 +5,8 @@ use std::{ sync::{Arc, OnceLock}, }; +const MAX_HTTP_BODY_LIMIT_BYTES: u64 = 128 * 1024 * 1024 * 1024; + #[derive(Debug, Config)] pub struct SensAppConfig { #[config(env = "SENSAPP_INSTANCE_ID", default = 0)] @@ -15,7 +17,7 @@ pub struct SensAppConfig { #[config(env = "SENSAPP_ENDPOINT", default = "127.0.0.1")] pub endpoint: IpAddr, - #[config(env = "SENSAPP_HTTP_BODY_LIMIT", default = "10mb")] + #[config(env = "SENSAPP_HTTP_BODY_LIMIT", default = "64MiB")] pub http_body_limit: String, #[config(env = "SENSAPP_HTTP_SERVER_TIMEOUT_SECONDS", default = 30)] @@ -42,6 +44,13 @@ pub struct SensAppConfig { #[config(env = "SENSAPP_INFLUXDB_WITH_NUMERIC", default = false)] pub influxdb_with_numeric: bool, + + /// JWT secret for optional authentication. + /// When set, all protected endpoints require a valid JWT bearer token. + /// Must be at least 32 characters long. + /// When unset, all endpoints are open (no security). + #[config(env = "SENSAPP_JWT_SECRET")] + pub jwt_secret: Option, } impl SensAppConfig { @@ -57,7 +66,7 @@ impl SensAppConfig { pub fn parse_http_body_limit(&self) -> Result { let size = byte_unit::Byte::parse_str(self.http_body_limit.clone(), true)?.as_u64(); - if size > 128 * 1024 * 1024 * 1024 { + if size > MAX_HTTP_BODY_LIMIT_BYTES { anyhow::bail!("Body size is too big: > 128GB"); } Ok(size as usize) @@ -131,26 +140,27 @@ mod tests { #[test] fn test_parse_http_body_limit() { let config = SensAppConfig::load().unwrap(); - assert_eq!(config.parse_http_body_limit().unwrap(), 10000000); + assert_eq!(config.http_body_limit, "64MiB"); + assert_eq!(config.parse_http_body_limit().unwrap(), 67108864); temp_env::with_var("SENSAPP_HTTP_BODY_LIMIT", Some("12345"), || { let config = SensAppConfig::load().unwrap(); assert_eq!(config.parse_http_body_limit().unwrap(), 12345); }); - temp_env::with_var("SENSAPP_HTTP_BODY_LIMIT", Some("10m"), || { + temp_env::with_var("SENSAPP_HTTP_BODY_LIMIT", Some("64m"), || { let config = SensAppConfig::load().unwrap(); - assert_eq!(config.parse_http_body_limit().unwrap(), 10000000); + assert_eq!(config.parse_http_body_limit().unwrap(), 64000000); }); - temp_env::with_var("SENSAPP_HTTP_BODY_LIMIT", Some("10mb"), || { + temp_env::with_var("SENSAPP_HTTP_BODY_LIMIT", Some("64mb"), || { let config = SensAppConfig::load().unwrap(); - assert_eq!(config.parse_http_body_limit().unwrap(), 10000000); + assert_eq!(config.parse_http_body_limit().unwrap(), 64000000); }); - temp_env::with_var("SENSAPP_HTTP_BODY_LIMIT", Some("10MiB"), || { + temp_env::with_var("SENSAPP_HTTP_BODY_LIMIT", Some("64MiB"), || { let config = SensAppConfig::load().unwrap(); - assert_eq!(config.parse_http_body_limit().unwrap(), 10485760); + assert_eq!(config.parse_http_body_limit().unwrap(), 67108864); }); temp_env::with_var("SENSAPP_HTTP_BODY_LIMIT", Some("1.5gb"), || { @@ -171,12 +181,13 @@ mod tests { #[test] fn test_load_configuration() { - assert!(SENSAPP_CONFIG.get().is_none()); + load_configuration().unwrap(); load_configuration().unwrap(); assert!(SENSAPP_CONFIG.get().is_some()); let config = get().unwrap(); - assert_eq!(config.port, 3000); + let config_again = get().unwrap(); + assert!(Arc::ptr_eq(&config, &config_again)); } #[test] diff --git a/src/datamodel/metric.rs b/src/datamodel/metric.rs index f5900aea..c8a1518f 100644 --- a/src/datamodel/metric.rs +++ b/src/datamodel/metric.rs @@ -21,6 +21,7 @@ pub struct Metric { } impl Metric { + #[allow(dead_code)] pub fn new( name: String, sensor_type: SensorType, diff --git a/src/datamodel/sensapp_datetime.rs b/src/datamodel/sensapp_datetime.rs index 37c64b7f..25ac091d 100644 --- a/src/datamodel/sensapp_datetime.rs +++ b/src/datamodel/sensapp_datetime.rs @@ -27,17 +27,10 @@ use hifitime::{UNIX_REF_EPOCH, Unit}; use sqlx::types::time::OffsetDateTime; #[allow(dead_code)] pub fn sensapp_datetime_to_offset_datetime(datetime: &SensAppDateTime) -> Result { - let unix_timestamp = datetime.to_unix_seconds().floor() as i128; - - let duration = datetime.to_et_duration(); - let (_sign, _days, _hours, _minutes, _seconds, miliseconds, microseconds, ns_left) = - duration.decompose(); - let sum_after_seconds: i128 = (miliseconds as i128) * 1_000_000_i128 - + (microseconds as i128) * 1_000_i128 - + ns_left as i128; - - let sum = unix_timestamp * 1_000_000_000_i128 + sum_after_seconds; - Ok(OffsetDateTime::from_unix_timestamp_nanos(sum)?) + let unix_timestamp_us = datetime.to_unix(Unit::Microsecond).floor() as i128; + Ok(OffsetDateTime::from_unix_timestamp_nanos( + unix_timestamp_us * 1_000_i128, + )?) } #[cfg(test)] @@ -84,10 +77,10 @@ mod tests { offset_now.unix_timestamp(), ); - // Compare sub-second precision: extract nanoseconds within the current second - let hifitime_total_nanoseconds = hifitime_now.to_et_duration().total_nanoseconds(); - let hifitime_subsec_nanoseconds = (hifitime_total_nanoseconds % 1_000_000_000) as u32; + // The database path stores timestamps at microsecond precision. + let hifitime_subsec_microseconds = hifitime_now.to_unix(Unit::Microsecond).floor() as i64; + let offset_subsec_microseconds = (offset_now.unix_timestamp_nanos() / 1_000) as i64; - assert_eq!(hifitime_subsec_nanoseconds, offset_now.nanosecond()); + assert_eq!(hifitime_subsec_microseconds, offset_subsec_microseconds); } } diff --git a/src/exporters/arrow/mod.rs b/src/exporters/arrow/mod.rs index 4302d958..3d000e1d 100644 --- a/src/exporters/arrow/mod.rs +++ b/src/exporters/arrow/mod.rs @@ -1,19 +1,20 @@ -use crate::datamodel::{SensAppDateTime, SensorData, TypedSamples}; +use crate::datamodel::{SensAppDateTime, Sensor, SensorData, SensorType, TypedSamples}; use anyhow::Result; use arrow::array::{ - ArrayRef, BinaryBuilder, BooleanBuilder, Decimal128Builder, Float64Builder, Int64Builder, - StringBuilder, TimestampMicrosecondBuilder, + Array, ArrayRef, BinaryBuilder, BooleanBuilder, Decimal128Builder, Float64Builder, + Int64Builder, StringBuilder, StructArray, TimestampMicrosecondBuilder, + builder::StringDictionaryBuilder, }; -use arrow::datatypes::{DataType, Field, Schema, TimeUnit}; +use arrow::datatypes::{DataType, Field, Int32Type, Schema, TimeUnit}; use arrow::record_batch::RecordBatch; -use arrow_ipc::writer::FileWriter; +use arrow_ipc::CompressionType; +use arrow_ipc::writer::{IpcWriteOptions, StreamWriter}; +use std::collections::HashMap; use std::sync::Arc; -/// Converter for SensorData to Apache Arrow format pub struct ArrowConverter; impl ArrowConverter { - /// Convert SensorData to Arrow RecordBatch pub fn to_record_batch(sensor_data: &SensorData) -> Result { let (schema, columns) = Self::convert_sensor_data_to_arrow(sensor_data)?; @@ -21,198 +22,189 @@ impl ArrowConverter { .map_err(|e| anyhow::anyhow!("Failed to create Arrow RecordBatch: {}", e)) } - /// Convert SensorData to Arrow IPC file format (bytes) - pub fn to_arrow_file(sensor_data: &SensorData) -> Result> { + pub fn to_arrow_stream(sensor_data: &SensorData) -> Result> { let batch = Self::to_record_batch(sensor_data)?; - Self::record_batch_to_arrow_file(&batch) + Self::record_batch_to_arrow_stream(&batch) } - /// Convert multiple SensorData to Arrow IPC file format (bytes) - /// Uses a "long" format similar to CSV (timestamp, sensor_id, sensor_name, value, type, labels) - /// which can accommodate multiple sensors in a single file. - pub fn to_arrow_file_multi(sensor_data_list: &[SensorData]) -> Result> { - if sensor_data_list.is_empty() { - // Return empty Arrow file with a minimal schema - let schema = Arc::new(Schema::new(vec![ - Field::new( - "timestamp", - DataType::Timestamp(TimeUnit::Microsecond, None), - false, - ), - Field::new("sensor_id", DataType::Utf8, false), - Field::new("sensor_name", DataType::Utf8, false), - Field::new("value", DataType::Utf8, false), - Field::new("type", DataType::Utf8, false), - Field::new("labels", DataType::Utf8, false), - ])); - let batch = RecordBatch::new_empty(schema); - return Self::record_batch_to_arrow_file(&batch); - } + pub fn to_arrow_stream_multi(sensor_data_list: &[SensorData]) -> Result> { + let batch = Self::to_record_batch_multi(sensor_data_list)?; + Self::record_batch_to_arrow_stream(&batch) + } - // Calculate total sample count + fn to_record_batch_multi(sensor_data_list: &[SensorData]) -> Result { + let present_types = PresentTypes::from_sensor_data_list(sensor_data_list); let total_samples: usize = sensor_data_list.iter().map(|sd| sd.samples.len()).sum(); - // Create builders for the long format let mut timestamp_builder = TimestampMicrosecondBuilder::with_capacity(total_samples); - let mut sensor_id_builder = StringBuilder::with_capacity(total_samples, total_samples * 36); // UUID is 36 chars - let mut sensor_name_builder = - StringBuilder::with_capacity(total_samples, total_samples * 32); - let mut value_builder = StringBuilder::with_capacity(total_samples, total_samples * 16); - let mut type_builder = StringBuilder::with_capacity(total_samples, total_samples * 8); - let mut labels_builder = StringBuilder::with_capacity(total_samples, total_samples * 64); - - // Process each sensor's data + let mut sensor_id_builder = StringDictionaryBuilder::::new(); + let mut sensor_name_builder = StringDictionaryBuilder::::new(); + let mut sensor_type_builder = StringDictionaryBuilder::::new(); + let mut unit_builder = StringDictionaryBuilder::::new(); + let mut labels_builder = StringDictionaryBuilder::::new(); + let mut value_builders = MultiValueBuilders::new(&present_types)?; + for sensor_data in sensor_data_list { - Self::append_samples_to_builders( + Self::append_sensor_data_to_multi_builders( sensor_data, + &present_types, &mut timestamp_builder, &mut sensor_id_builder, &mut sensor_name_builder, - &mut value_builder, - &mut type_builder, + &mut sensor_type_builder, + &mut unit_builder, &mut labels_builder, + &mut value_builders, )?; } - // Create schema for long format - let schema = Arc::new(Schema::new(vec![ + let sensor_id_array = sensor_id_builder.finish(); + let sensor_name_array = sensor_name_builder.finish(); + let sensor_type_array = sensor_type_builder.finish(); + let unit_array = unit_builder.finish(); + let labels_array = labels_builder.finish(); + + let mut fields = vec![ Field::new( "timestamp", DataType::Timestamp(TimeUnit::Microsecond, None), false, ), - Field::new("sensor_id", DataType::Utf8, false), - Field::new("sensor_name", DataType::Utf8, false), - Field::new("value", DataType::Utf8, false), - Field::new("type", DataType::Utf8, false), - Field::new("labels", DataType::Utf8, false), - ])); - - let columns: Vec = vec![ + Field::new("sensor_id", sensor_id_array.data_type().clone(), false), + Field::new("sensor_name", sensor_name_array.data_type().clone(), false), + Field::new("sensor_type", sensor_type_array.data_type().clone(), false), + Field::new("unit", unit_array.data_type().clone(), true), + Field::new("labels", labels_array.data_type().clone(), false), + ]; + + let mut columns: Vec = vec![ Arc::new(timestamp_builder.finish()), - Arc::new(sensor_id_builder.finish()), - Arc::new(sensor_name_builder.finish()), - Arc::new(value_builder.finish()), - Arc::new(type_builder.finish()), - Arc::new(labels_builder.finish()), + Arc::new(sensor_id_array), + Arc::new(sensor_name_array), + Arc::new(sensor_type_array), + Arc::new(unit_array), + Arc::new(labels_array), ]; - let batch = RecordBatch::try_new(schema, columns) - .map_err(|e| anyhow::anyhow!("Failed to create Arrow RecordBatch: {}", e))?; + value_builders.finish_into(&mut fields, &mut columns, &present_types); - Self::record_batch_to_arrow_file(&batch) - } - - /// Helper to convert labels to JSON string - fn labels_to_json_string(sensor_data: &SensorData) -> String { - use serde_json::{Map, Value}; - let mut map = Map::new(); - for (key, value) in sensor_data.sensor.labels.iter() { - map.insert(key.clone(), Value::String(value.clone())); - } - Value::Object(map).to_string() + let schema = Arc::new(Schema::new(fields)); + RecordBatch::try_new(schema, columns) + .map_err(|e| anyhow::anyhow!("Failed to create Arrow RecordBatch: {}", e)) } - /// Internal helper to append samples from a sensor to the builders - fn append_samples_to_builders( + #[allow(clippy::too_many_arguments)] + fn append_sensor_data_to_multi_builders( sensor_data: &SensorData, + present_types: &PresentTypes, timestamp_builder: &mut TimestampMicrosecondBuilder, - sensor_id_builder: &mut StringBuilder, - sensor_name_builder: &mut StringBuilder, - value_builder: &mut StringBuilder, - type_builder: &mut StringBuilder, - labels_builder: &mut StringBuilder, + sensor_id_builder: &mut StringDictionaryBuilder, + sensor_name_builder: &mut StringDictionaryBuilder, + sensor_type_builder: &mut StringDictionaryBuilder, + unit_builder: &mut StringDictionaryBuilder, + labels_builder: &mut StringDictionaryBuilder, + value_builders: &mut MultiValueBuilders, ) -> Result<()> { let sensor_id = sensor_data.sensor.uuid.to_string(); - let sensor_name = &sensor_data.sensor.name; - let labels_json = Self::labels_to_json_string(sensor_data); + let sensor_name = sensor_data.sensor.name.as_str(); + let sensor_type_name = sensor_type_name(&sensor_data.sensor.sensor_type); + let labels_json = labels_to_json_string(&sensor_data.sensor); match &sensor_data.samples { TypedSamples::Integer(samples) => { for sample in samples.iter() { timestamp_builder.append_value(sample.datetime.to_microseconds_since_epoch()); - sensor_id_builder.append_value(&sensor_id); - sensor_name_builder.append_value(sensor_name); - value_builder.append_value(sample.value.to_string()); - type_builder.append_value("integer"); - labels_builder.append_value(&labels_json); + sensor_id_builder.append(&sensor_id)?; + sensor_name_builder.append(sensor_name)?; + sensor_type_builder.append(sensor_type_name)?; + append_unit(unit_builder, sensor_data.sensor.unit.as_ref())?; + labels_builder.append(&labels_json)?; + value_builders.append_integer(sample.value, present_types); } } TypedSamples::Numeric(samples) => { for sample in samples.iter() { timestamp_builder.append_value(sample.datetime.to_microseconds_since_epoch()); - sensor_id_builder.append_value(&sensor_id); - sensor_name_builder.append_value(sensor_name); - value_builder.append_value(sample.value.to_string()); - type_builder.append_value("numeric"); - labels_builder.append_value(&labels_json); + sensor_id_builder.append(&sensor_id)?; + sensor_name_builder.append(sensor_name)?; + sensor_type_builder.append(sensor_type_name)?; + append_unit(unit_builder, sensor_data.sensor.unit.as_ref())?; + labels_builder.append(&labels_json)?; + value_builders.append_numeric( + sample.value.mantissa(), + sample.value.scale(), + present_types, + ); } } TypedSamples::Float(samples) => { for sample in samples.iter() { timestamp_builder.append_value(sample.datetime.to_microseconds_since_epoch()); - sensor_id_builder.append_value(&sensor_id); - sensor_name_builder.append_value(sensor_name); - value_builder.append_value(sample.value.to_string()); - type_builder.append_value("float"); - labels_builder.append_value(&labels_json); + sensor_id_builder.append(&sensor_id)?; + sensor_name_builder.append(sensor_name)?; + sensor_type_builder.append(sensor_type_name)?; + append_unit(unit_builder, sensor_data.sensor.unit.as_ref())?; + labels_builder.append(&labels_json)?; + value_builders.append_float(sample.value, present_types); } } TypedSamples::String(samples) => { for sample in samples.iter() { timestamp_builder.append_value(sample.datetime.to_microseconds_since_epoch()); - sensor_id_builder.append_value(&sensor_id); - sensor_name_builder.append_value(sensor_name); - value_builder.append_value(&sample.value); - type_builder.append_value("string"); - labels_builder.append_value(&labels_json); + sensor_id_builder.append(&sensor_id)?; + sensor_name_builder.append(sensor_name)?; + sensor_type_builder.append(sensor_type_name)?; + append_unit(unit_builder, sensor_data.sensor.unit.as_ref())?; + labels_builder.append(&labels_json)?; + value_builders.append_string(&sample.value, present_types); } } TypedSamples::Boolean(samples) => { for sample in samples.iter() { timestamp_builder.append_value(sample.datetime.to_microseconds_since_epoch()); - sensor_id_builder.append_value(&sensor_id); - sensor_name_builder.append_value(sensor_name); - value_builder.append_value(sample.value.to_string()); - type_builder.append_value("boolean"); - labels_builder.append_value(&labels_json); + sensor_id_builder.append(&sensor_id)?; + sensor_name_builder.append(sensor_name)?; + sensor_type_builder.append(sensor_type_name)?; + append_unit(unit_builder, sensor_data.sensor.unit.as_ref())?; + labels_builder.append(&labels_json)?; + value_builders.append_boolean(sample.value, present_types); } } TypedSamples::Location(samples) => { for sample in samples.iter() { timestamp_builder.append_value(sample.datetime.to_microseconds_since_epoch()); - sensor_id_builder.append_value(&sensor_id); - sensor_name_builder.append_value(sensor_name); - value_builder.append_value(format!( - "{},{}", + sensor_id_builder.append(&sensor_id)?; + sensor_name_builder.append(sensor_name)?; + sensor_type_builder.append(sensor_type_name)?; + append_unit(unit_builder, sensor_data.sensor.unit.as_ref())?; + labels_builder.append(&labels_json)?; + value_builders.append_location( sample.value.y(), - sample.value.x() - )); - type_builder.append_value("location"); - labels_builder.append_value(&labels_json); + sample.value.x(), + present_types, + ); } } - TypedSamples::Json(samples) => { + TypedSamples::Blob(samples) => { for sample in samples.iter() { timestamp_builder.append_value(sample.datetime.to_microseconds_since_epoch()); - sensor_id_builder.append_value(&sensor_id); - sensor_name_builder.append_value(sensor_name); - value_builder.append_value(sample.value.to_string()); - type_builder.append_value("json"); - labels_builder.append_value(&labels_json); + sensor_id_builder.append(&sensor_id)?; + sensor_name_builder.append(sensor_name)?; + sensor_type_builder.append(sensor_type_name)?; + append_unit(unit_builder, sensor_data.sensor.unit.as_ref())?; + labels_builder.append(&labels_json)?; + value_builders.append_blob(&sample.value, present_types); } } - TypedSamples::Blob(samples) => { + TypedSamples::Json(samples) => { for sample in samples.iter() { timestamp_builder.append_value(sample.datetime.to_microseconds_since_epoch()); - sensor_id_builder.append_value(&sensor_id); - sensor_name_builder.append_value(sensor_name); - value_builder.append_value(base64::Engine::encode( - &base64::engine::general_purpose::STANDARD, - &sample.value, - )); - type_builder.append_value("blob"); - labels_builder.append_value(&labels_json); + sensor_id_builder.append(&sensor_id)?; + sensor_name_builder.append(sensor_name)?; + sensor_type_builder.append(sensor_type_name)?; + append_unit(unit_builder, sensor_data.sensor.unit.as_ref())?; + labels_builder.append(&labels_json)?; + value_builders.append_json(&sample.value, present_types)?; } } } @@ -220,202 +212,477 @@ impl ArrowConverter { Ok(()) } - /// Internal method to convert SensorData to Arrow schema and columns fn convert_sensor_data_to_arrow( sensor_data: &SensorData, ) -> Result<(Arc, Vec)> { - let sample_count = sensor_data.samples.len(); - - // Create timestamp column (microsecond precision to match SensApp) - let timestamp_field = Field::new( - "timestamp", - DataType::Timestamp(TimeUnit::Microsecond, None), - false, - ); - let mut timestamp_builder = TimestampMicrosecondBuilder::new(); - let value_column: ArrayRef; let value_field: Field; + let value_column: ArrayRef; match &sensor_data.samples { TypedSamples::Integer(samples) => { value_field = Field::new("value", DataType::Int64, false); let mut builder = Int64Builder::new(); - for sample in samples.iter() { timestamp_builder.append_value(sample.datetime.to_microseconds_since_epoch()); builder.append_value(sample.value); } - value_column = Arc::new(builder.finish()); } - TypedSamples::Numeric(samples) => { - // Use Decimal128 for high precision numeric data value_field = Field::new("value", DataType::Decimal128(38, 18), false); let mut builder = Decimal128Builder::new().with_precision_and_scale(38, 18)?; - for sample in samples.iter() { timestamp_builder.append_value(sample.datetime.to_microseconds_since_epoch()); - // Convert Decimal to i128 representation for Arrow let decimal_i128 = sample.value.mantissa() * 10_i128.pow(18_u32.saturating_sub(sample.value.scale())); builder.append_value(decimal_i128); } - value_column = Arc::new(builder.finish()); } - TypedSamples::Float(samples) => { value_field = Field::new("value", DataType::Float64, false); let mut builder = Float64Builder::new(); - for sample in samples.iter() { timestamp_builder.append_value(sample.datetime.to_microseconds_since_epoch()); builder.append_value(sample.value); } - value_column = Arc::new(builder.finish()); } - TypedSamples::String(samples) => { value_field = Field::new("value", DataType::Utf8, false); let mut builder = StringBuilder::new(); - for sample in samples.iter() { timestamp_builder.append_value(sample.datetime.to_microseconds_since_epoch()); builder.append_value(&sample.value); } - value_column = Arc::new(builder.finish()); } - TypedSamples::Boolean(samples) => { value_field = Field::new("value", DataType::Boolean, false); let mut builder = BooleanBuilder::new(); - for sample in samples.iter() { timestamp_builder.append_value(sample.datetime.to_microseconds_since_epoch()); builder.append_value(sample.value); } - value_column = Arc::new(builder.finish()); } - TypedSamples::Location(samples) => { - // Represent location as a struct with lat/lon fields - let lat_field = Arc::new(Field::new("latitude", DataType::Float64, false)); - let lon_field = Arc::new(Field::new("longitude", DataType::Float64, false)); - let fields = vec![lat_field, lon_field]; + let latitude_field = Arc::new(Field::new("latitude", DataType::Float64, false)); + let longitude_field = Arc::new(Field::new("longitude", DataType::Float64, false)); + let struct_fields = vec![latitude_field, longitude_field]; - value_field = Field::new("value", DataType::Struct(fields.clone().into()), false); + value_field = Field::new( + "value", + DataType::Struct(struct_fields.clone().into()), + false, + ); - let mut lat_builder = Float64Builder::new(); - let mut lon_builder = Float64Builder::new(); + let mut latitude_builder = Float64Builder::new(); + let mut longitude_builder = Float64Builder::new(); for sample in samples.iter() { timestamp_builder.append_value(sample.datetime.to_microseconds_since_epoch()); - lat_builder.append_value(sample.value.y()); - lon_builder.append_value(sample.value.x()); + latitude_builder.append_value(sample.value.y()); + longitude_builder.append_value(sample.value.x()); } - let lat_array = Arc::new(lat_builder.finish()) as ArrayRef; - let lon_array = Arc::new(lon_builder.finish()) as ArrayRef; - let field_arrays = vec![lat_array, lon_array]; - value_column = Arc::new( - arrow::array::StructArray::try_new(fields.into(), field_arrays, None).map_err( - |e| anyhow::anyhow!("Failed to create struct array for location: {}", e), - )?, + StructArray::try_new( + struct_fields.into(), + vec![ + Arc::new(latitude_builder.finish()) as ArrayRef, + Arc::new(longitude_builder.finish()) as ArrayRef, + ], + None, + ) + .map_err(|e| { + anyhow::anyhow!("Failed to create location struct array: {}", e) + })?, ); } - TypedSamples::Blob(samples) => { value_field = Field::new("value", DataType::Binary, false); let mut builder = BinaryBuilder::new(); - for sample in samples.iter() { timestamp_builder.append_value(sample.datetime.to_microseconds_since_epoch()); builder.append_value(&sample.value); } - value_column = Arc::new(builder.finish()); } - TypedSamples::Json(samples) => { - // Store JSON as string (serialized JSON) value_field = Field::new("value", DataType::Utf8, false); let mut builder = StringBuilder::new(); - for sample in samples.iter() { timestamp_builder.append_value(sample.datetime.to_microseconds_since_epoch()); - let json_str = serde_json::to_string(&sample.value)?; - builder.append_value(json_str); + builder.append_value(serde_json::to_string(&sample.value)?); } - value_column = Arc::new(builder.finish()); } } - let timestamp_column = Arc::new(timestamp_builder.finish()) as ArrayRef; + let schema = Arc::new(Schema::new_with_metadata( + vec![ + Field::new( + "timestamp", + DataType::Timestamp(TimeUnit::Microsecond, None), + false, + ), + value_field, + ], + single_series_metadata(&sensor_data.sensor), + )); + + Ok(( + schema, + vec![ + Arc::new(timestamp_builder.finish()) as ArrayRef, + value_column, + ], + )) + } - // Add sensor metadata as fields - let mut fields = vec![timestamp_field, value_field]; - let mut columns = vec![timestamp_column, value_column]; + fn record_batch_to_arrow_stream(batch: &RecordBatch) -> Result> { + let options = IpcWriteOptions::default() + .try_with_compression(Some(CompressionType::LZ4_FRAME)) + .map_err(|e| anyhow::anyhow!("Failed to configure Arrow IPC compression: {}", e))?; - // Add sensor UUID as metadata field - let sensor_id_field = Field::new("sensor_id", DataType::Utf8, false); - let mut sensor_id_builder = StringBuilder::new(); - for _ in 0..sample_count { - sensor_id_builder.append_value(sensor_data.sensor.uuid.to_string()); + let mut buffer = Vec::new(); + { + let mut writer = + StreamWriter::try_new_with_options(&mut buffer, &batch.schema(), options) + .map_err(|e| anyhow::anyhow!("Failed to create Arrow stream writer: {}", e))?; + writer + .write(batch) + .map_err(|e| anyhow::anyhow!("Failed to write Arrow stream batch: {}", e))?; + writer + .finish() + .map_err(|e| anyhow::anyhow!("Failed to finish Arrow stream writer: {}", e))?; } - fields.push(sensor_id_field); - columns.push(Arc::new(sensor_id_builder.finish()) as ArrayRef); - - // Add sensor name - let name = &sensor_data.sensor.name; - let sensor_name_field = Field::new("sensor_name", DataType::Utf8, false); - let mut sensor_name_builder = StringBuilder::new(); - for _ in 0..sample_count { - sensor_name_builder.append_value(name); + Ok(buffer) + } +} + +#[derive(Default)] +struct PresentTypes { + integer: bool, + numeric: bool, + float: bool, + string: bool, + boolean: bool, + location: bool, + blob: bool, + json: bool, +} + +impl PresentTypes { + fn from_sensor_data_list(sensor_data_list: &[SensorData]) -> Self { + let mut present = Self::default(); + + for sensor_data in sensor_data_list { + match &sensor_data.samples { + TypedSamples::Integer(_) => present.integer = true, + TypedSamples::Numeric(_) => present.numeric = true, + TypedSamples::Float(_) => present.float = true, + TypedSamples::String(_) => present.string = true, + TypedSamples::Boolean(_) => present.boolean = true, + TypedSamples::Location(_) => present.location = true, + TypedSamples::Blob(_) => present.blob = true, + TypedSamples::Json(_) => present.json = true, + } } - fields.push(sensor_name_field); - columns.push(Arc::new(sensor_name_builder.finish()) as ArrayRef); - let schema = Arc::new(Schema::new(fields)); - Ok((schema, columns)) + present } +} - /// Convert a single RecordBatch to Arrow file format - fn record_batch_to_arrow_file(batch: &RecordBatch) -> Result> { - let mut buffer = Vec::new(); - { - let mut writer = FileWriter::try_new(&mut buffer, &batch.schema())?; - writer.write(batch)?; - writer.finish()?; +struct MultiValueBuilders { + integer: Option, + numeric: Option, + float: Option, + string: Option, + boolean: Option, + latitude: Option, + longitude: Option, + blob: Option, + json: Option, +} + +impl MultiValueBuilders { + fn new(present: &PresentTypes) -> Result { + Ok(Self { + integer: present.integer.then(Int64Builder::new), + numeric: if present.numeric { + Some(Decimal128Builder::new().with_precision_and_scale(38, 18)?) + } else { + None + }, + float: present.float.then(Float64Builder::new), + string: present.string.then(StringBuilder::new), + boolean: present.boolean.then(BooleanBuilder::new), + latitude: present.location.then(Float64Builder::new), + longitude: present.location.then(Float64Builder::new), + blob: present.blob.then(BinaryBuilder::new), + json: present.json.then(StringBuilder::new), + }) + } + + fn append_integer(&mut self, value: i64, present: &PresentTypes) { + append_int64(&mut self.integer, Some(value)); + self.append_other_nulls(present, SensorType::Integer); + } + + fn append_numeric(&mut self, mantissa: i128, scale: u32, present: &PresentTypes) { + let decimal_i128 = mantissa * 10_i128.pow(18_u32.saturating_sub(scale)); + append_decimal128(&mut self.numeric, Some(decimal_i128)); + self.append_other_nulls(present, SensorType::Numeric); + } + + fn append_float(&mut self, value: f64, present: &PresentTypes) { + append_float64(&mut self.float, Some(value)); + self.append_other_nulls(present, SensorType::Float); + } + + fn append_string(&mut self, value: &str, present: &PresentTypes) { + append_string(&mut self.string, Some(value)); + self.append_other_nulls(present, SensorType::String); + } + + fn append_boolean(&mut self, value: bool, present: &PresentTypes) { + append_boolean(&mut self.boolean, Some(value)); + self.append_other_nulls(present, SensorType::Boolean); + } + + fn append_location(&mut self, latitude: f64, longitude: f64, present: &PresentTypes) { + append_float64(&mut self.latitude, Some(latitude)); + append_float64(&mut self.longitude, Some(longitude)); + self.append_other_nulls(present, SensorType::Location); + } + + fn append_blob(&mut self, value: &[u8], present: &PresentTypes) { + append_binary(&mut self.blob, Some(value)); + self.append_other_nulls(present, SensorType::Blob); + } + + fn append_json(&mut self, value: &serde_json::Value, present: &PresentTypes) -> Result<()> { + append_string(&mut self.json, Some(&serde_json::to_string(value)?)); + self.append_other_nulls(present, SensorType::Json); + Ok(()) + } + + fn append_other_nulls(&mut self, present: &PresentTypes, current_type: SensorType) { + if present.integer && current_type != SensorType::Integer { + append_int64(&mut self.integer, None); + } + if present.numeric && current_type != SensorType::Numeric { + append_decimal128(&mut self.numeric, None); + } + if present.float && current_type != SensorType::Float { + append_float64(&mut self.float, None); + } + if present.string && current_type != SensorType::String { + append_string(&mut self.string, None); + } + if present.boolean && current_type != SensorType::Boolean { + append_boolean(&mut self.boolean, None); + } + if present.location && current_type != SensorType::Location { + append_float64(&mut self.latitude, None); + append_float64(&mut self.longitude, None); + } + if present.blob && current_type != SensorType::Blob { + append_binary(&mut self.blob, None); + } + if present.json && current_type != SensorType::Json { + append_string(&mut self.json, None); + } + } + + fn finish_into( + self, + fields: &mut Vec, + columns: &mut Vec, + present: &PresentTypes, + ) { + if present.integer { + fields.push(Field::new("integer_value", DataType::Int64, true)); + columns.push(Arc::new( + self.integer.expect("integer builder missing").finish(), + )); + } + if present.numeric { + fields.push(Field::new( + "numeric_value", + DataType::Decimal128(38, 18), + true, + )); + columns.push(Arc::new( + self.numeric.expect("numeric builder missing").finish(), + )); + } + if present.float { + fields.push(Field::new("float_value", DataType::Float64, true)); + columns.push(Arc::new( + self.float.expect("float builder missing").finish(), + )); + } + if present.string { + fields.push(Field::new("string_value", DataType::Utf8, true)); + columns.push(Arc::new( + self.string.expect("string builder missing").finish(), + )); + } + if present.boolean { + fields.push(Field::new("boolean_value", DataType::Boolean, true)); + columns.push(Arc::new( + self.boolean.expect("boolean builder missing").finish(), + )); + } + if present.location { + fields.push(Field::new("latitude", DataType::Float64, true)); + columns.push(Arc::new( + self.latitude.expect("latitude builder missing").finish(), + )); + fields.push(Field::new("longitude", DataType::Float64, true)); + columns.push(Arc::new( + self.longitude.expect("longitude builder missing").finish(), + )); + } + if present.blob { + fields.push(Field::new("blob_value", DataType::Binary, true)); + columns.push(Arc::new(self.blob.expect("blob builder missing").finish())); + } + if present.json { + fields.push(Field::new("json_value", DataType::Utf8, true)); + columns.push(Arc::new(self.json.expect("json builder missing").finish())); } - Ok(buffer) } } -// Implement conversion trait for SensAppDateTime to microseconds +fn append_unit( + builder: &mut StringDictionaryBuilder, + unit: Option<&crate::datamodel::unit::Unit>, +) -> Result<()> { + if let Some(unit) = unit { + builder.append(unit.name.as_str())?; + } else { + builder.append_null(); + } + Ok(()) +} + +fn append_int64(builder: &mut Option, value: Option) { + if let Some(builder) = builder.as_mut() { + match value { + Some(value) => builder.append_value(value), + None => builder.append_null(), + } + } +} + +fn append_decimal128(builder: &mut Option, value: Option) { + if let Some(builder) = builder.as_mut() { + match value { + Some(value) => builder.append_value(value), + None => builder.append_null(), + } + } +} + +fn append_float64(builder: &mut Option, value: Option) { + if let Some(builder) = builder.as_mut() { + match value { + Some(value) => builder.append_value(value), + None => builder.append_null(), + } + } +} + +fn append_string(builder: &mut Option, value: Option<&str>) { + if let Some(builder) = builder.as_mut() { + match value { + Some(value) => builder.append_value(value), + None => builder.append_null(), + } + } +} + +fn append_boolean(builder: &mut Option, value: Option) { + if let Some(builder) = builder.as_mut() { + match value { + Some(value) => builder.append_value(value), + None => builder.append_null(), + } + } +} + +fn append_binary(builder: &mut Option, value: Option<&[u8]>) { + if let Some(builder) = builder.as_mut() { + match value { + Some(value) => builder.append_value(value), + None => builder.append_null(), + } + } +} + +fn single_series_metadata(sensor: &Sensor) -> HashMap { + let mut metadata = HashMap::from([ + ("sensapp.sensor.uuid".to_string(), sensor.uuid.to_string()), + ("sensapp.sensor.name".to_string(), sensor.name.clone()), + ( + "sensapp.sensor.type".to_string(), + sensor_type_name(&sensor.sensor_type).to_string(), + ), + ( + "sensapp.sensor.labels".to_string(), + labels_to_json_string(sensor), + ), + ]); + + if let Some(unit) = &sensor.unit { + metadata.insert("sensapp.sensor.unit".to_string(), unit.name.clone()); + } + + metadata +} + +fn labels_to_json_string(sensor: &Sensor) -> String { + use serde_json::{Map, Value}; + + let mut map = Map::new(); + for (key, value) in sensor.labels.iter() { + map.insert(key.clone(), Value::String(value.clone())); + } + Value::Object(map).to_string() +} + +fn sensor_type_name(sensor_type: &SensorType) -> &'static str { + match sensor_type { + SensorType::Integer => "integer", + SensorType::Numeric => "numeric", + SensorType::Float => "float", + SensorType::String => "string", + SensorType::Boolean => "boolean", + SensorType::Location => "location", + SensorType::Json => "json", + SensorType::Blob => "blob", + } +} + trait ToMicroseconds { fn to_microseconds_since_epoch(&self) -> i64; } impl ToMicroseconds for SensAppDateTime { fn to_microseconds_since_epoch(&self) -> i64 { - // Convert to microseconds since Unix epoch - // SensApp uses hifitime internally which has microsecond precision - (self.to_duration_since_j1900().total_nanoseconds() / 1000 - - // J1900 to Unix epoch offset in microseconds - 2_208_988_800_000_000) as i64 + (self.to_duration_since_j1900().total_nanoseconds() / 1000 - 2_208_988_800_000_000) as i64 } } #[cfg(test)] pub mod test_data_helpers { use super::*; + use crate::datamodel::unit::Unit; use crate::datamodel::*; use geo::Point; use smallvec::{SmallVec, smallvec}; @@ -426,8 +693,8 @@ pub mod test_data_helpers { uuid: Uuid::new_v4(), name: "test_sensor".to_string(), sensor_type: SensorType::Integer, - unit: None, - labels: SmallVec::new(), + unit: Some(Unit::new("Celsius".to_string(), None)), + labels: SmallVec::from_vec(vec![("room".to_string(), "lab".to_string())]), }; let datetime1 = SensAppDateTime::now().unwrap(); @@ -436,11 +703,11 @@ pub mod test_data_helpers { let samples = TypedSamples::Integer(smallvec![ Sample { datetime: datetime1, - value: 42 + value: 42, }, Sample { datetime: datetime2, - value: 84 + value: 84, }, ]); @@ -457,12 +724,12 @@ pub mod test_data_helpers { }; let datetime1 = SensAppDateTime::now().unwrap(); - let location = Point::new(2.3522, 48.8566); // Paris coordinates + let location = Point::new(2.3522, 48.8566); let samples = TypedSamples::Location(smallvec![Sample { datetime: datetime1, - value: location - },]); + value: location, + }]); SensorData::new(sensor, samples) } @@ -471,8 +738,14 @@ pub mod test_data_helpers { #[cfg(test)] mod tests { use super::*; - use arrow_ipc::reader::FileReader; + use crate::datamodel::{Sample, Sensor, SensorData, SensorType, TypedSamples}; + use arrow::array::{Array, BooleanArray, Float64Array}; + use arrow_ipc::reader::StreamReader; + use geo::Point; + use smallvec::{SmallVec, smallvec}; + use std::collections::BTreeSet; use std::io::Cursor; + use uuid::Uuid; #[test] fn test_arrow_conversion_integer() { @@ -480,32 +753,51 @@ mod tests { let batch = ArrowConverter::to_record_batch(&sensor_data).unwrap(); assert_eq!(batch.num_rows(), 2); - assert_eq!(batch.num_columns(), 4); // timestamp, value, sensor_id, sensor_name + assert_eq!(batch.num_columns(), 2); - // Verify schema let schema = batch.schema(); assert_eq!(schema.field(0).name(), "timestamp"); assert_eq!(schema.field(1).name(), "value"); - assert_eq!(schema.field(2).name(), "sensor_id"); - assert_eq!(schema.field(3).name(), "sensor_name"); - - // Check data types assert!(matches!( schema.field(0).data_type(), DataType::Timestamp(TimeUnit::Microsecond, _) )); assert_eq!(schema.field(1).data_type(), &DataType::Int64); + assert_eq!( + schema + .metadata() + .get("sensapp.sensor.name") + .map(String::as_str), + Some("test_sensor") + ); + assert_eq!( + schema + .metadata() + .get("sensapp.sensor.type") + .map(String::as_str), + Some("integer") + ); + assert_eq!( + schema + .metadata() + .get("sensapp.sensor.unit") + .map(String::as_str), + Some("Celsius") + ); } #[test] - fn test_arrow_file_format() { + fn test_arrow_stream_format_roundtrip() { let sensor_data = test_data_helpers::create_test_sensor_data_integer(); - let arrow_bytes = ArrowConverter::to_arrow_file(&sensor_data).unwrap(); + let arrow_bytes = ArrowConverter::to_arrow_stream(&sensor_data).unwrap(); assert!(!arrow_bytes.is_empty()); - // Arrow file should start with the Arrow magic number - assert_eq!(&arrow_bytes[0..6], b"ARROW1"); + let reader = StreamReader::try_new(Cursor::new(arrow_bytes), None).unwrap(); + let batches: Vec<_> = reader.into_iter().map(|batch| batch.unwrap()).collect(); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 2); + assert_eq!(batches[0].num_columns(), 2); } #[test] @@ -514,155 +806,173 @@ mod tests { let batch = ArrowConverter::to_record_batch(&sensor_data).unwrap(); assert_eq!(batch.num_rows(), 1); - - // Check that location is converted to struct - let schema = batch.schema(); - assert!(matches!(schema.field(1).data_type(), DataType::Struct(_))); + assert!(matches!( + batch.schema().field(1).data_type(), + DataType::Struct(_) + )); } #[test] - fn test_arrow_multi_with_sensor_id_and_labels() { - use crate::datamodel::*; - use smallvec::smallvec; - use uuid::Uuid; - - // Create sensor with labels - let sensor_uuid = Uuid::new_v4(); - let sensor = Sensor { - uuid: sensor_uuid, - name: "test_sensor".to_string(), + fn test_arrow_multi_uses_sparse_typed_columns() { + let float_sensor = Sensor { + uuid: Uuid::new_v4(), + name: "temperature".to_string(), sensor_type: SensorType::Float, + unit: Some(crate::datamodel::unit::Unit::new("C".to_string(), None)), + labels: smallvec![("room".to_string(), "lab".to_string())], + }; + let bool_sensor = Sensor { + uuid: Uuid::new_v4(), + name: "occupied".to_string(), + sensor_type: SensorType::Boolean, unit: None, - labels: smallvec![ - ("env".to_string(), "production".to_string()), - ("region".to_string(), "us-east".to_string()), - ], + labels: SmallVec::new(), }; - let datetime = SensAppDateTime::now().unwrap(); - let samples = TypedSamples::Float(smallvec![Sample { - datetime, - value: 42.5 - }]); - - let sensor_data = SensorData::new(sensor, samples); - let sensor_data_list = vec![sensor_data]; - - let arrow_bytes = ArrowConverter::to_arrow_file_multi(&sensor_data_list).unwrap(); - - // Read it back - let cursor = Cursor::new(arrow_bytes); - let reader = FileReader::try_new(cursor, None).unwrap(); - let schema = reader.schema(); + let now = SensAppDateTime::now().unwrap(); + let results = vec![ + SensorData::new( + float_sensor, + TypedSamples::Float(smallvec![Sample { + datetime: now, + value: 21.5, + }]), + ), + SensorData::new( + bool_sensor, + TypedSamples::Boolean(smallvec![Sample { + datetime: now, + value: true, + }]), + ), + ]; - // Verify schema has all fields - let field_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); - assert!( - field_names.contains(&"sensor_id"), - "Schema should have sensor_id field, got: {:?}", - field_names - ); - assert!( - field_names.contains(&"sensor_name"), - "Schema should have sensor_name field, got: {:?}", - field_names - ); - assert!( - field_names.contains(&"labels"), - "Schema should have labels field, got: {:?}", - field_names - ); + let bytes = ArrowConverter::to_arrow_stream_multi(&results).unwrap(); + let reader = StreamReader::try_new(Cursor::new(bytes), None).unwrap(); + let batch = reader.into_iter().next().unwrap().unwrap(); + let schema = batch.schema(); + let field_names: BTreeSet<&str> = schema + .fields() + .iter() + .map(|field| field.name().as_str()) + .collect(); + + assert!(field_names.contains("timestamp")); + assert!(field_names.contains("sensor_id")); + assert!(field_names.contains("sensor_name")); + assert!(field_names.contains("sensor_type")); + assert!(field_names.contains("unit")); + assert!(field_names.contains("labels")); + assert!(field_names.contains("float_value")); + assert!(field_names.contains("boolean_value")); + assert!(!field_names.contains("integer_value")); - // Read batches and verify sensor_id value - let batches: Vec<_> = reader.into_iter().filter_map(|b| b.ok()).collect(); - assert_eq!(batches.len(), 1); - let batch = &batches[0]; + assert!(matches!( + schema.field(1).data_type(), + DataType::Dictionary(_, _) + )); + assert!(matches!( + schema.field(2).data_type(), + DataType::Dictionary(_, _) + )); + assert!(matches!( + schema.field(3).data_type(), + DataType::Dictionary(_, _) + )); - // Find sensor_id column - let sensor_id_idx = schema + let float_idx = schema .fields() .iter() - .position(|f| f.name() == "sensor_id") + .position(|field| field.name() == "float_value") .unwrap(); - let sensor_id_col = batch - .column(sensor_id_idx) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(sensor_id_col.value(0), sensor_uuid.to_string()); - - // Find labels column - let labels_idx = schema + let boolean_idx = schema .fields() .iter() - .position(|f| f.name() == "labels") + .position(|field| field.name() == "boolean_value") .unwrap(); - let labels_col = batch - .column(labels_idx) + + let float_col = batch + .column(float_idx) .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap(); - let labels_json = labels_col.value(0); - assert!( - labels_json.contains("env"), - "Labels should contain env, got: {}", - labels_json - ); - assert!( - labels_json.contains("production"), - "Labels should contain production, got: {}", - labels_json - ); + let boolean_col = batch + .column(boolean_idx) + .as_any() + .downcast_ref::() + .unwrap(); + + assert_eq!(float_col.value(0), 21.5); + assert!(float_col.is_null(1)); + assert!(boolean_col.is_null(0)); + assert!(boolean_col.value(1)); } #[test] - fn test_arrow_multi_without_labels() { - use crate::datamodel::*; - use smallvec::{SmallVec, smallvec}; - use uuid::Uuid; - - // Create sensor without labels + fn test_arrow_multi_location_uses_latitude_and_longitude_columns() { let sensor = Sensor { uuid: Uuid::new_v4(), - name: "test_sensor".to_string(), - sensor_type: SensorType::Float, + name: "gps".to_string(), + sensor_type: SensorType::Location, unit: None, labels: SmallVec::new(), }; - let datetime = SensAppDateTime::now().unwrap(); - let samples = TypedSamples::Float(smallvec![Sample { - datetime, - value: 42.5 - }]); - - let sensor_data = SensorData::new(sensor, samples); - let sensor_data_list = vec![sensor_data]; - - let arrow_bytes = ArrowConverter::to_arrow_file_multi(&sensor_data_list).unwrap(); - - // Read it back - let cursor = Cursor::new(arrow_bytes); - let reader = FileReader::try_new(cursor, None).unwrap(); - let schema = reader.schema(); + let now = SensAppDateTime::now().unwrap(); + let results = vec![SensorData::new( + sensor, + TypedSamples::Location(smallvec![Sample { + datetime: now, + value: Point::new(10.3951, 63.4305), + }]), + )]; + + let bytes = ArrowConverter::to_arrow_stream_multi(&results).unwrap(); + let reader = StreamReader::try_new(Cursor::new(bytes), None).unwrap(); + let batch = reader.into_iter().next().unwrap().unwrap(); + let schema = batch.schema(); - // Should still have labels field even if empty - let field_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); - assert!(field_names.contains(&"labels")); + assert!( + schema + .fields() + .iter() + .any(|field| field.name() == "latitude") + ); + assert!( + schema + .fields() + .iter() + .any(|field| field.name() == "longitude") + ); + assert!( + !schema + .fields() + .iter() + .any(|field| field.name() == "float_value") + ); - // Read batches and verify labels is empty object - let batches: Vec<_> = reader.into_iter().filter_map(|b| b.ok()).collect(); - let batch = &batches[0]; - let labels_idx = schema + let lat_idx = schema + .fields() + .iter() + .position(|field| field.name() == "latitude") + .unwrap(); + let lon_idx = schema .fields() .iter() - .position(|f| f.name() == "labels") + .position(|field| field.name() == "longitude") + .unwrap(); + + let lat = batch + .column(lat_idx) + .as_any() + .downcast_ref::() .unwrap(); - let labels_col = batch - .column(labels_idx) + let lon = batch + .column(lon_idx) .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap(); - assert_eq!(labels_col.value(0), "{}"); + assert_eq!(lat.value(0), 63.4305); + assert_eq!(lon.value(0), 10.3951); } } diff --git a/src/ingestors/http/app_error.rs b/src/http/app_error.rs similarity index 57% rename from src/ingestors/http/app_error.rs rename to src/http/app_error.rs index 1594b880..33229d62 100644 --- a/src/ingestors/http/app_error.rs +++ b/src/http/app_error.rs @@ -7,6 +7,20 @@ use serde_json::json; use tracing::error; use utoipa::ToSchema; +fn storage_error_is_unavailable(storage_error: &StorageError) -> bool { + let details = storage_error.to_string().to_ascii_lowercase(); + + details.contains("connection refused") + || details.contains("failed to connect") + || details.contains("pool timed out") + || details.contains("timed out") + || details.contains("connection closed") + || details.contains("connection reset") + || details.contains("broken pipe") + || details.contains("no such file or directory") + || details.contains("database is unavailable") +} + // Anyhow error handling with axum // https://github.com/tokio-rs/axum/blob/d3112a40d55f123bc5e65f995e2068e245f12055/examples/anyhow-error-response/src/main.rs #[derive(Debug, ToSchema)] @@ -17,6 +31,10 @@ pub enum AppError { BadRequest(anyhow::Error), #[schema(example = "Not Found", value_type = String)] NotFound(anyhow::Error), + #[schema(example = "Unauthorized", value_type = String)] + Unauthorized(String), + #[schema(example = "Forbidden", value_type = String)] + Forbidden(String), #[schema(example = "Storage Error", value_type = String)] Storage(StorageError), } @@ -33,6 +51,8 @@ impl IntoResponse for AppError { } AppError::BadRequest(error) => (StatusCode::BAD_REQUEST, error.to_string()), AppError::NotFound(error) => (StatusCode::NOT_FOUND, error.to_string()), + AppError::Unauthorized(message) => (StatusCode::UNAUTHORIZED, message), + AppError::Forbidden(message) => (StatusCode::FORBIDDEN, message), AppError::Storage(storage_error) => match &storage_error { StorageError::SensorNotFound { .. } | StorageError::MetricNotFound { .. } => { (StatusCode::NOT_FOUND, storage_error.to_string()) @@ -59,11 +79,19 @@ impl IntoResponse for AppError { ) } StorageError::Database(_) | StorageError::OperationFailed { .. } => { - error!("Storage operation failed: {}", storage_error); - ( - StatusCode::INTERNAL_SERVER_ERROR, - "Storage operation failed".to_string(), - ) + if storage_error_is_unavailable(&storage_error) { + error!("Storage backend unavailable: {}", storage_error); + ( + StatusCode::SERVICE_UNAVAILABLE, + "Database unavailable".to_string(), + ) + } else { + error!("Storage operation failed: {}", storage_error); + ( + StatusCode::INTERNAL_SERVER_ERROR, + "Storage operation failed".to_string(), + ) + } } }, }; @@ -81,7 +109,10 @@ impl From for AppError { // Generic conversion for anyhow::Error specifically impl From for AppError { fn from(err: anyhow::Error) -> Self { - Self::InternalServerError(err) + match err.downcast::() { + Ok(storage_error) => Self::Storage(storage_error), + Err(err) => Self::InternalServerError(err), + } } } @@ -91,10 +122,34 @@ impl AppError { } pub fn internal_server_error(err: impl Into) -> Self { - Self::InternalServerError(err.into()) + Self::from(err.into()) } pub fn not_found(err: impl Into) -> Self { Self::NotFound(err.into()) } } + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::to_bytes; + + #[tokio::test] + async fn test_internal_server_error_preserves_storage_error_category() { + let response = + AppError::internal_server_error(anyhow::Error::new(StorageError::OperationFailed { + operation: "publish batch".to_string(), + details: "connection refused".to_string(), + })) + .into_response(); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert_eq!( + serde_json::from_slice::(&body).unwrap(), + json!({ "error": "Database unavailable" }) + ); + } +} diff --git a/src/http/auth.rs b/src/http/auth.rs new file mode 100644 index 00000000..1b37ca67 --- /dev/null +++ b/src/http/auth.rs @@ -0,0 +1,472 @@ +use axum::extract::Request; +use axum::extract::State; +use axum::http::{HeaderMap, header}; +use axum::middleware::Next; +use axum::response::Response; +use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header, Validation, decode, encode}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +use super::app_error::AppError; + +// --------------------------------------------------------------------------- +// Auth configuration +// --------------------------------------------------------------------------- + +/// JWT authentication configuration. +/// +/// When present in the server state, all protected endpoints require a valid +/// JWT bearer token. When absent (`None`), all endpoints are open. +#[derive(Clone)] +pub struct AuthConfig { + decoding_key: Arc, + encoding_key: Arc, + validation: Arc, +} + +impl std::fmt::Debug for AuthConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("AuthConfig(…)") + } +} + +impl AuthConfig { + /// Create an AuthConfig from an HMAC secret (HS256). + /// + /// The secret must be at least 32 characters long. + pub fn from_secret(secret: &str) -> Result { + if secret.len() < 32 { + anyhow::bail!("JWT secret must be at least 32 characters long"); + } + + let mut validation = Validation::new(Algorithm::HS256); + validation.validate_exp = true; + validation.validate_nbf = true; + validation.set_required_spec_claims(&["exp", "sub"]); + + Ok(Self { + decoding_key: Arc::new(DecodingKey::from_secret(secret.as_bytes())), + encoding_key: Arc::new(EncodingKey::from_secret(secret.as_bytes())), + validation: Arc::new(validation), + }) + } + + /// Create a signed JWT token. + pub fn create_token( + &self, + sub: &str, + scope: &str, + duration_seconds: u64, + sensors: Option>, + ) -> Result { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_secs(); + + let claims = Claims { + sub: sub.to_string(), + exp: now + duration_seconds, + iat: Some(now), + nbf: Some(now), + scope: Some(scope.to_string()), + sensors, + }; + + let token = encode(&Header::default(), &claims, &self.encoding_key)?; + Ok(token) + } +} + +// --------------------------------------------------------------------------- +// JWT Claims +// --------------------------------------------------------------------------- + +/// JWT claims for SensApp authentication. +/// +/// # Required fields +/// - `sub`: Subject — identifies who the token was issued to +/// - `exp`: Expiration time (Unix timestamp) — tokens without this are rejected +/// +/// # Optional fields +/// - `iat`: Issued-at time (informational) +/// - `nbf`: Not-before time — if present, the token is rejected before this time +/// - `scope`: Space-separated scopes: `"read"`, `"write"`, or `"read write"` (default: `"read write"`) +/// - `sensors`: Allow list of sensor names; if absent all sensors are accessible +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Claims { + /// Subject — identifies who/what the token was issued to. + pub sub: String, + /// Expiration time (Unix timestamp). Required. + pub exp: u64, + /// Issued-at time (Unix timestamp). + #[serde(skip_serializing_if = "Option::is_none")] + pub iat: Option, + /// Not-before time (Unix timestamp). Validated when present. + #[serde(skip_serializing_if = "Option::is_none")] + pub nbf: Option, + /// Space-separated scopes: "read", "write", or "read write". + /// Defaults to "read write" if absent. + #[serde(skip_serializing_if = "Option::is_none")] + pub scope: Option, + /// Optional list of allowed sensor name patterns. + /// If absent, all sensors are accessible. + #[serde(skip_serializing_if = "Option::is_none")] + pub sensors: Option>, +} + +// --------------------------------------------------------------------------- +// Access context +// --------------------------------------------------------------------------- + +/// Validated access context extracted from a JWT token. +#[cfg_attr(not(test), allow(dead_code))] +#[derive(Clone, Debug)] +pub struct AccessContext { + pub subject: String, + pub can_read: bool, + pub can_write: bool, + pub sensor_allow_list: Option>, +} + +impl AccessContext { + /// Check if access to a specific sensor is allowed by the token. + /// + /// When no allow list is set, all sensors are accessible. + /// Otherwise the sensor name must match one of the entries exactly. + #[cfg_attr(not(test), allow(dead_code))] + pub fn can_access_sensor(&self, sensor_name: &str) -> bool { + match &self.sensor_allow_list { + Some(allow_list) => allow_list.iter().any(|allowed| allowed == sensor_name), + None => true, + } + } +} + +// --------------------------------------------------------------------------- +// Token validation +// --------------------------------------------------------------------------- + +/// Extract and validate a JWT from the `Authorization: Bearer ` header. +pub fn validate_token(headers: &HeaderMap, config: &AuthConfig) -> Result { + let token = headers + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.strip_prefix("Bearer ")) + .ok_or(AppError::Unauthorized( + "Missing or invalid Authorization header".into(), + ))?; + + let token_data = decode::(token, &config.decoding_key, &config.validation) + .map_err(|e| AppError::Unauthorized(format!("Invalid token: {}", e)))?; + + let claims = token_data.claims; + + let scope = claims.scope.unwrap_or_else(|| "read write".to_string()); + let can_read = scope + .split_whitespace() + .any(|s| s.eq_ignore_ascii_case("read")); + let can_write = scope + .split_whitespace() + .any(|s| s.eq_ignore_ascii_case("write")); + + Ok(AccessContext { + subject: claims.sub, + can_read, + can_write, + sensor_allow_list: claims.sensors, + }) +} + +// --------------------------------------------------------------------------- +// Axum middleware +// --------------------------------------------------------------------------- + +/// Middleware that requires a valid JWT with **read** scope. +/// +/// When `auth` is `None` (security disabled), all requests pass through. +pub async fn require_read_auth( + State(auth): State>, + request: Request, + next: Next, +) -> Result { + if let Some(config) = &auth { + let access = validate_token(request.headers(), config)?; + if !access.can_read { + return Err(AppError::Forbidden("Read access required".into())); + } + } + Ok(next.run(request).await) +} + +/// Middleware that requires a valid JWT with **write** scope. +/// +/// When `auth` is `None` (security disabled), all requests pass through. +pub async fn require_write_auth( + State(auth): State>, + request: Request, + next: Next, +) -> Result { + if let Some(config) = &auth { + let access = validate_token(request.headers(), config)?; + if !access.can_write { + return Err(AppError::Forbidden("Write access required".into())); + } + } + Ok(next.run(request).await) +} + +// --------------------------------------------------------------------------- +// Unit tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use axum::http::HeaderValue; + + const TEST_SECRET: &str = "test-secret-0123456789abcdef012345"; + + fn make_config() -> AuthConfig { + AuthConfig::from_secret(TEST_SECRET).expect("test config") + } + + fn bearer_headers(token: &str) -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert( + header::AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {token}")).expect("header should build"), + ); + headers + } + + fn encode_test_claims(claims: &Claims) -> String { + encode( + &Header::default(), + claims, + &EncodingKey::from_secret(TEST_SECRET.as_bytes()), + ) + .expect("token should encode") + } + + #[test] + fn secret_too_short_is_rejected() { + let result = AuthConfig::from_secret("too-short"); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("at least 32 characters") + ); + } + + #[test] + fn accepts_valid_token() { + let token = encode_test_claims(&Claims { + sub: "test-user".to_string(), + exp: 4_102_444_800, + iat: Some(1_000_000_000), + nbf: Some(1_000_000_000), + scope: Some("read write".to_string()), + sensors: None, + }); + + let access = + validate_token(&bearer_headers(&token), &make_config()).expect("token should be valid"); + + assert_eq!(access.subject, "test-user"); + assert!(access.can_read); + assert!(access.can_write); + assert!(access.can_access_sensor("anything")); + } + + #[test] + fn rejects_expired_token() { + let token = encode_test_claims(&Claims { + sub: "test-user".to_string(), + exp: 0, + iat: None, + nbf: None, + scope: Some("read write".to_string()), + sensors: None, + }); + + let result = validate_token(&bearer_headers(&token), &make_config()); + assert!(result.is_err()); + } + + #[test] + fn rejects_not_yet_valid_token() { + let token = encode_test_claims(&Claims { + sub: "test-user".to_string(), + exp: 4_102_444_800, + iat: None, + nbf: Some(4_102_444_800), // far in the future + scope: Some("read write".to_string()), + sensors: None, + }); + + let result = validate_token(&bearer_headers(&token), &make_config()); + assert!(result.is_err()); + } + + #[test] + fn rejects_wrong_secret() { + let wrong_key = EncodingKey::from_secret(b"wrong-secret-that-is-long-enough!!"); + let token = encode( + &Header::default(), + &Claims { + sub: "test-user".to_string(), + exp: 4_102_444_800, + iat: None, + nbf: None, + scope: None, + sensors: None, + }, + &wrong_key, + ) + .unwrap(); + + let result = validate_token(&bearer_headers(&token), &make_config()); + assert!(result.is_err()); + } + + #[test] + fn rejects_missing_authorization_header() { + let result = validate_token(&HeaderMap::new(), &make_config()); + assert!(result.is_err()); + } + + #[test] + fn rejects_non_bearer_scheme() { + let mut headers = HeaderMap::new(); + headers.insert( + header::AUTHORIZATION, + HeaderValue::from_static("Basic dXNlcjpwYXNz"), + ); + let result = validate_token(&headers, &make_config()); + assert!(result.is_err()); + } + + #[test] + fn scope_read_only() { + let token = encode_test_claims(&Claims { + sub: "reader".to_string(), + exp: 4_102_444_800, + iat: None, + nbf: None, + scope: Some("read".to_string()), + sensors: None, + }); + + let access = + validate_token(&bearer_headers(&token), &make_config()).expect("should decode"); + assert!(access.can_read); + assert!(!access.can_write); + } + + #[test] + fn scope_write_only() { + let token = encode_test_claims(&Claims { + sub: "writer".to_string(), + exp: 4_102_444_800, + iat: None, + nbf: None, + scope: Some("write".to_string()), + sensors: None, + }); + + let access = + validate_token(&bearer_headers(&token), &make_config()).expect("should decode"); + assert!(!access.can_read); + assert!(access.can_write); + } + + #[test] + fn missing_scope_defaults_to_read_write() { + let token = encode_test_claims(&Claims { + sub: "admin".to_string(), + exp: 4_102_444_800, + iat: None, + nbf: None, + scope: None, + sensors: None, + }); + + let access = + validate_token(&bearer_headers(&token), &make_config()).expect("should decode"); + assert!(access.can_read); + assert!(access.can_write); + } + + #[test] + fn sensor_allow_list_filters() { + let token = encode_test_claims(&Claims { + sub: "limited".to_string(), + exp: 4_102_444_800, + iat: None, + nbf: None, + scope: None, + sensors: Some(vec!["temperature".to_string(), "humidity".to_string()]), + }); + + let access = + validate_token(&bearer_headers(&token), &make_config()).expect("should decode"); + assert!(access.can_access_sensor("temperature")); + assert!(access.can_access_sensor("humidity")); + assert!(!access.can_access_sensor("pressure")); + } + + #[test] + fn no_sensor_allow_list_allows_all() { + let token = encode_test_claims(&Claims { + sub: "admin".to_string(), + exp: 4_102_444_800, + iat: None, + nbf: None, + scope: None, + sensors: None, + }); + + let access = + validate_token(&bearer_headers(&token), &make_config()).expect("should decode"); + assert!(access.can_access_sensor("anything")); + assert!(access.can_access_sensor("temperature")); + } + + #[test] + fn create_token_produces_valid_jwt() { + let config = make_config(); + let token = config + .create_token("my-service", "read write", 3600, None) + .expect("should create token"); + + let access = + validate_token(&bearer_headers(&token), &config).expect("token should validate"); + assert_eq!(access.subject, "my-service"); + assert!(access.can_read); + assert!(access.can_write); + } + + #[test] + fn create_token_with_sensor_filter() { + let config = make_config(); + let token = config + .create_token( + "sensor-reader", + "read", + 3600, + Some(vec!["cpu".to_string(), "mem".to_string()]), + ) + .expect("should create token"); + + let access = + validate_token(&bearer_headers(&token), &config).expect("token should validate"); + assert_eq!(access.subject, "sensor-reader"); + assert!(access.can_read); + assert!(!access.can_write); + assert!(access.can_access_sensor("cpu")); + assert!(access.can_access_sensor("mem")); + assert!(!access.can_access_sensor("disk")); + } +} diff --git a/src/ingestors/http/crud.rs b/src/http/crud.rs similarity index 50% rename from src/ingestors/http/crud.rs rename to src/http/crud.rs index c96413c0..684883ae 100644 --- a/src/ingestors/http/crud.rs +++ b/src/http/crud.rs @@ -1,9 +1,10 @@ use crate::datamodel::SensAppDateTime; use crate::datamodel::SensorType; use crate::exporters::{ArrowConverter, CsvConverter, JsonlConverter, SenMLConverter}; -use crate::ingestors::http::app_error::AppError; -use crate::ingestors::http::state::HttpServerState; +use crate::http::app_error::AppError; +use crate::http::state::HttpServerState; use crate::storage::query::{LabelMatcher, MatcherType}; +use crate::storage::{Aggregation, SensorDataQueryOptions, SimplifyOptions}; use axum::Json; use axum::extract::{Path, Query, State}; use regex::Regex; @@ -11,13 +12,90 @@ use rusty_promql_parser::{Expr, expr}; use serde::Deserialize; use serde_json::{Value, json}; use std::str::FromStr; +use std::time::Instant; + +mod promql_duration { + use anyhow::{Result, anyhow}; + use nom::{ + Parser, + branch::alt, + bytes::complete::tag, + character::complete::digit1, + combinator::{map, map_res}, + multi::many1, + sequence::pair, + }; + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum DurationUnit { + Millisecond, + Second, + Minute, + Hour, + Day, + Week, + Year, + } + + impl DurationUnit { + const fn millis(self) -> i64 { + match self { + Self::Millisecond => 1, + Self::Second => 1_000, + Self::Minute => 60_000, + Self::Hour => 3_600_000, + Self::Day => 86_400_000, + Self::Week => 604_800_000, + Self::Year => 31_536_000_000, + } + } + } + + fn duration_unit(input: &str) -> nom::IResult<&str, DurationUnit> { + alt(( + map(tag("ms"), |_| DurationUnit::Millisecond), + map(tag("s"), |_| DurationUnit::Second), + map(tag("m"), |_| DurationUnit::Minute), + map(tag("h"), |_| DurationUnit::Hour), + map(tag("d"), |_| DurationUnit::Day), + map(tag("w"), |_| DurationUnit::Week), + map(tag("y"), |_| DurationUnit::Year), + )) + .parse(input) + } + + fn duration_component(input: &str) -> nom::IResult<&str, (i64, DurationUnit)> { + pair(map_res(digit1, |s: &str| s.parse::()), duration_unit).parse(input) + } + + pub fn parse_duration_millis(input: &str) -> Result { + let (remaining, parts) = many1(duration_component) + .parse(input) + .map_err(|_| anyhow!("Invalid duration '{}'", input))?; + if !remaining.is_empty() { + return Err(anyhow!("Invalid duration '{}'", input)); + } + + let mut total_ms = 0i64; + for (value, unit) in parts { + let component = value + .checked_mul(unit.millis()) + .ok_or_else(|| anyhow!("Duration '{}' is too large", input))?; + total_ms = total_ms + .checked_add(component) + .ok_or_else(|| anyhow!("Duration '{}' is too large", input))?; + } + + Ok(total_ms) + } +} #[derive(Debug, Clone, PartialEq)] pub enum ExportFormat { Senml, // SenML JSON format (RFC 8428) - also accessible as "json" Csv, // Comma-separated values Jsonl, // JSON Lines (one JSON object per line) - Arrow, // Apache Arrow IPC file format (.arrow) + Arrow, // Apache Arrow IPC stream format (.arrow) } impl ExportFormat { @@ -27,7 +105,7 @@ impl ExportFormat { "json" | "senml" => Some(ExportFormat::Senml), // Both json and senml map to SenML "csv" => Some(ExportFormat::Csv), "jsonl" | "ndjson" => Some(ExportFormat::Jsonl), // Support both extensions - "arrow" | "ipc" => Some(ExportFormat::Arrow), // Apache Arrow file format + "arrow" | "ipc" => Some(ExportFormat::Arrow), // Apache Arrow IPC stream format _ => None, } } @@ -38,7 +116,7 @@ impl ExportFormat { ExportFormat::Senml => "application/json", // SenML is JSON ExportFormat::Csv => "text/csv", ExportFormat::Jsonl => "application/x-ndjson", - ExportFormat::Arrow => "application/vnd.apache.arrow.file", + ExportFormat::Arrow => "application/vnd.apache.arrow.stream", } } } @@ -62,6 +140,24 @@ pub struct SensorDataQuery { pub end: Option, pub limit: Option, pub format: Option, + pub step: Option, + pub aggregation: Option, + pub simplify: Option, + pub simplify_tolerance: Option, + pub simplify_high_quality: Option, +} + +#[derive(Debug, Deserialize)] +pub struct LastSampleQuery { + pub start: Option, + pub end: Option, +} + +#[derive(Debug, Deserialize)] +pub struct AvailabilityQuery { + pub start: String, + pub end: String, + pub step: Option, } #[derive(Debug, Deserialize)] @@ -69,6 +165,8 @@ pub struct SeriesQuery { pub metric: Option, /// PromQL-style label selector (e.g., `{env="prod",region=~"us.*"}`) pub selector: Option, + pub limit: Option, + pub bookmark: Option, } /// Query parameters for metrics filtering @@ -96,7 +194,7 @@ fn convert_label_match_op(op: rusty_promql_parser::LabelMatchOp) -> MatcherType /// Accepts formats like: /// - `{env="prod"}` - label matchers only /// - `my_metric{env="prod"}` - metric name with labels (metric name is ignored for series filtering) -fn parse_selector_to_matchers(selector: &str) -> Result, AppError> { +pub(crate) fn parse_selector_to_matchers(selector: &str) -> Result, AppError> { // If the selector is just braces with labels, wrap it in a dummy metric name let query_str = if selector.trim().starts_with('{') { format!("dummy{}", selector) @@ -142,7 +240,10 @@ fn parse_selector_to_matchers(selector: &str) -> Result, AppEr } /// Check if a sensor matches a set of label matchers -fn sensor_matches_matchers(sensor: &crate::datamodel::Sensor, matchers: &[LabelMatcher]) -> bool { +pub(crate) fn sensor_matches_matchers( + sensor: &crate::datamodel::Sensor, + matchers: &[LabelMatcher], +) -> bool { for matcher in matchers { // Find the label value in the sensor let label_value = sensor @@ -181,6 +282,87 @@ fn sensor_matches_matchers(sensor: &crate::datamodel::Sensor, matchers: &[LabelM true } +fn sensor_type_name(sensor_type: SensorType) -> &'static str { + match sensor_type { + SensorType::Integer => "integer", + SensorType::Numeric => "numeric", + SensorType::Float => "float", + SensorType::String => "string", + SensorType::Boolean => "boolean", + SensorType::Location => "location", + SensorType::Blob => "blob", + SensorType::Json => "json", + } +} + +fn parse_optional_time_bounds( + start: Option<&String>, + end: Option<&String>, +) -> Result<(Option, Option), AppError> { + let start_time = match start { + Some(value) => Some(parse_datetime_string(value).map_err(|e| { + AppError::bad_request(anyhow::anyhow!("Invalid start datetime: {}", e)) + })?), + None => None, + }; + + let end_time = + match end { + Some(value) => Some(parse_datetime_string(value).map_err(|e| { + AppError::bad_request(anyhow::anyhow!("Invalid end datetime: {}", e)) + })?), + None => None, + }; + + if let (Some(start_time), Some(end_time)) = (start_time, end_time) + && start_time > end_time + { + return Err(AppError::bad_request(anyhow::anyhow!( + "'start' must be less than or equal to 'end'" + ))); + } + + Ok((start_time, end_time)) +} + +fn last_sample_json(samples: &crate::datamodel::TypedSamples) -> Option<(String, Value)> { + match samples { + crate::datamodel::TypedSamples::Integer(samples) => samples + .last() + .map(|sample| (sample.datetime.to_rfc3339(), json!(sample.value))), + crate::datamodel::TypedSamples::Numeric(samples) => samples.last().map(|sample| { + ( + sample.datetime.to_rfc3339(), + json!(sample.value.to_string()), + ) + }), + crate::datamodel::TypedSamples::Float(samples) => samples + .last() + .map(|sample| (sample.datetime.to_rfc3339(), json!(sample.value))), + crate::datamodel::TypedSamples::String(samples) => samples + .last() + .map(|sample| (sample.datetime.to_rfc3339(), json!(sample.value))), + crate::datamodel::TypedSamples::Boolean(samples) => samples + .last() + .map(|sample| (sample.datetime.to_rfc3339(), json!(sample.value))), + crate::datamodel::TypedSamples::Location(samples) => samples.last().map(|sample| { + ( + sample.datetime.to_rfc3339(), + json!({ + "longitude": sample.value.x(), + "latitude": sample.value.y(), + }), + ) + }), + crate::datamodel::TypedSamples::Blob(samples) => samples + .last() + .map(|sample| (sample.datetime.to_rfc3339(), json!(sample.value))), + crate::datamodel::TypedSamples::Json(samples) => samples + .last() + .map(|sample| (sample.datetime.to_rfc3339(), sample.value.clone())), + } +} + /// List unique metrics (measurement types) with aggregated information in DCAT catalog format. #[utoipa::path( get, @@ -199,67 +381,64 @@ pub async fn list_metrics( State(state): State, Query(query): Query, ) -> Result, AppError> { - let metrics = state.storage.list_metrics().await?; - - // Compile regex if provided - let name_regex = match &query.name_regex { - Some(pattern) => Some(Regex::new(pattern).map_err(|e| { - AppError::bad_request(anyhow::anyhow!( - "Invalid regex pattern '{}': {}", - pattern, - e - )) - })?), - None => None, - }; + let metrics_registry = state.metrics.clone(); + let started = Instant::now(); + + let result = async move { + let metrics = state.storage.list_metrics().await?; + + let name_regex = match &query.name_regex { + Some(pattern) => Some(Regex::new(pattern).map_err(|e| { + AppError::bad_request(anyhow::anyhow!( + "Invalid regex pattern '{}': {}", + pattern, + e + )) + })?), + None => None, + }; - // Parse type filter if provided - let type_filter = match &query.r#type { - Some(type_str) => Some(SensorType::from_str(type_str).map_err(|e| { - AppError::bad_request(anyhow::anyhow!( - "Invalid sensor type '{}': {}. Valid types: float, integer, string, boolean, location, json, blob, numeric", - type_str, e - )) - })?), - None => None, - }; + let type_filter = match &query.r#type { + Some(type_str) => Some(SensorType::from_str(type_str).map_err(|e| { + AppError::bad_request(anyhow::anyhow!( + "Invalid sensor type '{}': {}. Valid types: float, integer, string, boolean, location, json, blob, numeric", + type_str, e + )) + })?), + None => None, + }; - // Filter metrics based on query parameters - let filtered_metrics: Vec<_> = metrics - .into_iter() - .filter(|metric| { - // Filter by name substring - if let Some(name_filter) = &query.name - && !metric - .name - .to_lowercase() - .contains(&name_filter.to_lowercase()) - { - return false; - } + let filtered_metrics: Vec<_> = metrics + .into_iter() + .filter(|metric| { + if let Some(name_filter) = &query.name + && !metric + .name + .to_lowercase() + .contains(&name_filter.to_lowercase()) + { + return false; + } - // Filter by name regex - if let Some(regex) = &name_regex - && !regex.is_match(&metric.name) - { - return false; - } + if let Some(regex) = &name_regex + && !regex.is_match(&metric.name) + { + return false; + } - // Filter by type - if let Some(type_filter) = &type_filter - && &metric.sensor_type != type_filter - { - return false; - } + if let Some(type_filter) = &type_filter + && &metric.sensor_type != type_filter + { + return false; + } - true - }) - .collect(); + true + }) + .collect(); - // Create DCAT catalog structure for metrics - let datasets: Vec = filtered_metrics - .iter() - .map(|metric| { + let datasets: Vec = filtered_metrics + .iter() + .map(|metric| { // Create keywords from metric type and label dimensions let mut keywords = vec!["metric", "aggregated", "time-series"]; keywords.push(match metric.sensor_type { @@ -317,29 +496,43 @@ pub async fn list_metrics( dataset["sensor:unit"] = json!(unit.name); } - dataset - }) - .collect(); + dataset + }) + .collect(); - let catalog = json!({ - "@context": { - "dcat": "http://www.w3.org/ns/dcat#", - "dct": "http://purl.org/dc/terms/", - "foaf": "http://xmlns.com/foaf/0.1/", - "sensor": "http://sensapp.io/ns/sensor#" - }, - "@type": "dcat:Catalog", - "@id": "sensapp_metrics_catalog", - "dct:title": "SensApp Metrics Catalog", - "dct:description": "Catalog of aggregated metrics available in SensApp platform", - "dct:publisher": { - "@type": "foaf:Organization", - "foaf:name": "SensApp" - }, - "dcat:dataset": datasets - }); + let catalog = json!({ + "@context": { + "dcat": "http://www.w3.org/ns/dcat#", + "dct": "http://purl.org/dc/terms/", + "foaf": "http://xmlns.com/foaf/0.1/", + "sensor": "http://sensapp.io/ns/sensor#" + }, + "@type": "dcat:Catalog", + "@id": "sensapp_metrics_catalog", + "dct:title": "SensApp Metrics Catalog", + "dct:description": "Catalog of aggregated metrics available in SensApp platform", + "dct:publisher": { + "@type": "foaf:Organization", + "foaf:name": "SensApp" + }, + "dcat:dataset": datasets + }); + + Ok::<_, AppError>((Json(catalog), filtered_metrics.len())) + } + .await; + + metrics_registry.observe_operation_result( + "read", + "list_metrics", + started.elapsed(), + result.is_ok(), + ); + if let Ok((_, items)) = &result { + metrics_registry.observe_series("read", "list_metrics", *items); + } - Ok(Json(catalog)) + result.map(|(catalog, _)| catalog) } /// List all series (time series) in DCAT catalog format. @@ -358,30 +551,38 @@ pub async fn list_metrics( pub async fn list_series( State(state): State, Query(query): Query, -) -> Result, AppError> { - // Get the series metadata including labels and UUIDs, optionally filtered by metric - let sensors = state.storage.list_series(query.metric.as_deref()).await?; +) -> Result { + let metrics_registry = state.metrics.clone(); + let started = Instant::now(); - // Parse the selector if provided - let label_matchers = match &query.selector { - Some(selector) => Some(parse_selector_to_matchers(selector)?), - None => None, - }; + let result = async move { + let limit = query + .limit + .map(|l| l.min(crate::storage::MAX_LIST_SERIES_LIMIT)); - // Filter sensors by label matchers if provided - let filtered_sensors: Vec<_> = if let Some(matchers) = &label_matchers { - sensors - .into_iter() - .filter(|sensor| sensor_matches_matchers(sensor, matchers)) - .collect() - } else { - sensors - }; + let result = state + .storage + .list_series(query.metric.as_deref(), limit, query.bookmark.as_deref()) + .await?; - // Create DCAT catalog structure - let datasets: Vec = filtered_sensors - .iter() - .map(|sensor| { + let label_matchers = match &query.selector { + Some(selector) => Some(parse_selector_to_matchers(selector)?), + None => None, + }; + + let filtered_sensors: Vec<_> = if let Some(matchers) = &label_matchers { + result + .series + .into_iter() + .filter(|sensor| sensor_matches_matchers(sensor, matchers)) + .collect() + } else { + result.series + }; + + let datasets: Vec = filtered_sensors + .iter() + .map(|sensor| { // Create keywords from sensor type, unit, and labels let mut keywords = vec!["sensor", "IoT", "time-series"]; keywords.push(match sensor.sensor_type { @@ -454,15 +655,16 @@ pub async fn list_series( dataset["sensor:unit"] = json!(unit.name); } - dataset - }) - .collect(); + dataset + }) + .collect(); - let catalog = json!({ + let mut catalog = json!({ "@context": { "dcat": "http://www.w3.org/ns/dcat#", "dct": "http://purl.org/dc/terms/", - "foaf": "http://xmlns.com/foaf/0.1/" + "foaf": "http://xmlns.com/foaf/0.1/", + "hydra": "http://www.w3.org/ns/hydra/core#" }, "@type": "dcat:Catalog", "@id": "sensapp_series_catalog", @@ -475,7 +677,61 @@ pub async fn list_series( "dcat:dataset": datasets }); - Ok(Json(catalog)) + let link_header = if let Some(bookmark) = &result.bookmark { + // Build the next page URL + let mut next_url = format!( + "/series?limit={}&bookmark={}", + limit.unwrap_or(crate::storage::DEFAULT_LIST_SERIES_LIMIT), + bookmark + ); + if let Some(metric) = &query.metric { + next_url = format!("{}&metric={}", next_url, urlencoding::encode(metric)); + } + + // Add Hydra view to catalog + catalog["hydra:view"] = json!({ + "@type": "hydra:PartialCollectionView", + "hydra:next": next_url, + "hydra:itemsPerPage": limit.unwrap_or(crate::storage::DEFAULT_LIST_SERIES_LIMIT) + }); + + // Return Link header value + Some(format!("<{}>; rel=\"next\"", next_url)) + } else { + None + }; + + let mut response = axum::response::Response::builder() + .status(200) + .header("Content-Type", "application/json"); + + if let Some(link) = link_header { + response = response.header("Link", link); + } + + let body = serde_json::to_string(&catalog).map_err(|e| { + AppError::internal_server_error(anyhow::anyhow!("Failed to serialize catalog: {}", e)) + })?; + + let response = response.body(axum::body::Body::from(body)).map_err(|e| { + AppError::internal_server_error(anyhow::anyhow!("Failed to build response: {}", e)) + })?; + + Ok::<_, AppError>((response, filtered_sensors.len())) + } + .await; + + metrics_registry.observe_operation_result( + "read", + "list_series", + started.elapsed(), + result.is_ok(), + ); + if let Ok((_, series)) = &result { + metrics_registry.observe_series("read", "list_series", *series); + } + + result.map(|(response, _)| response) } /// Get series data in various formats based on query parameter. @@ -488,7 +744,12 @@ pub async fn list_series( ("format" = Option, Query, description = "Output format: senml, csv, or jsonl (default: senml)"), ("start" = Option, Query, description = "Start datetime in ISO 8601 format (e.g., '2024-01-15T10:30:00Z')"), ("end" = Option, Query, description = "End datetime in ISO 8601 format (e.g., '2024-01-15T11:00:00Z')"), - ("limit" = Option, Query, description = "Maximum number of samples (default: 10,000,000)") + ("limit" = Option, Query, description = "Maximum number of samples (default: 10,000,000)"), + ("step" = Option, Query, description = "Bucket width using Prometheus duration syntax (e.g., '1h', '5m', '1d')"), + ("aggregation" = Option, Query, description = "Aggregation function: avg, min, max, sum, count, first, last"), + ("simplify" = Option, Query, description = "Explicitly enable simplify-based point reduction"), + ("simplify_tolerance" = Option, Query, description = "Dimensionless simplify tolerance on normalized time/value coordinates"), + ("simplify_high_quality" = Option, Query, description = "Use Douglas-Peucker only when simplifying") ), responses( (status = 200, description = "Series data in requested format", body = Value), @@ -501,90 +762,335 @@ pub async fn get_series_data( Path(series_uuid): Path, Query(query): Query, ) -> Result { - // Parse format from query parameter, default to SenML/JSON - let format = match query.format.as_deref() { - Some(format_str) => ExportFormat::from_extension(format_str).ok_or_else(|| { - AppError::bad_request(anyhow::anyhow!( - "Unsupported export format '{}'. Supported formats: senml, csv, jsonl, arrow", - format_str - )) - })?, - None => ExportFormat::Senml, // Default to SenML/JSON format - }; + let metrics_registry = state.metrics.clone(); + let started = Instant::now(); + + let result = async move { + let format = match query.format.as_deref() { + Some(format_str) => ExportFormat::from_extension(format_str).ok_or_else(|| { + AppError::bad_request(anyhow::anyhow!( + "Unsupported export format '{}'. Supported formats: senml, csv, jsonl, arrow", + format_str + )) + })?, + None => ExportFormat::Senml, + }; - // Validate UUID format - let _parsed_uuid = uuid::Uuid::from_str(&series_uuid).map_err(|_| { - AppError::bad_request(anyhow::anyhow!("Invalid UUID format: '{}'", series_uuid)) - })?; + let _parsed_uuid = uuid::Uuid::from_str(&series_uuid).map_err(|_| { + AppError::bad_request(anyhow::anyhow!("Invalid UUID format: '{}'", series_uuid)) + })?; - // Parse datetime parameters - let start_time = match query.start.as_ref() { - Some(start_str) => Some(parse_datetime_string(start_str).map_err(|e| { - AppError::bad_request(anyhow::anyhow!("Invalid start datetime: {}", e)) - })?), - None => None, - }; + let start_time = match query.start.as_ref() { + Some(start_str) => Some(parse_datetime_string(start_str).map_err(|e| { + AppError::bad_request(anyhow::anyhow!("Invalid start datetime: {}", e)) + })?), + None => None, + }; - let end_time = - match query.end.as_ref() { + let end_time = match query.end.as_ref() { Some(end_str) => Some(parse_datetime_string(end_str).map_err(|e| { AppError::bad_request(anyhow::anyhow!("Invalid end datetime: {}", e)) })?), None => None, }; - // Query series data from storage by UUID - let series_data = state - .storage - .query_sensor_data(&series_uuid, start_time, end_time, query.limit) - .await?; - - let series_data = match series_data { - Some(data) => data, - None => { - return Err(AppError::not_found(anyhow::anyhow!( - "Series with UUID '{}' not found", - series_uuid - ))); - } - }; + let step_ms = match query.step.as_deref() { + Some(step) => Some(promql_duration::parse_duration_millis(step).map_err(|e| { + AppError::bad_request(anyhow::anyhow!("Invalid step duration: {}", e)) + })?), + None => None, + }; - // Convert based on requested format - let response = match format { - ExportFormat::Senml => { - let json_value = SenMLConverter::to_senml_json(&series_data) - .map_err(AppError::internal_server_error)?; - axum::response::Response::builder() - .header("content-type", format.content_type()) - .body(json_value.to_string().into()) - } - ExportFormat::Csv => { - let csv_content = - CsvConverter::to_csv(&series_data).map_err(AppError::internal_server_error)?; - axum::response::Response::builder() - .header("content-type", format.content_type()) - .body(csv_content.into()) - } - ExportFormat::Jsonl => { - let jsonl_content = - JsonlConverter::to_jsonl(&series_data).map_err(AppError::internal_server_error)?; - axum::response::Response::builder() - .header("content-type", format.content_type()) - .body(jsonl_content.into()) - } - ExportFormat::Arrow => { - let arrow_bytes = ArrowConverter::to_arrow_file(&series_data) - .map_err(AppError::internal_server_error)?; - axum::response::Response::builder() - .header("content-type", format.content_type()) - .body(arrow_bytes.into()) + let aggregation = match query.aggregation.as_deref() { + Some(value) => Some( + value + .parse::() + .map_err(AppError::bad_request)?, + ), + None => None, + }; + + let simplify = if query.simplify.unwrap_or(false) { + let tolerance = query.simplify_tolerance.ok_or_else(|| { + AppError::bad_request(anyhow::anyhow!( + "'simplify_tolerance' is required when simplify=true" + )) + })?; + Some(SimplifyOptions { + tolerance, + high_quality: query.simplify_high_quality.unwrap_or(false), + }) + } else { + if query.simplify_tolerance.is_some() || query.simplify_high_quality.is_some() { + return Err(AppError::bad_request(anyhow::anyhow!( + "'simplify=true' is required when simplify options are provided" + ))); + } + None + }; + + let query_options = SensorDataQueryOptions { + start_time, + end_time, + limit: query.limit, + step_ms, + aggregation, + simplify, + }; + + query_options.validate().map_err(AppError::bad_request)?; + + let series_data = state + .storage + .query_sensor_data_advanced(&series_uuid, &query_options) + .await?; + + let series_data = match series_data { + Some(data) => data, + None => { + return Err(AppError::not_found(anyhow::anyhow!( + "Series with UUID '{}' not found", + series_uuid + ))); + } + }; + + let sample_count = series_data.samples.len(); + + let response = match format { + ExportFormat::Senml => { + let json_value = SenMLConverter::to_senml_json(&series_data) + .map_err(AppError::internal_server_error)?; + axum::response::Response::builder() + .header("content-type", format.content_type()) + .body(json_value.to_string().into()) + } + ExportFormat::Csv => { + let csv_content = + CsvConverter::to_csv(&series_data).map_err(AppError::internal_server_error)?; + axum::response::Response::builder() + .header("content-type", format.content_type()) + .body(csv_content.into()) + } + ExportFormat::Jsonl => { + let jsonl_content = JsonlConverter::to_jsonl(&series_data) + .map_err(AppError::internal_server_error)?; + axum::response::Response::builder() + .header("content-type", format.content_type()) + .body(jsonl_content.into()) + } + ExportFormat::Arrow => { + let arrow_bytes = ArrowConverter::to_arrow_stream(&series_data) + .map_err(AppError::internal_server_error)?; + axum::response::Response::builder() + .header("content-type", format.content_type()) + .body(arrow_bytes.into()) + } } + .map_err(|e| { + AppError::internal_server_error(anyhow::anyhow!("Failed to build response: {}", e)) + })?; + + Ok::<_, AppError>((response, sample_count)) + } + .await; + + metrics_registry.observe_operation_result( + "read", + "get_series_data", + started.elapsed(), + result.is_ok(), + ); + if let Ok((_, sample_count)) = &result { + metrics_registry.observe_series("read", "get_series_data", 1); + metrics_registry.observe_samples("read", "get_series_data", *sample_count); } - .map_err(|e| { - AppError::internal_server_error(anyhow::anyhow!("Failed to build response: {}", e)) - })?; - Ok(response) + result.map(|(response, _)| response) +} + +/// Get the most recent sample for a series, optionally within a bounded time window. +#[utoipa::path( + get, + path = "/series/{series_uuid}/last", + tag = "SensApp", + params( + ("series_uuid" = String, Path, description = "UUID of the series"), + ("start" = Option, Query, description = "Optional inclusive start datetime in ISO 8601 format"), + ("end" = Option, Query, description = "Optional inclusive end datetime in ISO 8601 format") + ), + responses( + (status = 200, description = "Latest sample for the requested series", body = Value), + (status = 404, description = "Series not found or no sample matched the requested window"), + (status = 400, description = "Invalid query parameters") + ) +)] +pub async fn get_series_last_sample( + State(state): State, + Path(series_uuid): Path, + Query(query): Query, +) -> Result, AppError> { + let metrics_registry = state.metrics.clone(); + let started = Instant::now(); + + let result = async move { + let _parsed_uuid = uuid::Uuid::from_str(&series_uuid).map_err(|_| { + AppError::bad_request(anyhow::anyhow!("Invalid UUID format: '{}'", series_uuid)) + })?; + + let (start_time, end_time) = + parse_optional_time_bounds(query.start.as_ref(), query.end.as_ref())?; + + let sensor_data = state + .storage + .query_sensor_data_latest(&series_uuid, start_time, end_time) + .await? + .ok_or_else(|| { + AppError::not_found(anyhow::anyhow!( + "Series with UUID '{}' has no sample in the requested window", + series_uuid + )) + })?; + + let (timestamp, value) = last_sample_json(&sensor_data.samples).ok_or_else(|| { + AppError::not_found(anyhow::anyhow!( + "Series with UUID '{}' has no sample in the requested window", + series_uuid + )) + })?; + + Ok::<_, AppError>(Json(json!({ + "series_uuid": sensor_data.sensor.uuid, + "sensor_name": sensor_data.sensor.name, + "sensor_type": sensor_type_name(sensor_data.sensor.sensor_type), + "unit": sensor_data.sensor.unit.as_ref().map(|unit| unit.name.clone()), + "timestamp": timestamp, + "value": value, + }))) + } + .await; + + metrics_registry.observe_operation_result( + "read", + "get_series_last_sample", + started.elapsed(), + result.is_ok(), + ); + if result.is_ok() { + metrics_registry.observe_series("read", "get_series_last_sample", 1); + metrics_registry.observe_samples("read", "get_series_last_sample", 1); + } + + result +} + +/// Get presence and optional bucket coverage for a series over a time window. +#[utoipa::path( + get, + path = "/series/{series_uuid}/availability", + tag = "SensApp", + params( + ("series_uuid" = String, Path, description = "UUID of the series"), + ("start" = String, Query, description = "Inclusive start datetime in ISO 8601 format"), + ("end" = String, Query, description = "Inclusive end datetime in ISO 8601 format"), + ("step" = Option, Query, description = "Optional bucket width using Prometheus duration syntax for coverage calculation") + ), + responses( + (status = 200, description = "Availability information for the requested series and window", body = Value), + (status = 404, description = "Series not found"), + (status = 400, description = "Invalid query parameters") + ) +)] +pub async fn get_series_availability( + State(state): State, + Path(series_uuid): Path, + Query(query): Query, +) -> Result, AppError> { + let metrics_registry = state.metrics.clone(); + let started = Instant::now(); + + let result = async move { + let _parsed_uuid = uuid::Uuid::from_str(&series_uuid).map_err(|_| { + AppError::bad_request(anyhow::anyhow!("Invalid UUID format: '{}'", series_uuid)) + })?; + + let (start_time, end_time) = + parse_optional_time_bounds(Some(&query.start), Some(&query.end))?; + let start_time = start_time.expect("validated required start time"); + let end_time = end_time.expect("validated required end time"); + let parsed_step_ms = match query.step.as_deref() { + Some(step) => Some(promql_duration::parse_duration_millis(step).map_err(|e| { + AppError::bad_request(anyhow::anyhow!("Invalid step duration: {}", e)) + })?), + None => None, + }; + + let summary = state + .storage + .query_sensor_data_availability(&series_uuid, start_time, end_time, parsed_step_ms) + .await? + .ok_or_else(|| { + AppError::not_found(anyhow::anyhow!( + "Series with UUID '{}' not found", + series_uuid + )) + })?; + + let sample_count = summary.sample_count; + + let (step_value, covered_buckets, total_buckets, coverage_ratio) = + if let (Some(step), Some(step_ms)) = (query.step.as_deref(), parsed_step_ms) { + let step_us = step_ms + .checked_mul(1000) + .ok_or_else(|| AppError::bad_request(anyhow::anyhow!("step is too large")))?; + + let start_us = crate::storage::common::datetime_to_micros(&start_time); + let end_us = crate::storage::common::datetime_to_micros(&end_time); + let total = ((end_us - start_us).div_euclid(step_us) + 1).max(1) as usize; + let covered = summary.covered_buckets.unwrap_or(0); + + ( + Some(step.to_string()), + Some(covered), + Some(total), + Some(covered as f64 / total as f64), + ) + } else { + (None, None, None, None) + }; + + Ok::<_, AppError>(( + Json(json!({ + "series_uuid": summary.sensor.uuid, + "sensor_name": summary.sensor.name, + "start": start_time.to_rfc3339(), + "end": end_time.to_rfc3339(), + "present": summary.sample_count > 0, + "sample_count": summary.sample_count, + "first_sample_at": summary.first_sample_at.map(|value| value.to_rfc3339()), + "last_sample_at": summary.last_sample_at.map(|value| value.to_rfc3339()), + "step": step_value, + "covered_buckets": covered_buckets, + "total_buckets": total_buckets, + "coverage_ratio": coverage_ratio, + })), + sample_count, + )) + } + .await; + + metrics_registry.observe_operation_result( + "read", + "get_series_availability", + started.elapsed(), + result.is_ok(), + ); + if let Ok((_, sample_count)) = &result { + metrics_registry.observe_series("read", "get_series_availability", 1); + metrics_registry.observe_samples("read", "get_series_availability", *sample_count); + } + + result.map(|(response, _)| response) } #[cfg(test)] @@ -628,7 +1134,7 @@ mod tests { assert_eq!(ExportFormat::Jsonl.content_type(), "application/x-ndjson"); assert_eq!( ExportFormat::Arrow.content_type(), - "application/vnd.apache.arrow.file" + "application/vnd.apache.arrow.stream" ); } @@ -714,6 +1220,23 @@ mod tests { assert!(result.is_ok(), "Should handle end of year"); } + #[test] + fn test_parse_duration_millis() { + assert_eq!( + promql_duration::parse_duration_millis("5m").unwrap(), + 300_000 + ); + assert_eq!( + promql_duration::parse_duration_millis("1h30m").unwrap(), + 5_400_000 + ); + assert_eq!( + promql_duration::parse_duration_millis("250ms").unwrap(), + 250 + ); + assert!(promql_duration::parse_duration_millis("bad").is_err()); + } + #[test] fn test_prometheus_id_generation() { use crate::datamodel::{Sensor, SensorType, sensapp_vec::SensAppLabels, unit::Unit}; diff --git a/src/ingestors/http/health.rs b/src/http/health.rs similarity index 100% rename from src/ingestors/http/health.rs rename to src/http/health.rs diff --git a/src/ingestors/http/influxdb.rs b/src/http/influxdb.rs similarity index 67% rename from src/ingestors/http/influxdb.rs rename to src/http/influxdb.rs index 80bf2600..ccd0e801 100644 --- a/src/ingestors/http/influxdb.rs +++ b/src/http/influxdb.rs @@ -14,6 +14,7 @@ use influxdb_line_protocol::{FieldValue, parse_lines}; use rust_decimal::Decimal; use serde::Deserialize; use std::str::FromStr; +use std::time::Instant; use std::{io::Read, str::from_utf8}; use std::{str, sync::Arc}; use tokio_util::bytes::Bytes; @@ -97,17 +98,12 @@ fn influxdb_field_to_sensapp( } FieldValue::F64(value) => { if with_numeric { + let decimal = Decimal::from_str(&value.to_string()).map_err(|e| { + anyhow::anyhow!("Failed to convert f64 value {} to Decimal: {}", value, e) + })?; Ok(( SensorType::Numeric, - TypedSamples::one_numeric( - Decimal::from_f64_retain(value).ok_or_else(|| { - anyhow::anyhow!( - "Failed to convert f64 value {} to Decimal - precision may be too high", - value - ) - })?, - datetime, - ), + TypedSamples::one_numeric(decimal, datetime), )) } else { Ok((SensorType::Float, TypedSamples::one_float(value, datetime))) @@ -185,141 +181,166 @@ pub async fn publish_influxdb( }): Query, bytes: Bytes, ) -> Result { - debug!( - "InfluxDB publish: bucket={}, org={:?}, org_id={:?}, precision={:?}", - bucket, org, org_id, precision - ); - - // Requires org or org_id - if org.is_none() && org_id.is_none() { - return Err(AppError::bad_request(anyhow::anyhow!( - "org or org_id must be specified" - ))); - } - - // Org or org_id, this is the same for SensApp. - let common_org_name = match org { - Some(org) => org, - None => org_id.unwrap_or_default(), - }; - - // Convert the precision string to a Precision enum - let precision_enum = match precision { - Some(precision) => match precision.parse() { - Ok(precision) => precision, - Err(_) => { - return Err(AppError::bad_request(anyhow::anyhow!( - "Invalid precision: {}", - precision - ))); - } - }, - None => Precision::default(), - }; - - let bytes_string = bytes_to_string(&headers, &bytes)?; - let parser = parse_lines(&bytes_string); + let metrics = state.metrics.clone(); + let started = Instant::now(); - let mut batch_builder = BatchBuilder::new()?; - - for line in parser { - match line { - Ok(line) => { - let measurement = line.series.measurement; + let result = async move { + debug!( + "InfluxDB publish: bucket={}, org={:?}, org_id={:?}, precision={:?}", + bucket, org, org_id, precision + ); - let tags = match &line.series.tag_set { - None => None, - Some(tags) => { - let mut tags_vec = SensAppLabels::new(); - tags_vec.push(("influxdb_bucket".to_string(), bucket.clone())); - tags_vec.push(("influxdb_org".to_string(), common_org_name.clone())); + if org.is_none() && org_id.is_none() { + return Err(AppError::bad_request(anyhow::anyhow!( + "org or org_id must be specified" + ))); + } - for (key, value) in tags.iter() { - tags_vec.push((key.to_string(), value.to_string())); + let common_org_name = match org { + Some(org) => org, + None => org_id.unwrap_or_default(), + }; + + let precision_enum = match precision { + Some(precision) => match precision.parse() { + Ok(precision) => precision, + Err(_) => { + return Err(AppError::bad_request(anyhow::anyhow!( + "Invalid precision: {}", + precision + ))); + } + }, + None => Precision::default(), + }; + + let bytes_string = bytes_to_string(&headers, &bytes)?; + let parser = parse_lines(&bytes_string); + + let mut batch_builder = BatchBuilder::new()?; + let mut series = 0usize; + let mut samples = 0usize; + + for line in parser { + match line { + Ok(line) => { + let measurement = line.series.measurement; + + let tags = match &line.series.tag_set { + None => None, + Some(tags) => { + let mut tags_vec = SensAppLabels::new(); + tags_vec.push(("influxdb_bucket".to_string(), bucket.clone())); + tags_vec.push(("influxdb_org".to_string(), common_org_name.clone())); + + for (key, value) in tags.iter() { + tags_vec.push((key.to_string(), value.to_string())); + } + Some(tags_vec) } - Some(tags_vec) - } - }; + }; - let datetime = match line.timestamp { - Some(timestamp) => match precision_enum { - Precision::Nanoseconds => { - SensAppDateTime::from_unix_nanoseconds_i64(timestamp) - } - Precision::Microseconds => { - SensAppDateTime::from_unix_microseconds_i64(timestamp) - } - Precision::Milliseconds => { - SensAppDateTime::from_unix_milliseconds_i64(timestamp) - } - Precision::Seconds => SensAppDateTime::from_unix_seconds_i64(timestamp), - }, - None => match SensAppDateTime::now() { - Ok(datetime) => datetime, - Err(error) => { - return Err(AppError::internal_server_error(error)); - } - }, - }; - - let url_encoded_field_name = urlencoding::encode(&measurement).to_string(); - - for (field_key, field_value) in line.field_set { - let unit = None; - let (sensor_type, value) = match influxdb_field_to_sensapp( - field_value, - datetime, - state.influxdb_with_numeric, - ) { - Ok((sensor_type, value)) => (sensor_type, value), - Err(error) => { - return Err(AppError::bad_request(error)); - } + let datetime = match line.timestamp { + Some(timestamp) => match precision_enum { + Precision::Nanoseconds => { + SensAppDateTime::from_unix_nanoseconds_i64(timestamp) + } + Precision::Microseconds => { + SensAppDateTime::from_unix_microseconds_i64(timestamp) + } + Precision::Milliseconds => { + SensAppDateTime::from_unix_milliseconds_i64(timestamp) + } + Precision::Seconds => SensAppDateTime::from_unix_seconds_i64(timestamp), + }, + None => match SensAppDateTime::now() { + Ok(datetime) => datetime, + Err(error) => { + return Err(AppError::internal_server_error(error)); + } + }, }; - let name = compute_field_name(&url_encoded_field_name, &field_key); - let sensor = Sensor::new_without_uuid(name, sensor_type, unit, tags.clone())?; - batch_builder.add(Arc::new(sensor), value).await?; + + let url_encoded_field_name = urlencoding::encode(&measurement).to_string(); + + for (field_key, field_value) in line.field_set { + let unit = None; + let (sensor_type, value) = match influxdb_field_to_sensapp( + field_value, + datetime, + state.influxdb_with_numeric, + ) { + Ok((sensor_type, value)) => (sensor_type, value), + Err(error) => { + return Err(AppError::bad_request(error)); + } + }; + let name = compute_field_name(&url_encoded_field_name, &field_key); + let sensor = + Sensor::new_without_uuid(name, sensor_type, unit, tags.clone())?; + series += 1; + samples += value.len(); + batch_builder.add(Arc::new(sensor), value).await?; + } } + Err(error) => { + return Err(AppError::bad_request(error)); + } + } + } + + match batch_builder.send_what_is_left(state.storage.clone()).await { + Ok(true) => { + info!("InfluxDB: Batch sent successfully"); + } + Ok(false) => { + debug!("InfluxDB: No data to send"); } Err(error) => { - return Err(AppError::bad_request(error)); + error!("InfluxDB: Error sending batch: {:?}", error); + return Err(AppError::internal_server_error(error)); } } + + Ok::<_, AppError>((StatusCode::NO_CONTENT, series, samples)) } + .await; - match batch_builder.send_what_is_left(state.storage.clone()).await { - Ok(true) => { - info!("InfluxDB: Batch sent successfully"); - } - Ok(false) => { - debug!("InfluxDB: No data to send"); - } - Err(error) => { - error!("InfluxDB: Error sending batch: {:?}", error); - return Err(AppError::internal_server_error(error)); - } + metrics.observe_operation_result("write", "influxdb_write", started.elapsed(), result.is_ok()); + if let Ok((_, series, samples)) = &result { + metrics.observe_series("write", "influxdb_write", *series); + metrics.observe_samples("write", "influxdb_write", *samples); } - // OK no content - Ok(StatusCode::NO_CONTENT) + result.map(|(status, _, _)| status) } #[cfg(test)] mod tests { use super::*; use crate::config::load_configuration_for_tests; + use crate::http::metrics::HttpMetrics; use crate::storage::storage_factory::create_storage_from_connection_string; use flate2::Compression; use flate2::write::GzEncoder; use influxdb_line_protocol::EscapedStr; use serial_test::serial; use std::io::Write; + use uuid::Uuid; /// Helper to get test database URL - uses the centralized constant from test_utils fn get_test_database_url() -> String { sensapp::test_utils::get_test_database_url() } + fn storage_backend_test_available() -> bool { + cfg!(feature = "postgres") || std::env::var("TEST_DATABASE_URL").is_ok() + } + + fn unique_test_bucket(prefix: &str) -> String { + format!("{}-{}", prefix, Uuid::new_v4()) + } + #[test] #[serial] fn test_bytes_to_string() { @@ -356,9 +377,22 @@ mod tests { #[tokio::test] #[serial] async fn test_publish_influxdb() { + if !storage_backend_test_available() { + return; + } + _ = load_configuration_for_tests(); let connection_string = get_test_database_url(); + sensapp::test_utils::ensure_test_database_exists(&connection_string) + .await + .unwrap(); + let bucket = unique_test_bucket("test"); + let no_tag_measurement = format!("cpu_without_tags_{}", Uuid::new_v4().simple()); + let current_seconds = SensAppDateTime::now().unwrap().to_unix_seconds().floor() as i64; + let current_nanoseconds = current_seconds * 1_000_000_000; + let current_microseconds = current_seconds * 1_000_000; + let current_milliseconds = current_seconds * 1_000; let storage = create_storage_from_connection_string(&connection_string) .await .unwrap(); @@ -368,16 +402,21 @@ mod tests { let state = State(HttpServerState { name: Arc::new("influxdb test".to_string()), storage, + metrics: Arc::new(HttpMetrics::new()), influxdb_with_numeric: false, + auth: None, }); let headers = HeaderMap::new(); let query = Query(InfluxDBQueryParams { - bucket: "test".to_string(), + bucket: bucket.clone(), org: Some("test".to_string()), org_id: None, precision: None, }); - let bytes = Bytes::from("cpu,host=A,region=west usage_system=64i 1590488773254420000"); + let bytes = Bytes::from(format!( + "cpu_first,host=A,region=west usage_system=64i {}", + current_nanoseconds + )); let result = publish_influxdb(state.clone(), headers, query, bytes) .await .unwrap(); @@ -387,7 +426,7 @@ mod tests { let mut headers = HeaderMap::new(); headers.insert("content-encoding", "gzip".parse().unwrap()); let query = Query(InfluxDBQueryParams { - bucket: "test".to_string(), + bucket: bucket.clone(), org: None, org_id: Some("test".to_string()), precision: None, @@ -401,7 +440,7 @@ mod tests { // With wrong line protocol let headers = HeaderMap::new(); let query = Query(InfluxDBQueryParams { - bucket: "test".to_string(), + bucket: bucket.clone(), org: Some("test".to_string()), org_id: Some("test2".to_string()), precision: None, @@ -415,12 +454,15 @@ mod tests { // With no org or org_id let headers = HeaderMap::new(); let query = Query(InfluxDBQueryParams { - bucket: "test".to_string(), + bucket: bucket.clone(), org: None, org_id: None, precision: None, }); - let bytes = Bytes::from("cpu,host=A,region=west usage_system=64i 1590488773254420000"); + let bytes = Bytes::from(format!( + "cpu,host=A,region=west usage_system=64i {}", + current_nanoseconds + )); let result = publish_influxdb(state.clone(), headers, query, bytes).await; assert!(result.is_err()); assert!(matches!(result, Err(AppError::BadRequest(_)))); @@ -428,12 +470,15 @@ mod tests { // Without tags let headers = HeaderMap::new(); let query = Query(InfluxDBQueryParams { - bucket: "test".to_string(), + bucket: bucket.clone(), org: Some("test".to_string()), org_id: None, precision: None, }); - let bytes = Bytes::from("cpu usage_system=64i 1590488773254420000"); + let bytes = Bytes::from(format!( + "{} usage_system=64i {}", + no_tag_measurement, current_nanoseconds + )); let result = publish_influxdb(state.clone(), headers, query, bytes) .await .unwrap(); @@ -442,12 +487,12 @@ mod tests { // Without datetime let headers = HeaderMap::new(); let query = Query(InfluxDBQueryParams { - bucket: "test".to_string(), + bucket: bucket.clone(), org: Some("test".to_string()), org_id: None, precision: None, }); - let bytes = Bytes::from("cpu,host=A,region=west usage_system=64i"); + let bytes = Bytes::from("cpu_without_datetime,host=A,region=west usage_system=64i"); let result = publish_influxdb(state.clone(), headers, query, bytes) .await .unwrap(); @@ -456,7 +501,7 @@ mod tests { // Too high u64 value let headers = HeaderMap::new(); let query = Query(InfluxDBQueryParams { - bucket: "test".to_string(), + bucket: bucket.clone(), org: Some("test".to_string()), org_id: None, precision: None, @@ -469,12 +514,15 @@ mod tests { // With various precisions let headers = HeaderMap::new(); let query = Query(InfluxDBQueryParams { - bucket: "test".to_string(), + bucket: bucket.clone(), org: Some("test".to_string()), org_id: None, precision: Some("ns".to_string()), }); - let bytes = Bytes::from("cpu,host=A,region=west usage_system=64i 1590488773254420000"); + let bytes = Bytes::from(format!( + "cpu_ns,host=A,region=west usage_system=64i {}", + current_nanoseconds + )); let result = publish_influxdb(state.clone(), headers, query, bytes) .await .unwrap(); @@ -482,12 +530,15 @@ mod tests { let headers = HeaderMap::new(); let query = Query(InfluxDBQueryParams { - bucket: "test".to_string(), + bucket: bucket.clone(), org: Some("test".to_string()), org_id: None, precision: Some("us".to_string()), }); - let bytes = Bytes::from("cpu,host=A,region=west usage_system=64i 1590488773254420"); + let bytes = Bytes::from(format!( + "cpu_us,host=A,region=west usage_system=64i {}", + current_microseconds + )); let result = publish_influxdb(state.clone(), headers, query, bytes) .await .unwrap(); @@ -495,12 +546,15 @@ mod tests { let headers = HeaderMap::new(); let query = Query(InfluxDBQueryParams { - bucket: "test".to_string(), + bucket: bucket.clone(), org: Some("test".to_string()), org_id: None, precision: Some("ms".to_string()), }); - let bytes = Bytes::from("cpu,host=A,region=west usage_system=64i 1590488773254"); + let bytes = Bytes::from(format!( + "cpu_ms,host=A,region=west usage_system=64i {}", + current_milliseconds + )); let result = publish_influxdb(state.clone(), headers, query, bytes) .await .unwrap(); @@ -508,12 +562,15 @@ mod tests { let headers = HeaderMap::new(); let query = Query(InfluxDBQueryParams { - bucket: "test".to_string(), + bucket: bucket.clone(), org: Some("test".to_string()), org_id: None, precision: Some("s".to_string()), }); - let bytes = Bytes::from("cpu,host=A,region=west usage_system=64i 1590488773"); + let bytes = Bytes::from(format!( + "cpu_s,host=A,region=west usage_system=64i {}", + current_seconds + )); let result = publish_influxdb(state.clone(), headers, query, bytes) .await .unwrap(); @@ -522,12 +579,15 @@ mod tests { // With wrong precision let headers = HeaderMap::new(); let query = Query(InfluxDBQueryParams { - bucket: "test".to_string(), + bucket, org: Some("test".to_string()), org_id: None, precision: Some("wrong".to_string()), }); - let bytes = Bytes::from("cpu,host=A,region=west usage_system=64i 1590488773"); + let bytes = Bytes::from(format!( + "cpu,host=A,region=west usage_system=64i {}", + current_seconds + )); let result = publish_influxdb(state.clone(), headers, query, bytes).await; assert!(result.is_err()); assert!(matches!(result, Err(AppError::BadRequest(_)))); @@ -586,7 +646,7 @@ mod tests { result, ( SensorType::Numeric, - TypedSamples::one_numeric(Decimal::from_f64_retain(42.0).unwrap(), datetime) + TypedSamples::one_numeric(Decimal::from(42), datetime) ) ); @@ -639,9 +699,20 @@ mod tests { #[tokio::test] #[serial] async fn test_publish_influxdb_with_numeric_enabled() { + if !storage_backend_test_available() { + return; + } + _ = load_configuration_for_tests(); let connection_string = get_test_database_url(); + sensapp::test_utils::ensure_test_database_exists(&connection_string) + .await + .unwrap(); + let bucket = unique_test_bucket("test-numeric"); + let no_tag_measurement = format!("memory_no_tags_{}", Uuid::new_v4().simple()); + let current_nanoseconds = + (SensAppDateTime::now().unwrap().to_unix_seconds().floor() as i64) * 1_000_000_000; let storage = create_storage_from_connection_string(&connection_string) .await .unwrap(); @@ -652,18 +723,23 @@ mod tests { let state = State(HttpServerState { name: Arc::new("influxdb numeric test".to_string()), storage: storage.clone(), + metrics: Arc::new(HttpMetrics::new()), influxdb_with_numeric: true, + auth: None, }); // Test with integer value let headers = HeaderMap::new(); let query = Query(InfluxDBQueryParams { - bucket: "test_numeric".to_string(), - org: Some("test_numeric".to_string()), + bucket: bucket.clone(), + org: Some(bucket.clone()), org_id: None, precision: None, }); - let bytes = Bytes::from("memory,host=B usage_int=42i,usage_float=3.14 1590488773254420000"); + let bytes = Bytes::from(format!( + "memory,host=B usage_int=42i,usage_float=3.14 {}", + current_nanoseconds + )); let result = publish_influxdb(state.clone(), headers, query, bytes) .await .unwrap(); @@ -672,12 +748,15 @@ mod tests { // Test with high u64 value that exceeds i64::MAX - should succeed with numeric enabled let headers = HeaderMap::new(); let query = Query(InfluxDBQueryParams { - bucket: "test_numeric".to_string(), - org: Some("test_numeric".to_string()), + bucket: bucket.clone(), + org: Some(bucket), org_id: None, precision: None, }); - let bytes = Bytes::from("memory usage_big=9223372036854775808u 1590488773254420000"); + let bytes = Bytes::from(format!( + "{} usage_big=9223372036854775808u {}", + no_tag_measurement, current_nanoseconds + )); let result = publish_influxdb(state.clone(), headers, query, bytes) .await .unwrap(); diff --git a/src/http/metrics.rs b/src/http/metrics.rs new file mode 100644 index 00000000..fd1d1ed6 --- /dev/null +++ b/src/http/metrics.rs @@ -0,0 +1,607 @@ +use super::app_error::AppError; +use super::crud::{parse_selector_to_matchers, sensor_matches_matchers}; +use super::state::HttpServerState; +use axum::body::Body; +use axum::extract::{MatchedPath, Query, Request, State}; +use axum::http::{Method, StatusCode, header}; +use axum::middleware::Next; +use axum::response::Response; +use prometheus_client::encoding::EncodeLabelSet; +use prometheus_client::encoding::text::encode; +use prometheus_client::metrics::counter::Counter; +use prometheus_client::metrics::family::Family; +use prometheus_client::metrics::gauge::Gauge; +use prometheus_client::metrics::histogram::{Histogram, exponential_buckets}; +use prometheus_client::registry::Registry; +use serde::Deserialize; +use std::fmt; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crate::parsing::prometheus::converter::datetime_to_millis; + +const PROMETHEUS_CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8"; +const LATEST_SERIES_PAGE_SIZE: usize = crate::storage::MAX_LIST_SERIES_LIMIT; + +#[derive(Clone, Debug, Default, Deserialize)] +pub struct PrometheusMetricsQuery { + #[serde(default)] + pub include_latest_samples: bool, + pub metric: Option, + pub selector: Option, +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +struct HttpRequestLabels { + method: String, + path: String, + status: String, +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +struct HttpRequestDurationLabels { + method: String, + path: String, +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +struct OperationLabels { + kind: String, + operation: String, + status: String, +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +struct OperationDurationLabels { + kind: String, + operation: String, +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +struct DataVolumeLabels { + kind: String, + operation: String, +} + +pub struct HttpMetrics { + registry: Registry, + http_requests_total: Family, + http_request_duration_seconds: Family, + operations_total: Family, + operation_duration_seconds: Family, + samples_processed_total: Family, + series_processed_total: Family, + http_requests_in_flight: Gauge, + uptime_seconds: Gauge, + storage_ready: Gauge, + start_time: Instant, +} + +impl fmt::Debug for HttpMetrics { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("HttpMetrics") + .finish_non_exhaustive() + } +} + +impl Default for HttpMetrics { + fn default() -> Self { + Self::new() + } +} + +impl HttpMetrics { + pub fn new() -> Self { + let mut registry = Registry::default(); + + let http_requests_total = Family::::default(); + registry.register( + "sensapp_http_requests", + "Total number of handled HTTP requests", + http_requests_total.clone(), + ); + + let http_request_duration_seconds = + Family::::new_with_constructor(|| { + Histogram::new(exponential_buckets(0.005, 2.0, 12)) + }); + registry.register( + "sensapp_http_request_duration_seconds", + "HTTP request duration in seconds", + http_request_duration_seconds.clone(), + ); + + let operations_total = Family::::default(); + registry.register( + "sensapp_operations", + "Total number of application read/write operations", + operations_total.clone(), + ); + + let operation_duration_seconds = + Family::::new_with_constructor(|| { + Histogram::new(exponential_buckets(0.001, 2.0, 16)) + }); + registry.register( + "sensapp_operation_duration_seconds", + "Application read/write operation duration in seconds", + operation_duration_seconds.clone(), + ); + + let samples_processed_total = Family::::default(); + registry.register( + "sensapp_samples_processed", + "Total number of samples processed by read/write operations", + samples_processed_total.clone(), + ); + + let series_processed_total = Family::::default(); + registry.register( + "sensapp_series_processed", + "Total number of series processed by read/write operations", + series_processed_total.clone(), + ); + + let http_requests_in_flight = Gauge::default(); + registry.register( + "sensapp_http_requests_in_flight", + "Number of HTTP requests currently being handled", + http_requests_in_flight.clone(), + ); + + let uptime_seconds = Gauge::default(); + registry.register( + "sensapp_uptime_seconds", + "Process uptime in seconds", + uptime_seconds.clone(), + ); + + let storage_ready = Gauge::default(); + registry.register( + "sensapp_storage_ready", + "Storage health status reported during the latest metrics scrape (1=ready, 0=not ready)", + storage_ready.clone(), + ); + + Self { + registry, + http_requests_total, + http_request_duration_seconds, + operations_total, + operation_duration_seconds, + samples_processed_total, + series_processed_total, + http_requests_in_flight, + uptime_seconds, + storage_ready, + start_time: Instant::now(), + } + } + + pub fn observe_http_request( + &self, + method: &Method, + path: &str, + status: StatusCode, + duration: Duration, + ) { + self.http_requests_total + .get_or_create(&HttpRequestLabels { + method: method.to_string(), + path: path.to_string(), + status: status.as_u16().to_string(), + }) + .inc(); + + self.http_request_duration_seconds + .get_or_create(&HttpRequestDurationLabels { + method: method.to_string(), + path: path.to_string(), + }) + .observe(duration.as_secs_f64()); + } + + pub fn observe_operation_result( + &self, + kind: &str, + operation: &str, + duration: Duration, + success: bool, + ) { + self.operations_total + .get_or_create(&OperationLabels { + kind: kind.to_string(), + operation: operation.to_string(), + status: if success { + "success".to_string() + } else { + "error".to_string() + }, + }) + .inc(); + + self.operation_duration_seconds + .get_or_create(&OperationDurationLabels { + kind: kind.to_string(), + operation: operation.to_string(), + }) + .observe(duration.as_secs_f64()); + } + + pub fn observe_samples(&self, kind: &str, operation: &str, samples: usize) { + let Ok(samples) = u64::try_from(samples) else { + return; + }; + + self.samples_processed_total + .get_or_create(&DataVolumeLabels { + kind: kind.to_string(), + operation: operation.to_string(), + }) + .inc_by(samples); + } + + pub fn observe_series(&self, kind: &str, operation: &str, series: usize) { + let Ok(series) = u64::try_from(series) else { + return; + }; + + self.series_processed_total + .get_or_create(&DataVolumeLabels { + kind: kind.to_string(), + operation: operation.to_string(), + }) + .inc_by(series); + } + + pub fn increment_in_flight(&self) { + self.http_requests_in_flight.inc(); + } + + pub fn decrement_in_flight(&self) { + self.http_requests_in_flight.dec(); + } + + pub fn render(&self, storage_is_ready: bool) -> Result { + self.uptime_seconds.set( + self.start_time + .elapsed() + .as_secs() + .try_into() + .unwrap_or(i64::MAX), + ); + self.storage_ready.set(if storage_is_ready { 1 } else { 0 }); + + let mut encoded = String::new(); + encode(&mut encoded, &self.registry)?; + Ok(encoded) + } +} + +#[utoipa::path( + get, + path = "/prometheus/metrics", + tag = "Observability", + params( + ("include_latest_samples" = Option, Query, description = "Append the most recent sample for Prometheus-compatible series"), + ("metric" = Option, Query, description = "Optional metric name filter used when include_latest_samples=true"), + ("selector" = Option, Query, description = "Optional PromQL-style label selector used when include_latest_samples=true") + ), + responses( + (status = 200, description = "Prometheus-compatible metrics", body = String) + ) +)] +pub async fn prometheus_metrics( + State(state): State, + Query(query): Query, +) -> Result { + let mut body = state + .metrics + .render(state.storage.health_check().await.is_ok()) + .map_err(AppError::internal_server_error)?; + + let latest_metrics = render_latest_sample_metrics(&state, &query).await?; + if !latest_metrics.is_empty() { + if !body.ends_with('\n') { + body.push('\n'); + } + body.push_str(&latest_metrics); + } + + Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, PROMETHEUS_CONTENT_TYPE) + .body(Body::from(body)) + .map_err(AppError::internal_server_error) +} + +async fn render_latest_sample_metrics( + state: &HttpServerState, + query: &PrometheusMetricsQuery, +) -> Result { + if !query.include_latest_samples { + return Ok(String::new()); + } + + let label_matchers = match &query.selector { + Some(selector) => Some(parse_selector_to_matchers(selector)?), + None => None, + }; + + let mut bookmark = None; + let mut rendered = String::new(); + + loop { + let result = state + .storage + .list_series( + query.metric.as_deref(), + Some(LATEST_SERIES_PAGE_SIZE), + bookmark.as_deref(), + ) + .await?; + + for sensor in result.series { + if !is_prometheus_scrape_compatible_sensor(&sensor) { + continue; + } + + if let Some(matchers) = &label_matchers + && !sensor_matches_matchers(&sensor, matchers) + { + continue; + } + + let Some(sensor_data) = state + .storage + .query_sensor_data_latest(&sensor.uuid.to_string(), None, None) + .await? + else { + continue; + }; + + if let Some(line) = sensor_data_to_latest_prometheus_line(&sensor_data) { + rendered.push_str(&line); + rendered.push('\n'); + } + } + + match result.bookmark { + Some(next) => bookmark = Some(next), + None => break, + } + } + + Ok(rendered) +} + +fn is_prometheus_scrape_compatible_sensor(sensor: &crate::datamodel::Sensor) -> bool { + matches!( + sensor.sensor_type, + crate::datamodel::SensorType::Integer + | crate::datamodel::SensorType::Numeric + | crate::datamodel::SensorType::Float + ) && is_valid_prometheus_metric_name(&sensor.name) + && sensor + .labels + .iter() + .all(|(name, _)| is_valid_prometheus_label_name(name)) +} + +fn is_valid_prometheus_metric_name(name: &str) -> bool { + let mut chars = name.chars(); + let Some(first) = chars.next() else { + return false; + }; + + if !(first.is_ascii_alphabetic() || first == '_' || first == ':') { + return false; + } + + chars.all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == ':') +} + +fn is_valid_prometheus_label_name(name: &str) -> bool { + let mut chars = name.chars(); + let Some(first) = chars.next() else { + return false; + }; + + if name.starts_with("__") || !(first.is_ascii_alphabetic() || first == '_') { + return false; + } + + chars.all(|ch| ch.is_ascii_alphanumeric() || ch == '_') +} + +fn sensor_data_to_latest_prometheus_line( + sensor_data: &crate::datamodel::SensorData, +) -> Option { + let labels = + crate::parsing::prometheus::converter::build_prometheus_labels(&sensor_data.sensor); + let metric_name = labels + .iter() + .find(|label| label.name == "__name__")? + .value + .clone(); + let (value, timestamp_ms) = latest_sample_value_and_timestamp(&sensor_data.samples)?; + + let rendered_labels = labels + .into_iter() + .filter(|label| label.name != "__name__") + .map(|label| { + format!( + r#"{}="{}""#, + label.name, + escape_prometheus_label_value(&label.value) + ) + }) + .collect::>() + .join(","); + + let sample_value = format_prometheus_sample_value(value); + if rendered_labels.is_empty() { + Some(format!("{} {} {}", metric_name, sample_value, timestamp_ms)) + } else { + Some(format!( + "{}{{{}}} {} {}", + metric_name, rendered_labels, sample_value, timestamp_ms + )) + } +} + +fn latest_sample_value_and_timestamp( + samples: &crate::datamodel::TypedSamples, +) -> Option<(f64, i64)> { + match samples { + crate::datamodel::TypedSamples::Float(values) => values + .last() + .map(|sample| (sample.value, datetime_to_millis(&sample.datetime))), + crate::datamodel::TypedSamples::Integer(values) => values + .last() + .map(|sample| (sample.value as f64, datetime_to_millis(&sample.datetime))), + crate::datamodel::TypedSamples::Numeric(values) => values.last().and_then(|sample| { + use rust_decimal::prelude::ToPrimitive; + + sample + .value + .to_f64() + .map(|value| (value, datetime_to_millis(&sample.datetime))) + }), + crate::datamodel::TypedSamples::String(_) + | crate::datamodel::TypedSamples::Boolean(_) + | crate::datamodel::TypedSamples::Location(_) + | crate::datamodel::TypedSamples::Blob(_) + | crate::datamodel::TypedSamples::Json(_) => None, + } +} + +fn escape_prometheus_label_value(value: &str) -> String { + value + .replace('\\', r#"\\"#) + .replace('\n', r#"\n"#) + .replace('"', r#"\""#) +} + +fn format_prometheus_sample_value(value: f64) -> String { + if value.is_nan() { + "NaN".to_string() + } else if value == f64::INFINITY { + "+Inf".to_string() + } else if value == f64::NEG_INFINITY { + "-Inf".to_string() + } else { + value.to_string() + } +} + +pub async fn track_http_metrics( + State(metrics): State>, + request: Request, + next: Next, +) -> Response { + let method = request.method().clone(); + let path = request + .extensions() + .get::() + .map(|matched_path| matched_path.as_str().to_string()) + .unwrap_or_else(|| request.uri().path().to_string()); + + if path == "/prometheus/metrics" { + return next.run(request).await; + } + + metrics.increment_in_flight(); + let start = Instant::now(); + let response = next.run(request).await; + metrics.decrement_in_flight(); + metrics.observe_http_request(&method, &path, response.status(), start.elapsed()); + response +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::datamodel::{Sample, SensAppDateTime, Sensor, SensorType, TypedSamples}; + use smallvec::smallvec; + + fn create_sensor(name: &str, sensor_type: SensorType, labels: Vec<(&str, &str)>) -> Sensor { + Sensor::new_without_uuid( + name.to_string(), + sensor_type, + None, + Some( + labels + .into_iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(), + ), + ) + .unwrap() + } + + #[test] + fn metrics_render_contains_registered_metrics() { + let metrics = HttpMetrics::new(); + metrics.observe_http_request( + &Method::GET, + "/health/live", + StatusCode::OK, + Duration::from_millis(5), + ); + metrics.observe_operation_result("read", "list_metrics", Duration::from_millis(2), true); + metrics.observe_series("read", "list_metrics", 3); + metrics.observe_samples("read", "list_metrics", 7); + let body = metrics.render(true).unwrap(); + + assert!(body.contains("sensapp_http_requests_total")); + assert!(body.contains("sensapp_operations_total")); + assert!(body.contains("sensapp_samples_processed_total")); + assert!(body.contains("sensapp_uptime_seconds")); + assert!(body.contains("sensapp_storage_ready 1")); + } + + #[test] + fn scrape_compatibility_requires_numeric_sensor_and_valid_names() { + let compatible = create_sensor( + "room_temperature_celsius", + SensorType::Float, + vec![("room", "lab")], + ); + let invalid_metric = create_sensor("demo-temperature", SensorType::Float, vec![]); + let invalid_label = create_sensor( + "room_temperature_celsius", + SensorType::Float, + vec![("bad-label", "lab")], + ); + let string_sensor = create_sensor("status_text", SensorType::String, vec![]); + + assert!(is_prometheus_scrape_compatible_sensor(&compatible)); + assert!(!is_prometheus_scrape_compatible_sensor(&invalid_metric)); + assert!(!is_prometheus_scrape_compatible_sensor(&invalid_label)); + assert!(!is_prometheus_scrape_compatible_sensor(&string_sensor)); + } + + #[test] + fn latest_prometheus_line_uses_original_timestamp() { + let sensor = create_sensor( + "room_temperature_celsius", + SensorType::Float, + vec![("room", "lab")], + ); + let sensor_data = crate::datamodel::SensorData { + sensor, + samples: TypedSamples::Float(smallvec![Sample { + datetime: SensAppDateTime::from_unix_seconds(1_704_067_201.0), + value: 42.5, + }]), + }; + + let line = sensor_data_to_latest_prometheus_line(&sensor_data).unwrap(); + assert_eq!( + line, + r#"room_temperature_celsius{room="lab"} 42.5 1704067201000"# + ); + } +} diff --git a/src/ingestors/http/mod.rs b/src/http/mod.rs similarity index 84% rename from src/ingestors/http/mod.rs rename to src/http/mod.rs index ba39915a..95c6b6c7 100644 --- a/src/ingestors/http/mod.rs +++ b/src/http/mod.rs @@ -1,7 +1,9 @@ pub mod app_error; +pub mod auth; pub mod crud; pub mod health; pub mod influxdb; +pub mod metrics; pub mod prometheus_read; pub mod prometheus_write; pub mod server; diff --git a/src/ingestors/http/prometheus_read.rs b/src/http/prometheus_read.rs similarity index 60% rename from src/ingestors/http/prometheus_read.rs rename to src/http/prometheus_read.rs index 50e6c577..34343c67 100644 --- a/src/ingestors/http/prometheus_read.rs +++ b/src/http/prometheus_read.rs @@ -3,7 +3,7 @@ use crate::datamodel::sensapp_datetime::SensAppDateTimeExt; use crate::parsing::prometheus::chunk_encoder::ChunkEncoder; use crate::parsing::prometheus::converter::{build_prometheus_labels, sensor_data_to_timeseries}; use crate::parsing::prometheus::remote_read_models::{ - QueryResult, ReadResponse, read_request::ResponseType, + Query, QueryResult, ReadHints, ReadResponse, read_request::ResponseType, }; use crate::parsing::prometheus::remote_read_parser::{ parse_remote_read_request, serialize_read_response, @@ -11,6 +11,7 @@ use crate::parsing::prometheus::remote_read_parser::{ use crate::parsing::prometheus::remote_write_models::Sample as PromSample; use crate::parsing::prometheus::stream_writer::StreamWriter; use crate::storage::query::LabelMatcher; +use crate::storage::{Aggregation, SensorDataQueryOptions}; use super::{app_error::AppError, state::HttpServerState}; use axum::{ @@ -19,8 +20,104 @@ use axum::{ http::{HeaderMap, StatusCode}, response::Response, }; +use std::time::Instant; use tokio_util::bytes::Bytes; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; + +fn aggregation_from_read_hints(hints: &ReadHints) -> Option { + match hints.func.trim().to_ascii_lowercase().as_str() { + "avg_over_time" | "avg" => Some(Aggregation::Avg), + "min_over_time" | "min" => Some(Aggregation::Min), + "max_over_time" | "max" => Some(Aggregation::Max), + "sum_over_time" | "sum" => Some(Aggregation::Sum), + "count_over_time" | "count" => Some(Aggregation::Count), + "first_over_time" | "first" => Some(Aggregation::First), + "last_over_time" | "last" => Some(Aggregation::Last), + _ => None, + } +} + +fn query_options_from_read_hints(query: &Query) -> Option { + let hints = query.hints.as_ref()?; + let aggregation = aggregation_from_read_hints(hints)?; + + if hints.step_ms <= 0 { + return None; + } + + Some(SensorDataQueryOptions { + start_time: Some(SensAppDateTime::from_unix_milliseconds_i64( + query.start_timestamp_ms, + )), + end_time: Some(SensAppDateTime::from_unix_milliseconds_i64( + query.end_timestamp_ms, + )), + limit: None, + step_ms: Some(hints.step_ms), + aggregation: Some(aggregation), + simplify: None, + }) +} + +async fn query_sensor_data_for_prometheus( + state: &HttpServerState, + query: &Query, +) -> Result, AppError> { + let matchers: Vec = query.matchers.iter().map(LabelMatcher::from).collect(); + let start_time = SensAppDateTime::from_unix_milliseconds_i64(query.start_timestamp_ms); + let end_time = SensAppDateTime::from_unix_milliseconds_i64(query.end_timestamp_ms); + + if let Some(options) = query_options_from_read_hints(query) { + let discovered = state + .storage + .query_sensors_by_labels(&matchers, Some(start_time), Some(end_time), Some(1), true) + .await + .map_err(|e| { + AppError::internal_server_error(anyhow::anyhow!( + "Storage query failed during discovery: {}", + e + )) + })?; + + if let Some(hints) = &query.hints { + info!( + "Prometheus remote read: applying hints func='{}' step={}ms to {} discovered series", + hints.func, + hints.step_ms, + discovered.len() + ); + } + + let mut aggregated = Vec::with_capacity(discovered.len()); + for sensor_data in discovered { + let sensor_uuid = sensor_data.sensor.uuid.to_string(); + if let Some(sensor_data) = state + .storage + .query_sensor_data_advanced(&sensor_uuid, &options) + .await + .map_err(|e| { + AppError::internal_server_error(anyhow::anyhow!( + "Advanced storage query failed: {}", + e + )) + })? + && !sensor_data.samples.is_empty() + { + aggregated.push(sensor_data); + } + } + + return Ok(aggregated); + } + + state + .storage + .query_sensors_by_labels(&matchers, Some(start_time), Some(end_time), None, true) + .await + .map_err(|e| { + AppError::internal_server_error(anyhow::anyhow!("Storage query failed: {}", e)) + }) +} fn verify_read_headers(headers: &HeaderMap) -> Result<(), AppError> { // Check that we have the right content encoding, that must be snappy @@ -107,104 +204,110 @@ pub async fn prometheus_remote_read( headers: HeaderMap, bytes: Bytes, ) -> Result, AppError> { - debug!("Prometheus remote read: received {} bytes", bytes.len()); + let metrics = state.metrics.clone(); + let started = Instant::now(); - // Verify headers - verify_read_headers(&headers)?; + let result = async move { + debug!("Prometheus remote read: received {} bytes", bytes.len()); - // Parse the read request - let read_request = parse_remote_read_request(&bytes).map_err(|e| { - AppError::bad_request(anyhow::anyhow!("Failed to parse read request: {}", e)) - })?; + verify_read_headers(&headers)?; - info!( - "Prometheus remote read: Processing {} queries", - read_request.queries.len() - ); + let read_request = parse_remote_read_request(&bytes).map_err(|e| { + AppError::bad_request(anyhow::anyhow!("Failed to parse read request: {}", e)) + })?; - // Log detailed information about each query for debugging - for (i, query) in read_request.queries.iter().enumerate() { - println!( - "[DEBUG PROM READ] Query {}: time range {}ms - {}ms ({} matchers)", - i, - query.start_timestamp_ms, - query.end_timestamp_ms, - query.matchers.len() - ); info!( - "Query {}: time range {}ms - {}ms ({} matchers)", - i, - query.start_timestamp_ms, - query.end_timestamp_ms, - query.matchers.len() + "Prometheus remote read: Processing {} queries", + read_request.queries.len() ); - for matcher in &query.matchers { + for (i, query) in read_request.queries.iter().enumerate() { println!( - "[DEBUG PROM READ] Matcher: {}={} (type={})", - matcher.name, matcher.value, matcher.r#type + "[DEBUG PROM READ] Query {}: time range {}ms - {}ms ({} matchers)", + i, + query.start_timestamp_ms, + query.end_timestamp_ms, + query.matchers.len() ); - debug!( - " Matcher: {}={} (type={})", - matcher.name, matcher.value, matcher.r#type + info!( + "Query {}: time range {}ms - {}ms ({} matchers)", + i, + query.start_timestamp_ms, + query.end_timestamp_ms, + query.matchers.len() ); + + for matcher in &query.matchers { + println!( + "[DEBUG PROM READ] Matcher: {}={} (type={})", + matcher.name, matcher.value, matcher.r#type + ); + debug!( + " Matcher: {}={} (type={})", + matcher.name, matcher.value, matcher.r#type + ); + } + + if let Some(hints) = &query.hints { + debug!(" Hints: step={}ms, func='{}'", hints.step_ms, hints.func); + } } - if let Some(hints) = &query.hints { - debug!(" Hints: step={}ms, func='{}'", hints.step_ms, hints.func); + debug!( + "Accepted response types: {:?}", + read_request.accepted_response_types + ); + + let use_streaming = read_request + .accepted_response_types + .contains(&(ResponseType::StreamedXorChunks as i32)); + + if use_streaming { + handle_streamed_response(&state, &read_request).await + } else { + handle_samples_response(&state, &read_request).await } } + .await; - debug!( - "Accepted response types: {:?}", - read_request.accepted_response_types + metrics.observe_operation_result( + "read", + "prometheus_remote_read", + started.elapsed(), + result.is_ok(), ); - - // Check if client accepts streamed XOR chunks - let use_streaming = read_request - .accepted_response_types - .contains(&(ResponseType::StreamedXorChunks as i32)); - - if use_streaming { - // Return streamed chunked response - handle_streamed_response(&state, &read_request).await - } else { - // Return standard SAMPLES response - handle_samples_response(&state, &read_request).await + if let Ok((_, series, samples)) = &result { + metrics.observe_series("read", "prometheus_remote_read", *series); + metrics.observe_samples("read", "prometheus_remote_read", *samples); } + + result.map(|(response, _, _)| response) } /// Handle standard SAMPLES response type async fn handle_samples_response( state: &HttpServerState, read_request: &crate::parsing::prometheus::remote_read_models::ReadRequest, -) -> Result, AppError> { +) -> Result<(Response, usize, usize), AppError> { let mut results = Vec::with_capacity(read_request.queries.len()); + let mut series_count = 0usize; + let mut sample_count = 0usize; for query in &read_request.queries { - // Convert Prometheus matchers to SensApp matchers - let matchers: Vec = query.matchers.iter().map(LabelMatcher::from).collect(); - - // Convert timestamps from milliseconds to SensAppDateTime - let start_time = SensAppDateTime::from_unix_milliseconds_i64(query.start_timestamp_ms); - let end_time = SensAppDateTime::from_unix_milliseconds_i64(query.end_timestamp_ms); - - // Query storage (numeric_only=true for Prometheus compatibility) - let sensor_data = state - .storage - .query_sensors_by_labels(&matchers, Some(start_time), Some(end_time), None, true) - .await - .map_err(|e| { - AppError::internal_server_error(anyhow::anyhow!("Storage query failed: {}", e)) - })?; + let sensor_data = query_sensor_data_for_prometheus(state, query).await?; + sample_count += sensor_data + .iter() + .map(|series| series.samples.len()) + .sum::(); debug!("Query returned {} sensors", sensor_data.len()); // Convert to Prometheus TimeSeries - let timeseries = sensor_data + let timeseries: Vec<_> = sensor_data .iter() .filter_map(sensor_data_to_timeseries) .collect(); + series_count += timeseries.len(); results.push(QueryResult { timeseries }); } @@ -222,41 +325,31 @@ async fn handle_samples_response( ); // Build HTTP response with appropriate headers - Response::builder() + let response = Response::builder() .status(StatusCode::OK) .header("content-type", "application/x-protobuf") .header("content-encoding", "snappy") .body(axum::body::Body::from(response_bytes)) .map_err(|e| { AppError::internal_server_error(anyhow::anyhow!("Failed to build response: {}", e)) - }) + })?; + + Ok((response, series_count, sample_count)) } /// Handle STREAMED_XOR_CHUNKS response type async fn handle_streamed_response( state: &HttpServerState, read_request: &crate::parsing::prometheus::remote_read_models::ReadRequest, -) -> Result, AppError> { +) -> Result<(Response, usize, usize), AppError> { // Prometheus expects ONE ChunkedReadResponse per series, not one per query // Each message contains exactly one series let mut chunked_responses = Vec::new(); + let mut series_count = 0usize; + let mut sample_count = 0usize; for (query_index, query) in read_request.queries.iter().enumerate() { - // Convert Prometheus matchers to SensApp matchers - let matchers: Vec = query.matchers.iter().map(LabelMatcher::from).collect(); - - // Convert timestamps from milliseconds to SensAppDateTime - let start_time = SensAppDateTime::from_unix_milliseconds_i64(query.start_timestamp_ms); - let end_time = SensAppDateTime::from_unix_milliseconds_i64(query.end_timestamp_ms); - - // Query storage (numeric_only=true for Prometheus compatibility) - let sensor_data = state - .storage - .query_sensors_by_labels(&matchers, Some(start_time), Some(end_time), None, true) - .await - .map_err(|e| { - AppError::internal_server_error(anyhow::anyhow!("Storage query failed: {}", e)) - })?; + let sensor_data = query_sensor_data_for_prometheus(state, query).await?; println!( "[DEBUG PROM READ] Query {} returned {} sensors from storage", @@ -298,6 +391,7 @@ async fn handle_streamed_response( Some(s) => s, None => continue, // Skip non-numeric types }; + sample_count += samples.len(); println!( "[DEBUG PROM READ] Encoding {} samples for sensor {} (first ts: {}, last ts: {})", @@ -310,13 +404,22 @@ async fn handle_streamed_response( // Encode as XOR chunks let chunked_series = match ChunkEncoder::encode_series(labels, samples) { Ok(cs) => cs, - Err(_) => continue, // Skip on encoding error + Err(error) => { + warn!( + sensor = %sd.sensor.name, + query_index, + error = %error, + "Failed to encode Prometheus remote read chunks" + ); + continue; + } }; // Create ONE response per series (this is what Prometheus expects!) let chunked_response = ChunkEncoder::create_response(query_index as i64, vec![chunked_series]); chunked_responses.push(chunked_response); + series_count += 1; } println!( @@ -346,7 +449,7 @@ async fn handle_streamed_response( ); // Build HTTP response with appropriate headers for streaming - Response::builder() + let response = Response::builder() .status(StatusCode::OK) .header( "content-type", @@ -355,7 +458,9 @@ async fn handle_streamed_response( .body(axum::body::Body::from(body)) .map_err(|e| { AppError::internal_server_error(anyhow::anyhow!("Failed to build response: {}", e)) - }) + })?; + + Ok((response, series_count, sample_count)) } /// Extract Prometheus samples from SensorData for chunk encoding. @@ -411,6 +516,7 @@ fn extract_prom_samples_for_chunks( #[cfg(test)] mod tests { use super::*; + use crate::parsing::prometheus::remote_read_models::ReadHints; use axum::http::HeaderValue; fn create_test_headers() -> HeaderMap { @@ -463,4 +569,61 @@ mod tests { ); assert!(verify_read_headers(&headers).is_err()); } + + #[test] + fn test_aggregation_from_read_hints() { + let hints = ReadHints { + step_ms: 60_000, + func: "avg_over_time".to_string(), + start_ms: 0, + end_ms: 0, + grouping: vec![], + by: false, + range_ms: 60_000, + }; + + assert_eq!(aggregation_from_read_hints(&hints), Some(Aggregation::Avg)); + + let unknown = ReadHints { + func: "rate".to_string(), + ..hints + }; + assert_eq!(aggregation_from_read_hints(&unknown), None); + } + + #[test] + fn test_query_options_from_read_hints_requires_supported_func() { + let query = Query { + start_timestamp_ms: 1_000, + end_timestamp_ms: 5_000, + matchers: vec![], + hints: Some(ReadHints { + step_ms: 2_000, + func: "max_over_time".to_string(), + start_ms: 1_000, + end_ms: 5_000, + grouping: vec![], + by: false, + range_ms: 2_000, + }), + }; + + let options = query_options_from_read_hints(&query).unwrap(); + assert_eq!(options.step_ms, Some(2_000)); + assert_eq!(options.aggregation, Some(Aggregation::Max)); + + let unsupported = Query { + hints: Some(ReadHints { + step_ms: 2_000, + func: "rate".to_string(), + start_ms: 1_000, + end_ms: 5_000, + grouping: vec![], + by: false, + range_ms: 2_000, + }), + ..query + }; + assert!(query_options_from_read_hints(&unsupported).is_none()); + } } diff --git a/src/ingestors/http/prometheus.rs b/src/http/prometheus_write.rs similarity index 57% rename from src/ingestors/http/prometheus.rs rename to src/http/prometheus_write.rs index 1707fd0f..0f0a0109 100644 --- a/src/ingestors/http/prometheus.rs +++ b/src/http/prometheus_write.rs @@ -15,6 +15,7 @@ use axum::{ extract::State, http::{HeaderMap, StatusCode}, }; +use std::time::Instant; use tokio_util::bytes::Bytes; use tracing::{debug, info}; @@ -25,6 +26,7 @@ use tracing::{debug, info}; /// - `content-type`: must be "application/x-protobuf" (protobuf format) /// - `x-prometheus-remote-write-version`: must be "0.1.0" (API version) fn verify_headers(headers: &HeaderMap) -> Result<(), AppError> { + // Check that we have the right content encoding, that must be snappy match headers.get("content-encoding") { Some(content_encoding) => match content_encoding.to_str() { Ok("snappy") | Ok("SNAPPY") => {} @@ -108,80 +110,94 @@ pub async fn publish_prometheus( headers: HeaderMap, bytes: Bytes, ) -> Result { - debug!("Prometheus remote write: received {} bytes", bytes.len()); + let metrics = state.metrics.clone(); + let started = Instant::now(); - // Verify headers - verify_headers(&headers)?; + let result = async move { + debug!("Prometheus remote write: received {} bytes", bytes.len()); - // Parse the content - let write_request = parse_remote_write_request(&bytes)?; + verify_headers(&headers)?; - // Regularly, prometheus sends metadata on the undocumented reserved field, - // so we stop immediately when it happens. - if write_request.timeseries.is_empty() { - return Ok(StatusCode::NO_CONTENT); - } + let write_request = parse_remote_write_request(&bytes)?; - debug!("Processing {} timeseries", write_request.timeseries.len()); - - let mut batch_builder = BatchBuilder::new()?; - for time_serie in write_request.timeseries { - let mut labels = SensAppLabels::with_capacity(time_serie.labels.len()); - let mut name: Option = None; - let mut unit: Option = None; - // Extract special labels: __name__ (metric name) and "unit" (custom SensApp field) - // All labels including these are stored as-is in SensApp for full metadata preservation - for label in time_serie.labels { - match label.name.as_str() { - "__name__" => { - name = Some(label.value.clone()); - } - "unit" => { - unit = Some(Unit::new(label.value.clone(), None)); + if write_request.timeseries.is_empty() { + return Ok::<_, AppError>((StatusCode::NO_CONTENT, 0usize, 0usize)); + } + + debug!("Processing {} timeseries", write_request.timeseries.len()); + + let mut batch_builder = BatchBuilder::new()?; + let mut series = 0usize; + let mut sample_count = 0usize; + for time_serie in write_request.timeseries { + let mut labels = SensAppLabels::with_capacity(time_serie.labels.len()); + let mut name: Option = None; + let mut unit: Option = None; + for label in time_serie.labels { + match label.name.as_str() { + "__name__" => { + name = Some(label.value.clone()); + } + "unit" => { + unit = Some(Unit::new(label.value.clone(), None)); + } + _ => {} } - _ => {} + labels.push((label.name, label.value)); } - labels.push((label.name, label.value)); + let name = match name { + Some(name) => name, + None => { + return Err(AppError::bad_request(anyhow::anyhow!( + "A time serie is missing its __name__ label" + ))); + } + }; + + let sensor = Sensor::new_without_uuid(name, SensorType::Float, unit, Some(labels))?; + + let samples = TypedSamples::Float( + time_serie + .samples + .into_iter() + .map(|sample| Sample { + datetime: SensAppDateTime::from_unix_milliseconds_i64(sample.timestamp), + value: sample.value, + }) + .collect(), + ); + + series += 1; + sample_count += samples.len(); + batch_builder.add(Arc::new(sensor), samples).await?; } - let name = match name { - Some(name) => name, - None => { - return Err(AppError::bad_request(anyhow::anyhow!( - "A time serie is missing its __name__ label" - ))); - } - }; - - // Prometheus has a very simple model, it's always a float. - let sensor = Sensor::new_without_uuid(name, SensorType::Float, unit, Some(labels))?; - - // We can now add the samples - let samples = TypedSamples::Float( - time_serie - .samples - .into_iter() - .map(|sample| Sample { - datetime: SensAppDateTime::from_unix_milliseconds_i64(sample.timestamp), - value: sample.value, - }) - .collect(), - ); - - batch_builder.add(Arc::new(sensor), samples).await?; - } - match batch_builder.send_what_is_left(state.storage.clone()).await { - Ok(true) => { - info!("Prometheus: Batch sent successfully"); - } - Ok(false) => { - debug!("Prometheus: No data to send"); - } - Err(error) => { - return Err(AppError::internal_server_error(error)); + match batch_builder.send_what_is_left(state.storage.clone()).await { + Ok(true) => { + info!("Prometheus: Batch sent successfully"); + } + Ok(false) => { + debug!("Prometheus: No data to send"); + } + Err(error) => { + return Err(AppError::internal_server_error(error)); + } } + + Ok((StatusCode::NO_CONTENT, series, sample_count)) + } + .await; + + metrics.observe_operation_result( + "write", + "prometheus_remote_write", + started.elapsed(), + result.is_ok(), + ); + if let Ok((_, series, samples)) = &result { + metrics.observe_series("write", "prometheus_remote_write", *series); + metrics.observe_samples("write", "prometheus_remote_write", *samples); } - // OK no content - Ok(StatusCode::NO_CONTENT) + result.map(|(status, _, _)| status) } diff --git a/src/ingestors/http/server.rs b/src/http/server.rs similarity index 69% rename from src/ingestors/http/server.rs rename to src/http/server.rs index 2d8dca69..4d4ba2b4 100644 --- a/src/ingestors/http/server.rs +++ b/src/http/server.rs @@ -1,20 +1,26 @@ use super::app_error::AppError; -use super::crud::{get_series_data, list_metrics, list_series}; +use super::auth::{require_read_auth, require_write_auth}; +use super::crud::{ + get_series_availability, get_series_data, get_series_last_sample, list_metrics, list_series, +}; use super::influxdb::publish_influxdb; +use super::metrics::{prometheus_metrics, track_http_metrics}; use super::prometheus_read::prometheus_remote_read; use super::prometheus_write::publish_prometheus; use super::simple_promql::simple_promql_query; use super::state::HttpServerState; use crate::config; -use crate::importers::csv::publish_csv_async; -use crate::ingestors::http::crud::{ - __path_get_series_data, __path_list_metrics, __path_list_series, +use crate::http::crud::{ + __path_get_series_availability, __path_get_series_data, __path_get_series_last_sample, + __path_list_metrics, __path_list_series, }; -use crate::ingestors::http::health::{__path_liveness, __path_readiness, liveness, readiness}; -use crate::ingestors::http::influxdb::__path_publish_influxdb; -use crate::ingestors::http::prometheus_read::__path_prometheus_remote_read; -use crate::ingestors::http::prometheus_write::__path_publish_prometheus; -use crate::ingestors::http::simple_promql::__path_simple_promql_query; +use crate::http::health::{__path_liveness, __path_readiness, liveness, readiness}; +use crate::http::influxdb::__path_publish_influxdb; +use crate::http::metrics::__path_prometheus_metrics; +use crate::http::prometheus_read::__path_prometheus_remote_read; +use crate::http::prometheus_write::__path_publish_prometheus; +use crate::http::simple_promql::__path_simple_promql_query; +use crate::importers::csv::publish_csv_async; use crate::storage::StorageInstance; use anyhow::Result; use axum::Json; @@ -30,7 +36,7 @@ use futures::TryStreamExt; use std::io; use std::net::SocketAddr; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; use tower::ServiceBuilder; use tower_http::trace; use tower_http::{ServiceBuilderExt, timeout::TimeoutLayer, trace::TraceLayer}; @@ -43,11 +49,12 @@ use utoipa_scalar::{Scalar, Servable as ScalarServable}; tags( (name = "SensApp", description = "SensApp API"), (name = "InfluxDB", description = "InfluxDB Write API"), + (name = "Observability", description = "Prometheus-compatible service metrics"), (name = "Prometheus", description = "Prometheus Remote Write and Read API"), (name = "Admin", description = "Administrative operations"), (name = "Health", description = "Health check endpoints"), ), - paths(frontpage, publish_sensors_data, list_metrics, list_series, get_series_data, publish_influxdb, publish_prometheus, prometheus_remote_read, simple_promql_query, vacuum_database, liveness, readiness), + paths(frontpage, publish_sensors_data, prometheus_metrics, list_metrics, list_series, get_series_data, get_series_last_sample, get_series_availability, publish_influxdb, publish_prometheus, prometheus_remote_read, simple_promql_query, vacuum_database, liveness, readiness), )] struct ApiDoc; @@ -78,38 +85,60 @@ pub async fn run_http_server(state: HttpServerState, address: SocketAddr) -> Res .compression() .into_inner(); - // Create our application with a single route - let app = Router::new() + // Create our application with route groups split by auth requirements. + // + // Public routes — always accessible (health checks, docs, prometheus scrape): + let public_routes = Router::new() .route("/", get(frontpage)) .merge(Scalar::with_url("/docs", ApiDoc::openapi())) - // Metrics and Series CRUD + .route("/prometheus/metrics", get(prometheus_metrics)) + .route("/health/live", get(liveness)) + .route("/health/ready", get(readiness)); + + // Read-protected routes — require a valid JWT with "read" scope when auth is enabled: + let read_routes = Router::new() .route("/metrics", get(list_metrics)) .route("/series", get(list_series)) .route("/series/{series_uuid}", get(get_series_data)) - // SensApp Write API + .route("/series/{series_uuid}/last", get(get_series_last_sample)) + .route( + "/series/{series_uuid}/availability", + get(get_series_availability), + ) + .route("/api/v1/query", get(simple_promql_query)) + .route( + "/api/v1/prometheus_remote_read", + post(prometheus_remote_read).layer(max_body_layer), + ) + .route_layer(axum::middleware::from_fn_with_state( + state.auth.clone(), + require_read_auth, + )); + + // Write-protected routes — require a valid JWT with "write" scope when auth is enabled: + let write_routes = Router::new() .route("/publish", post(publish_sensors_data).layer(max_body_layer)) - // InfluxDB Write API .route( "/api/v2/write", post(publish_influxdb).layer(max_body_layer), ) - // Prometheus Remote Write API .route( "/api/v1/prometheus_remote_write", post(publish_prometheus).layer(max_body_layer), ) - // Prometheus Remote Read API - .route( - "/api/v1/prometheus_remote_read", - post(prometheus_remote_read).layer(max_body_layer), - ) - // Simple PromQL Query API - .route("/api/v1/query", get(simple_promql_query)) - // Admin API .route("/api/v1/admin/vacuum", post(vacuum_database)) - // Health check endpoints - .route("/health/live", get(liveness)) - .route("/health/ready", get(readiness)) + .route_layer(axum::middleware::from_fn_with_state( + state.auth.clone(), + require_write_auth, + )); + + let app = public_routes + .merge(read_routes) + .merge(write_routes) + .layer(axum::middleware::from_fn_with_state( + state.metrics.clone(), + track_http_metrics, + )) .layer(middleware) .with_state(state); @@ -158,7 +187,7 @@ async fn frontpage(State(state): State) -> Result, /// Accepts sensor data in one of the following formats: /// - **SenML JSON** (RFC 8428): `Content-Type: application/json` /// - **CSV**: `Content-Type: text/csv` or `application/csv` -/// - **Apache Arrow IPC**: `Content-Type: application/vnd.apache.arrow.file` +/// - **Apache Arrow IPC**: `Content-Type: application/vnd.apache.arrow.stream` /// /// If no Content-Type header is provided, defaults to CSV format. #[utoipa::path( @@ -180,26 +209,39 @@ async fn publish_sensors_data( headers: HeaderMap, body: axum::body::Body, ) -> Result { - let content_type = headers - .get(header::CONTENT_TYPE) - .and_then(|v| v.to_str().ok()) - .unwrap_or("text/csv"); - - match content_type { - ct if ct.contains("application/json") => { - publish_json_format(body, state.storage.clone()).await?; - } - ct if ct.contains("application/vnd.apache.arrow.file") => { - publish_arrow_format(body, state.storage.clone()).await?; - } - ct if ct.contains("text/csv") || ct.contains("application/csv") => { - publish_csv_format(body, state.storage.clone()).await?; - } - _ => { - publish_csv_format(body, state.storage.clone()).await?; + let metrics = state.metrics.clone(); + let started = Instant::now(); + + let result = async move { + let content_type = headers + .get(header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .unwrap_or("text/csv"); + + match content_type { + ct if ct.contains("application/json") => { + publish_json_format(body, state.storage.clone()).await + } + ct if ct.contains("application/vnd.apache.arrow.stream") + || ct.contains("application/vnd.apache.arrow.file") => + { + publish_arrow_format(body, state.storage.clone()).await + } + ct if ct.contains("text/csv") || ct.contains("application/csv") => { + publish_csv_format(body, state.storage.clone()).await + } + _ => publish_csv_format(body, state.storage.clone()).await, } } + .await; + metrics.observe_operation_result("write", "native_publish", started.elapsed(), result.is_ok()); + if let Ok(stats) = &result { + metrics.observe_series("write", "native_publish", stats.series); + metrics.observe_samples("write", "native_publish", stats.samples); + } + + result?; Ok("ok".to_string()) } @@ -207,7 +249,7 @@ async fn publish_sensors_data( async fn publish_json_format( body: axum::body::Body, storage: Arc, -) -> Result<(), AppError> { +) -> Result { let body_bytes = axum::body::to_bytes(body, usize::MAX) .await .map_err(|e| AppError::bad_request(anyhow::anyhow!("Failed to read JSON body: {}", e)))?; @@ -222,7 +264,7 @@ async fn publish_json_format( async fn publish_arrow_format( body: axum::body::Body, storage: Arc, -) -> Result<(), AppError> { +) -> Result { let stream = body.into_data_stream(); let stream = stream.map_err(io::Error::other); let reader = stream.into_async_read(); @@ -236,7 +278,7 @@ async fn publish_arrow_format( async fn publish_csv_format( body: axum::body::Body, storage: Arc, -) -> Result<(), AppError> { +) -> Result { let stream = body.into_data_stream(); let stream = stream.map_err(io::Error::other); let reader = stream.into_async_read(); @@ -256,7 +298,7 @@ async fn publish_csv_format( pub async fn publish_senml_data( json_str: &str, storage: Arc, -) -> Result<(), AppError> { +) -> Result { use crate::datamodel::batch_builder::BatchBuilder; use crate::importers::senml::SenMLImporter; @@ -272,8 +314,12 @@ pub async fn publish_senml_data( // Convert to SensApp format and publish let mut batch_builder = BatchBuilder::new().map_err(AppError::internal_server_error)?; + let mut series = 0usize; + let mut samples = 0usize; for (_sensor_name, sensor_data) in sensor_data_list { + series += 1; + samples += sensor_data.samples.len(); let sensor = std::sync::Arc::new(sensor_data.sensor); batch_builder @@ -287,7 +333,7 @@ pub async fn publish_senml_data( .await .map_err(AppError::internal_server_error)?; - Ok(()) + Ok(crate::importers::IngestionStats::new(series, samples)) } /// Database Vacuuming @@ -310,6 +356,7 @@ async fn vacuum_database(State(state): State) -> Result bool { + cfg!(feature = "postgres") || std::env::var("TEST_DATABASE_URL").is_ok() + } + #[tokio::test] #[serial] async fn test_frontpage_handler() { + if !storage_backend_test_available() { + return; + } + use crate::storage::storage_factory::create_storage_from_connection_string; let connection_string = get_test_database_url(); + sensapp::test_utils::ensure_test_database_exists(&connection_string) + .await + .expect("Failed to create test database"); let storage = create_storage_from_connection_string(&connection_string) .await .expect("Failed to create storage"); @@ -343,7 +401,9 @@ mod tests { let state = HttpServerState { name: Arc::new("hello world".to_string()), storage, + metrics: Arc::new(HttpMetrics::new()), influxdb_with_numeric: false, + auth: None, }; let app = Router::new().route("/", get(frontpage)).with_state(state); let request = Request::builder().uri("/").body(Body::empty()).unwrap(); diff --git a/src/ingestors/http/simple_promql.rs b/src/http/simple_promql.rs similarity index 77% rename from src/ingestors/http/simple_promql.rs rename to src/http/simple_promql.rs index 53a7fb51..e8706ccf 100644 --- a/src/ingestors/http/simple_promql.rs +++ b/src/http/simple_promql.rs @@ -12,14 +12,15 @@ use crate::datamodel::SensAppDateTime; use crate::exporters::{ArrowConverter, CsvConverter, JsonlConverter, SenMLConverter}; -use crate::ingestors::http::app_error::AppError; -use crate::ingestors::http::crud::ExportFormat; -use crate::ingestors::http::state::HttpServerState; +use crate::http::app_error::AppError; +use crate::http::crud::ExportFormat; +use crate::http::state::HttpServerState; use crate::storage::query::{LabelMatcher, MatcherType}; use axum::extract::{Query, State}; use axum::response::Response; use rusty_promql_parser::{Expr, expr}; use serde::Deserialize; +use std::time::Instant; /// Default time range for instant queries (1 hour in milliseconds) const DEFAULT_LOOKBACK_MS: i64 = 3600 * 1000; @@ -79,6 +80,21 @@ struct ParsedQuery { end_time: Option, } +fn binary_operation_error_message(query: &str) -> String { + let trimmed = query.trim(); + let looks_like_hyphenated_metric = trimmed.contains('-') + && !trimmed.contains(" + ") + && !trimmed.contains(" - ") + && !trimmed.contains(" * ") + && !trimmed.contains(" / "); + + if looks_like_hyphenated_metric { + return "Binary operations (like +, -, *, /) are not supported. If you intended to query a sensor named like 'demo-temperature', PromQL parses '-' as subtraction. Query it as '{__name__=\"demo-temperature\"}[5m]' or use an underscore-friendly metric name for bare selectors.".to_string(); + } + + "Binary operations (like +, -, *, /) are not supported. Only simple selectors like 'metric_name{label=\"value\"}' or 'metric_name[5m]' are supported.".to_string() +} + /// Parse and validate a PromQL query, returning the extracted information fn parse_promql_query(query: &str) -> Result { // Parse the query @@ -154,7 +170,7 @@ fn parse_promql_query(query: &str) -> Result { "Function calls (like rate(), increase(), histogram_quantile()) are not supported. Only simple selectors like 'metric_name{{label=\"value\"}}' or 'metric_name[5m]' are supported." ))), Expr::Binary(_) => Err(AppError::bad_request(anyhow::anyhow!( - "Binary operations (like +, -, *, /) are not supported. Only simple selectors like 'metric_name{{label=\"value\"}}' or 'metric_name[5m]' are supported." + binary_operation_error_message(query) ))), Expr::Unary(_) => Err(AppError::bad_request(anyhow::anyhow!( "Unary operations are not supported. Only simple selectors like 'metric_name{{label=\"value\"}}' or 'metric_name[5m]' are supported." @@ -259,68 +275,89 @@ pub async fn simple_promql_query( State(state): State, Query(query): Query, ) -> Result { - // Parse and validate the query - let parsed = parse_promql_query(&query.query)?; - - // Execute the query (numeric_only=false since we support all types in export formats) - let results = state - .storage - .query_sensors_by_labels( - &parsed.matchers, - parsed.start_time, - parsed.end_time, - None, - false, - ) - .await?; - - // Parse format from query parameter, default to SenML/JSON - let format = match query.format.as_deref() { - Some(format_str) => ExportFormat::from_extension(format_str).ok_or_else(|| { - AppError::bad_request(anyhow::anyhow!( - "Unsupported export format '{}'. Supported formats: senml, csv, jsonl, arrow", - format_str - )) - })?, - None => ExportFormat::Senml, // Default to SenML/JSON format - }; - - // Convert based on requested format - let response = match format { - ExportFormat::Senml => { - let json_value = SenMLConverter::to_senml_json_multi(&results) - .map_err(AppError::internal_server_error)?; - axum::response::Response::builder() - .header("content-type", format.content_type()) - .body(json_value.to_string().into()) - } - ExportFormat::Csv => { - let csv_content = - CsvConverter::to_csv_multi(&results).map_err(AppError::internal_server_error)?; - axum::response::Response::builder() - .header("content-type", format.content_type()) - .body(csv_content.into()) - } - ExportFormat::Jsonl => { - let jsonl_content = JsonlConverter::to_jsonl_multi(&results) - .map_err(AppError::internal_server_error)?; - axum::response::Response::builder() - .header("content-type", format.content_type()) - .body(jsonl_content.into()) - } - ExportFormat::Arrow => { - let arrow_bytes = ArrowConverter::to_arrow_file_multi(&results) - .map_err(AppError::internal_server_error)?; - axum::response::Response::builder() - .header("content-type", format.content_type()) - .body(arrow_bytes.into()) + let metrics = state.metrics.clone(); + let started = Instant::now(); + + let result = async move { + let parsed = parse_promql_query(&query.query)?; + + let results = state + .storage + .query_sensors_by_labels( + &parsed.matchers, + parsed.start_time, + parsed.end_time, + None, + false, + ) + .await?; + + let series = results.len(); + let samples = results + .iter() + .map(|sensor_data| sensor_data.samples.len()) + .sum(); + + let format = match query.format.as_deref() { + Some(format_str) => ExportFormat::from_extension(format_str).ok_or_else(|| { + AppError::bad_request(anyhow::anyhow!( + "Unsupported export format '{}'. Supported formats: senml, csv, jsonl, arrow", + format_str + )) + })?, + None => ExportFormat::Senml, + }; + + let response = match format { + ExportFormat::Senml => { + let json_value = SenMLConverter::to_senml_json_multi(&results) + .map_err(AppError::internal_server_error)?; + axum::response::Response::builder() + .header("content-type", format.content_type()) + .body(json_value.to_string().into()) + } + ExportFormat::Csv => { + let csv_content = CsvConverter::to_csv_multi(&results) + .map_err(AppError::internal_server_error)?; + axum::response::Response::builder() + .header("content-type", format.content_type()) + .body(csv_content.into()) + } + ExportFormat::Jsonl => { + let jsonl_content = JsonlConverter::to_jsonl_multi(&results) + .map_err(AppError::internal_server_error)?; + axum::response::Response::builder() + .header("content-type", format.content_type()) + .body(jsonl_content.into()) + } + ExportFormat::Arrow => { + let arrow_bytes = ArrowConverter::to_arrow_stream_multi(&results) + .map_err(AppError::internal_server_error)?; + axum::response::Response::builder() + .header("content-type", format.content_type()) + .body(arrow_bytes.into()) + } } + .map_err(|e| { + AppError::internal_server_error(anyhow::anyhow!("Failed to build response: {}", e)) + })?; + + Ok::<_, AppError>((response, series, samples)) + } + .await; + + metrics.observe_operation_result( + "read", + "simple_promql_query", + started.elapsed(), + result.is_ok(), + ); + if let Ok((_, series, samples)) = &result { + metrics.observe_series("read", "simple_promql_query", *series); + metrics.observe_samples("read", "simple_promql_query", *samples); } - .map_err(|e| { - AppError::internal_server_error(anyhow::anyhow!("Failed to build response: {}", e)) - })?; - Ok(response) + result.map(|(response, _, _)| response) } #[cfg(test)] diff --git a/src/ingestors/http/state.rs b/src/http/state.rs similarity index 60% rename from src/ingestors/http/state.rs rename to src/http/state.rs index 14d15f40..104ee059 100644 --- a/src/ingestors/http/state.rs +++ b/src/http/state.rs @@ -1,3 +1,5 @@ +use crate::http::auth::AuthConfig; +use crate::http::metrics::HttpMetrics; use crate::storage::StorageInstance; use std::sync::Arc; @@ -8,6 +10,11 @@ pub struct HttpServerState { pub name: Arc, /// Storage backend (PostgreSQL, SQLite, etc.) pub storage: Arc, + /// Shared Prometheus metrics registry and counters + pub metrics: Arc, /// If true, InfluxDB numeric types are stored as Decimal/Numeric instead of Integer/Float pub influxdb_with_numeric: bool, + /// Optional JWT authentication configuration. + /// When `None`, all endpoints are open (no security). + pub auth: Option, } diff --git a/src/importers/arrow.rs b/src/importers/arrow.rs index 62dd3af3..86772d5e 100644 --- a/src/importers/arrow.rs +++ b/src/importers/arrow.rs @@ -1,7 +1,9 @@ use crate::{ datamodel::{ Sample, SensAppDateTime, Sensor, SensorType, TypedSamples, batch_builder::BatchBuilder, + unit::Unit, }, + importers::IngestionStats, storage::StorageInstance, }; use anyhow::{Result, anyhow}; @@ -11,7 +13,7 @@ use arrow::array::{ }; use arrow::datatypes::{DataType, TimeUnit}; use arrow::record_batch::RecordBatch; -use arrow_ipc::reader::FileReader; +use arrow_ipc::reader::{FileReader, StreamReader}; use futures::io::{AsyncRead, AsyncReadExt}; use geo::Point; use rust_decimal::Decimal; @@ -26,46 +28,66 @@ use uuid::Uuid; pub async fn publish_arrow_async( mut arrow_reader: R, storage: Arc, -) -> Result<()> { +) -> Result { // Read all data into a buffer first let mut buffer = Vec::new(); arrow_reader.read_to_end(&mut buffer).await?; // Parse the Arrow data let record_batches = parse_arrow_file(&buffer)?; + let mut stats = IngestionStats::default(); // Convert Arrow data to SensApp format and publish let mut batch_builder = BatchBuilder::new()?; for record_batch in record_batches { let sensor_data_map = convert_record_batch_to_sensors(&record_batch)?; + stats.series += sensor_data_map.len(); for (_sensor_key, (sensor, sample_entries)) in sensor_data_map { for (_datetime, typed_samples) in sample_entries { + stats.samples += typed_samples.len(); batch_builder.add(sensor.clone(), typed_samples).await?; } } } batch_builder.send_what_is_left(storage).await?; - Ok(()) + Ok(stats) } -/// Parse Arrow IPC file format from bytes +/// Parse Arrow IPC stream or file format from bytes. fn parse_arrow_file(buffer: &[u8]) -> Result> { - let cursor = Cursor::new(buffer); - let reader = FileReader::try_new(cursor, None) - .map_err(|e| anyhow!("Failed to create Arrow file reader: {}", e))?; - - let mut batches = Vec::new(); - for batch_result in reader { - let batch = - batch_result.map_err(|e| anyhow!("Failed to read Arrow batch from file: {}", e))?; - batches.push(batch); - } + let stream_batches = StreamReader::try_new(Cursor::new(buffer), None) + .ok() + .map(|reader| { + reader + .into_iter() + .map(|batch| { + batch.map_err(|e| anyhow!("Failed to read Arrow batch from stream: {}", e)) + }) + .collect::>>() + }) + .transpose()?; + + let batches = if let Some(batches) = stream_batches { + batches + } else { + let cursor = Cursor::new(buffer); + let reader = FileReader::try_new(cursor, None) + .map_err(|e| anyhow!("Failed to create Arrow file reader: {}", e))?; + + let mut batches = Vec::new(); + for batch_result in reader { + let batch = + batch_result.map_err(|e| anyhow!("Failed to read Arrow batch from file: {}", e))?; + batches.push(batch); + } + batches + }; if batches.is_empty() { - return Err(anyhow!("Arrow file contains no data batches")); + return Err(anyhow!("Arrow IPC payload contains no data batches")); } Ok(batches) @@ -75,6 +97,10 @@ fn parse_arrow_file(buffer: &[u8]) -> Result> { fn convert_record_batch_to_sensors(batch: &RecordBatch) -> Result { let schema = batch.schema(); + if batch.num_rows() == 0 { + return Err(anyhow!("Arrow IPC payload contains an empty batch")); + } + // Find required columns let timestamp_idx = find_column_index(&schema, "timestamp") .ok_or_else(|| anyhow!("Arrow data must contain 'timestamp' column"))?; @@ -113,20 +139,23 @@ fn convert_record_batch_to_sensors(batch: &RecordBatch) -> Result let timestamps = timestamps?; // Convert values based on data type - let (sensor_type, typed_samples) = + let (derived_sensor_type, typed_samples) = convert_arrow_array_to_typed_samples(value_array, ×tamps)?; // Extract sensor metadata let sensor_id = extract_sensor_id(batch, sensor_id_idx)?; let sensor_name = extract_sensor_name(batch, sensor_name_idx); + let sensor_type = extract_sensor_type(batch).unwrap_or(derived_sensor_type); + let unit = extract_sensor_unit(batch); + let labels = extract_sensor_labels(batch)?; let sensor_name = sensor_name.unwrap_or_else(|| sensor_id.to_string()); let sensor = Arc::new(Sensor { uuid: sensor_id, name: sensor_name.clone(), sensor_type, - unit: None, // Unit information not preserved in basic Arrow format - labels: SmallVec::new(), // Labels not preserved in basic Arrow format + unit, + labels, }); let mut result = HashMap::new(); @@ -302,6 +331,11 @@ fn find_column_index(schema: &arrow::datatypes::Schema, column_name: &str) -> Op } fn extract_sensor_id(batch: &RecordBatch, sensor_id_idx: Option) -> Result { + if let Some(uuid_str) = batch.schema().metadata().get("sensapp.sensor.uuid") { + return Uuid::parse_str(uuid_str) + .map_err(|e| anyhow!("Invalid UUID in sensapp.sensor.uuid metadata: {}", e)); + } + if let Some(idx) = sensor_id_idx { let array = batch .column(idx) @@ -321,6 +355,10 @@ fn extract_sensor_id(batch: &RecordBatch, sensor_id_idx: Option) -> Resul } fn extract_sensor_name(batch: &RecordBatch, sensor_name_idx: Option) -> Option { + if let Some(name) = batch.schema().metadata().get("sensapp.sensor.name") { + return Some(name.clone()); + } + sensor_name_idx.and_then(|idx| { batch .column(idx) @@ -336,6 +374,55 @@ fn extract_sensor_name(batch: &RecordBatch, sensor_name_idx: Option) -> O }) } +fn extract_sensor_type(batch: &RecordBatch) -> Option { + batch + .schema() + .metadata() + .get("sensapp.sensor.type") + .and_then(|value| value.parse().ok()) +} + +fn extract_sensor_unit(batch: &RecordBatch) -> Option { + batch + .schema() + .metadata() + .get("sensapp.sensor.unit") + .map(|value| Unit::new(value.clone(), None)) +} + +fn extract_sensor_labels(batch: &RecordBatch) -> Result> { + let schema = batch.schema(); + + let Some(labels_json) = schema.metadata().get("sensapp.sensor.labels") else { + return Ok(SmallVec::new()); + }; + + let labels_map = + serde_json::from_str::>(labels_json) + .map_err(|e| anyhow!("Invalid sensapp.sensor.labels metadata: {}", e))?; + + let mut labels: Vec<(String, String)> = labels_map + .into_iter() + .map(|(key, value)| { + let value = value.as_str().ok_or_else(|| { + anyhow!( + "Invalid sensapp.sensor.labels metadata value for key '{}'", + key + ) + })?; + Ok((key, value.to_string())) + }) + .collect::>()?; + + labels.sort_by(|(left_key, left_value), (right_key, right_value)| { + left_key + .cmp(right_key) + .then_with(|| left_value.cmp(right_value)) + }); + + Ok(SmallVec::from_vec(labels)) +} + /// Helper function to convert microseconds since epoch to SensAppDateTime fn microseconds_to_sensapp_datetime(micros: i64) -> Option { // Convert microseconds since Unix epoch to SensAppDateTime @@ -347,18 +434,19 @@ fn microseconds_to_sensapp_datetime(micros: i64) -> Option { #[cfg(test)] pub mod test_utils { use super::*; + use crate::datamodel::unit::Unit; use crate::datamodel::*; use crate::exporters::arrow::ArrowConverter; - use smallvec::{SmallVec, smallvec}; + use smallvec::smallvec; use uuid::Uuid; - pub fn create_test_arrow_file_data() -> Vec { + pub fn create_test_arrow_stream_data() -> Vec { let sensor = Sensor { uuid: Uuid::new_v4(), name: "test_sensor".to_string(), sensor_type: SensorType::Integer, - unit: None, - labels: SmallVec::new(), + unit: Some(Unit::new("Celsius".to_string(), None)), + labels: smallvec![("room".to_string(), "lab".to_string())], }; let datetime1 = SensAppDateTime::now().unwrap(); @@ -376,7 +464,7 @@ pub mod test_utils { ]); let sensor_data = SensorData::new(sensor, samples); - ArrowConverter::to_arrow_file(&sensor_data).unwrap() + ArrowConverter::to_arrow_stream(&sensor_data).unwrap() } } @@ -387,24 +475,41 @@ mod tests { #[tokio::test] async fn test_parse_arrow_file() { - let arrow_data = create_test_arrow_file_data(); + let arrow_data = create_test_arrow_stream_data(); let batches = parse_arrow_file(&arrow_data).unwrap(); assert!(!batches.is_empty()); assert_eq!(batches[0].num_rows(), 2); - assert_eq!(batches[0].num_columns(), 4); // timestamp, value, sensor_id, sensor_name + assert_eq!(batches[0].num_columns(), 2); } #[tokio::test] async fn test_convert_record_batch_to_sensors() { - let arrow_data = create_test_arrow_file_data(); + let arrow_data = create_test_arrow_stream_data(); let batches = parse_arrow_file(&arrow_data).unwrap(); let sensor_map = convert_record_batch_to_sensors(&batches[0]).unwrap(); - assert!(!sensor_map.is_empty()); - - let (_sensor, _samples) = sensor_map.values().next().unwrap(); - // Further assertions would depend on the actual data structure + assert_eq!(sensor_map.len(), 1); + + let (sensor, sample_entries) = sensor_map.values().next().unwrap(); + assert_eq!(sensor.name, "test_sensor"); + assert_eq!(sensor.sensor_type, SensorType::Integer); + assert_eq!( + sensor.unit.as_ref().map(|unit| unit.name.as_str()), + Some("Celsius") + ); + assert_eq!(sensor.labels.len(), 1); + assert_eq!(sensor.labels[0], ("room".to_string(), "lab".to_string())); + assert_eq!(sample_entries.len(), 1); + + match &sample_entries[0].1 { + TypedSamples::Integer(samples) => { + assert_eq!(samples.len(), 2); + assert_eq!(samples[0].value, 42); + assert_eq!(samples[1].value, 84); + } + other => panic!("expected integer samples, got {:?}", other), + } } #[test] diff --git a/src/importers/csv.rs b/src/importers/csv.rs index e4e8fd7f..56985a4f 100644 --- a/src/importers/csv.rs +++ b/src/importers/csv.rs @@ -3,6 +3,7 @@ use crate::{ Sample, SensAppDateTime, Sensor, SensorType, TypedSamples, batch_builder::BatchBuilder, unit::Unit, }, + importers::IngestionStats, infer::{ columns::{InferedColumn, infer_column}, datagrid::StringDataGrid, @@ -22,7 +23,7 @@ type SensorDataMap = HashMap, Vec<(SensAppDateTime, Infered pub async fn publish_csv_async( mut csv_reader: AsyncReader, storage: Arc, -) -> Result<()> { +) -> Result { // Read all CSV data into a StringDataGrid let headers = csv_reader.headers().await?.clone(); let column_names = headers.iter().map(|s| s.to_string()).collect::>(); @@ -40,6 +41,8 @@ pub async fn publish_csv_async( // Parse the CSV data let parsed_data = parse_csv_data_grid(data_grid)?; + let series = parsed_data.len(); + let samples = parsed_data.values().map(|(_, values)| values.len()).sum(); // Use BatchBuilder to publish the data let mut batch_builder = BatchBuilder::new()?; @@ -51,7 +54,7 @@ pub async fn publish_csv_async( // Send all batches to storage batch_builder.send_what_is_left(storage).await?; - Ok(()) + Ok(IngestionStats::new(series, samples)) } /// Parse CSV data grid into sensors and their samples diff --git a/src/importers/mod.rs b/src/importers/mod.rs index 0b01c72b..0e25adff 100644 --- a/src/importers/mod.rs +++ b/src/importers/mod.rs @@ -1,3 +1,15 @@ pub mod arrow; pub mod csv; pub mod senml; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct IngestionStats { + pub series: usize, + pub samples: usize, +} + +impl IngestionStats { + pub const fn new(series: usize, samples: usize) -> Self { + Self { series, samples } + } +} diff --git a/src/ingestors/http/prometheus_write.rs b/src/ingestors/http/prometheus_write.rs deleted file mode 100644 index 305529ac..00000000 --- a/src/ingestors/http/prometheus_write.rs +++ /dev/null @@ -1,180 +0,0 @@ -use std::sync::Arc; - -use crate::{ - datamodel::{ - Sample, SensAppDateTime, Sensor, SensorType, TypedSamples, batch_builder::BatchBuilder, - sensapp_datetime::SensAppDateTimeExt, sensapp_vec::SensAppLabels, unit::Unit, - }, - parsing::prometheus::remote_write_parser::parse_remote_write_request, -}; - -use super::{app_error::AppError, state::HttpServerState}; -use anyhow::Result; -use axum::{ - debug_handler, - extract::State, - http::{HeaderMap, StatusCode}, -}; -use tokio_util::bytes::Bytes; -use tracing::{debug, info}; - -fn verify_headers(headers: &HeaderMap) -> Result<(), AppError> { - // Check that we have the right content encoding, that must be snappy - match headers.get("content-encoding") { - Some(content_encoding) => match content_encoding.to_str() { - Ok("snappy") | Ok("SNAPPY") => {} - _ => { - return Err(AppError::bad_request(anyhow::anyhow!( - "Unsupported content-encoding, must be snappy" - ))); - } - }, - None => { - return Err(AppError::bad_request(anyhow::anyhow!( - "Missing content-encoding header" - ))); - } - } - - // Check that the content type is protocol buffer - match headers.get("content-type") { - Some(content_type) => match content_type.to_str() { - Ok("application/x-protobuf") | Ok("APPLICATION/X-PROTOBUF") => {} - _ => { - return Err(AppError::bad_request(anyhow::anyhow!( - "Unsupported content-type, must be application/x-protobuf" - ))); - } - }, - None => { - return Err(AppError::bad_request(anyhow::anyhow!( - "Missing content-type header" - ))); - } - } - - // Check that the remote write version is supported - match headers.get("x-prometheus-remote-write-version") { - Some(version) => match version.to_str() { - Ok("0.1.0") => {} - _ => { - return Err(AppError::bad_request(anyhow::anyhow!( - "Unsupported x-prometheus-remote-write-version, must be 0.1.0" - ))); - } - }, - None => { - return Err(AppError::bad_request(anyhow::anyhow!( - "Missing x-prometheus-remote-write-version header" - ))); - } - } - - Ok(()) -} - -/// Prometheus Remote Write API. -/// -/// Allows you to write data from Prometheus to SensApp. -/// -/// It follows the [Prometheus Remote Write specification](https://prometheus.io/docs/concepts/remote_write_spec/). -#[utoipa::path( - post, - path = "/api/v1/prometheus_remote_write", - tag = "Prometheus", - request_body( - content_type = "application/x-protobuf", - description = "Prometheus Remote Write endpoint. [Reference](https://prometheus.io/docs/concepts/remote_write_spec/)", - ), - params( - ("content-encoding" = String, Header, format = "snappy", description = "Content encoding, must be snappy"), - ("content-type" = String, Header, format = "application/x-protobuf", description = "Content type, must be application/x-protobuf"), - ("x-prometheus-remote-write-version" = String, Header, format = "0.1.0", description = "Prometheus Remote Write version, must be 0.1.0"), - ), - responses( - (status = 204, description = "No Content"), - (status = 400, description = "Bad Request", body = AppError), - (status = 500, description = "Internal Server Error", body = AppError), - ) -)] -#[debug_handler] -pub async fn publish_prometheus( - State(state): State, - headers: HeaderMap, - bytes: Bytes, -) -> Result { - debug!("Prometheus remote write: received {} bytes", bytes.len()); - - // Verify headers - verify_headers(&headers)?; - - // Parse the content - let write_request = parse_remote_write_request(&bytes)?; - - // Regularly, prometheus sends metadata on the undocumented reserved field, - // so we stop immediately when it happens. - if write_request.timeseries.is_empty() { - return Ok(StatusCode::NO_CONTENT); - } - - debug!("Processing {} timeseries", write_request.timeseries.len()); - - let mut batch_builder = BatchBuilder::new()?; - for time_serie in write_request.timeseries { - let mut labels = SensAppLabels::with_capacity(time_serie.labels.len()); - let mut name: Option = None; - let mut unit: Option = None; - for label in time_serie.labels { - match label.name.as_str() { - "__name__" => { - name = Some(label.value.clone()); - } - "unit" => { - unit = Some(Unit::new(label.value.clone(), None)); - } - _ => {} - } - labels.push((label.name, label.value)); - } - let name = match name { - Some(name) => name, - None => { - return Err(AppError::bad_request(anyhow::anyhow!( - "A time serie is missing its __name__ label" - ))); - } - }; - - // Prometheus has a very simple model, it's always a float. - let sensor = Sensor::new_without_uuid(name, SensorType::Float, unit, Some(labels))?; - - // We can now add the samples - let samples = TypedSamples::Float( - time_serie - .samples - .into_iter() - .map(|sample| Sample { - datetime: SensAppDateTime::from_unix_milliseconds_i64(sample.timestamp), - value: sample.value, - }) - .collect(), - ); - - batch_builder.add(Arc::new(sensor), samples).await?; - } - - match batch_builder.send_what_is_left(state.storage.clone()).await { - Ok(true) => { - info!("Prometheus: Batch sent successfully"); - } - Ok(false) => { - debug!("Prometheus: No data to send"); - } - Err(error) => { - return Err(AppError::internal_server_error(error)); - } - } - - // OK no content - Ok(StatusCode::NO_CONTENT) -} diff --git a/src/ingestors/mod.rs b/src/ingestors/mod.rs deleted file mode 100644 index 3883215f..00000000 --- a/src/ingestors/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod http; diff --git a/src/lib.rs b/src/lib.rs index 2fa6dd77..84d01f54 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,9 +3,9 @@ pub mod config; pub mod datamodel; pub mod exporters; +pub mod http; pub mod importers; pub mod infer; -pub mod ingestors; pub mod parsing; pub mod storage; diff --git a/src/main.rs b/src/main.rs index 0d32a51a..6719a80b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,9 @@ #![forbid(unsafe_code)] use crate::config::load_configuration; -use crate::ingestors::http::server::run_http_server; -use crate::ingestors::http::state::HttpServerState; +use crate::http::auth::AuthConfig; +use crate::http::metrics::HttpMetrics; +use crate::http::server::run_http_server; +use crate::http::state::HttpServerState; use anyhow::{Context, Result}; use std::net::SocketAddr; use std::sync::Arc; @@ -11,13 +13,19 @@ use tracing::event; mod config; mod datamodel; mod exporters; +mod http; mod importers; mod infer; -mod ingestors; mod parsing; mod storage; fn main() -> Result<()> { + // Handle `generate-token` subcommand before any server setup + let args: Vec = std::env::args().collect(); + if args.len() > 1 && args[1] == "generate-token" { + return generate_token_command(&args[2..]); + } + rustls::crypto::aws_lc_rs::default_provider() .install_default() .map_err(|e| anyhow::anyhow!("Failed to install CryptoProvider: {:?}", e))?; @@ -87,12 +95,28 @@ async fn async_main() -> Result<()> { let port = config.port; let address = SocketAddr::from((endpoint, port)); - println!("📡 Starting HTTP server on {}...", address); + println!("📡 Starting HTTP server on http://{}...", address); + // Build optional JWT authentication config + let auth = match &config.jwt_secret { + Some(secret) => { + let auth_config = AuthConfig::from_secret(secret) + .context("Failed to configure JWT authentication")?; + println!("🔐 JWT authentication enabled"); + Some(auth_config) + } + None => { + println!("🔓 No JWT secret configured — all endpoints are open"); + None + } + }; + match run_http_server( HttpServerState { name: Arc::new("SensApp".to_string()), storage, + metrics: Arc::new(HttpMetrics::new()), influxdb_with_numeric: config.influxdb_with_numeric, + auth, }, address, ) @@ -109,3 +133,79 @@ async fn async_main() -> Result<()> { } } } + +/// CLI subcommand: generate a signed JWT token. +/// +/// Usage: sensapp generate-token [OPTIONS] +/// --scope (default: "read write") +/// --duration (default: 3600) +/// --sensors (optional sensor allow list) +fn generate_token_command(args: &[String]) -> Result<()> { + load_configuration().context("Failed to load configuration")?; + let config = config::get().context("Failed to get configuration")?; + + let secret = config + .jwt_secret + .as_deref() + .context("SENSAPP_JWT_SECRET must be set to generate tokens")?; + + let auth_config = + AuthConfig::from_secret(secret).context("Failed to configure JWT authentication")?; + + if args.is_empty() { + eprintln!("Usage: sensapp generate-token [OPTIONS]"); + eprintln!(); + eprintln!("Options:"); + eprintln!(" --scope Token scope (default: \"read write\")"); + eprintln!(" --duration Token validity duration (default: 3600)"); + eprintln!(" --sensors Restrict to specific sensors"); + eprintln!(); + eprintln!("Environment:"); + eprintln!(" SENSAPP_JWT_SECRET Required. The shared secret for signing."); + std::process::exit(1); + } + + let subject = &args[0]; + let mut scope = "read write".to_string(); + let mut duration: u64 = 3600; + let mut sensors: Option> = None; + + let mut i = 1; + while i < args.len() { + match args[i].as_str() { + "--scope" => { + i += 1; + let raw = args.get(i).context("--scope requires a value")?; + scope = match raw.as_str() { + "read" => "read".to_string(), + "write" => "write".to_string(), + "readwrite" | "read write" | "read,write" => "read write".to_string(), + other => anyhow::bail!("Unknown scope: {other}. Use read, write, or readwrite"), + }; + } + "--duration" => { + i += 1; + let raw = args.get(i).context("--duration requires a value")?; + duration = raw + .parse() + .context("--duration must be a number of seconds")?; + } + "--sensors" => { + i += 1; + let raw = args.get(i).context("--sensors requires a value")?; + sensors = Some(raw.split(',').map(|s| s.trim().to_string()).collect()); + } + other => { + anyhow::bail!("Unknown option: {other}"); + } + } + i += 1; + } + + let token = auth_config + .create_token(subject, &scope, duration, sensors) + .context("Failed to create token")?; + + println!("{token}"); + Ok(()) +} diff --git a/src/parsing/prometheus/chunk_encoder.rs b/src/parsing/prometheus/chunk_encoder.rs index 7fd8549d..5f92d3e9 100644 --- a/src/parsing/prometheus/chunk_encoder.rs +++ b/src/parsing/prometheus/chunk_encoder.rs @@ -4,6 +4,8 @@ use anyhow::Result; use rusty_chunkenc::xor::{XORChunk, XORSample}; use tracing::debug; +const MAX_SAMPLES_PER_XOR_CHUNK: usize = u16::MAX as usize; + /// Encodes time series samples into XOR-compressed chunks for Prometheus remote read. pub struct ChunkEncoder; @@ -30,7 +32,16 @@ impl ChunkEncoder { labels.len() ); - // Convert samples to XORSample format for rusty-chunkenc + Ok(ChunkedSeries { + labels, + chunks: samples + .chunks(MAX_SAMPLES_PER_XOR_CHUNK) + .map(Self::encode_chunk) + .collect::>>()?, + }) + } + + fn encode_chunk(samples: &[Sample]) -> Result { let xor_samples: Vec = samples .iter() .map(|sample| XORSample { @@ -39,15 +50,12 @@ impl ChunkEncoder { }) .collect(); - // Track min/max timestamps for the chunk metadata let min_time_ms = samples.first().map(|s| s.timestamp).unwrap_or(0); let max_time_ms = samples.last().map(|s| s.timestamp).unwrap_or(0); - // Create the XOR chunk let xor_chunk = XORChunk::new(xor_samples); - // Encode just the raw XOR chunk data (NOT the full chunk format with length/type/crc) - // Prometheus remote read expects raw XOR data starting with 2-byte BE sample count + // Prometheus remote read expects raw XOR data starting with the 2-byte BE sample count. let mut encoded_data = Vec::new(); xor_chunk.write(&mut encoded_data)?; @@ -59,17 +67,11 @@ impl ChunkEncoder { max_time_ms ); - // Create the protobuf chunk - let proto_chunk = ProtoChunk { + Ok(ProtoChunk { min_time_ms, max_time_ms, r#type: chunk::Encoding::Xor as i32, data: encoded_data, - }; - - Ok(ChunkedSeries { - labels, - chunks: vec![proto_chunk], }) } @@ -98,6 +100,7 @@ impl ChunkEncoder { #[cfg(test)] mod tests { use super::*; + use rusty_chunkenc::xor::read_xor_chunk_data; #[test] fn test_encode_empty_series() { @@ -145,6 +148,47 @@ mod tests { assert!(!chunk.data.is_empty()); } + #[test] + fn test_encode_series_splits_large_sample_sets() { + let labels = vec![Label { + name: "__name__".to_string(), + value: "large_metric".to_string(), + }]; + + let total_samples = MAX_SAMPLES_PER_XOR_CHUNK + 123; + let samples: Vec = (0..total_samples) + .map(|index| Sample { + timestamp: index as i64 * 1000, + value: index as f64, + }) + .collect(); + + let series = ChunkEncoder::encode_series(labels, samples).unwrap(); + + assert_eq!(series.chunks.len(), 2); + + let first_chunk = &series.chunks[0]; + let (_, decoded_first_chunk) = read_xor_chunk_data(&first_chunk.data).unwrap(); + assert_eq!( + decoded_first_chunk.samples().len(), + MAX_SAMPLES_PER_XOR_CHUNK + ); + assert_eq!(first_chunk.min_time_ms, 0); + assert_eq!( + first_chunk.max_time_ms, + (MAX_SAMPLES_PER_XOR_CHUNK as i64 - 1) * 1000 + ); + + let second_chunk = &series.chunks[1]; + let (_, decoded_second_chunk) = read_xor_chunk_data(&second_chunk.data).unwrap(); + assert_eq!(decoded_second_chunk.samples().len(), 123); + assert_eq!( + second_chunk.min_time_ms, + MAX_SAMPLES_PER_XOR_CHUNK as i64 * 1000 + ); + assert_eq!(second_chunk.max_time_ms, (total_samples as i64 - 1) * 1000); + } + #[test] fn test_create_response() { let series1 = ChunkedSeries { diff --git a/src/parsing/prometheus/converter.rs b/src/parsing/prometheus/converter.rs index b599641e..a00f7a81 100644 --- a/src/parsing/prometheus/converter.rs +++ b/src/parsing/prometheus/converter.rs @@ -125,7 +125,7 @@ fn typed_samples_to_prom_samples(typed_samples: &TypedSamples) -> Option i64 { +pub(crate) fn datetime_to_millis(datetime: &crate::datamodel::SensAppDateTime) -> i64 { (datetime.to_unix_milliseconds()).floor() as i64 } diff --git a/src/storage/bigquery/bigquery_labels_utilities.rs b/src/storage/bigquery/bigquery_labels_utilities.rs index ef5aa9aa..60a3db5a 100644 --- a/src/storage/bigquery/bigquery_labels_utilities.rs +++ b/src/storage/bigquery/bigquery_labels_utilities.rs @@ -1,11 +1,12 @@ use std::{collections::HashSet, num::NonZeroUsize}; use crate::storage::StorageError; -use anyhow::{Result, anyhow}; +use anyhow::Result; use clru::CLruCache; use gcp_bigquery_client::model::{ query_parameter::QueryParameter, query_parameter_type::QueryParameterType, query_parameter_value::QueryParameterValue, query_request::QueryRequest, + query_response::ResultSet, }; use hybridmap::HybridMap; use once_cell::sync::Lazy; @@ -145,13 +146,14 @@ async fn get_existing_labels_name_ids( }; query_request.query_parameters = Some(vec![query_parameter]); - let mut result = bqs + let result = bqs .client() .read() .await .job() .query(bqs.project_id(), query_request) .await?; + let mut result = ResultSet::new_from_query_response(result); let mut results_map = HybridMap::with_capacity(result.row_count()); @@ -320,13 +322,14 @@ async fn get_existing_labels_description_ids( }; query_request.query_parameters = Some(vec![query_parameter]); - let mut result = bqs + let result = bqs .client() .read() .await .job() .query(bqs.project_id(), query_request) .await?; + let mut result = ResultSet::new_from_query_response(result); let mut results_map = HybridMap::with_capacity(result.row_count()); diff --git a/src/storage/bigquery/bigquery_sensors_utilities.rs b/src/storage/bigquery/bigquery_sensors_utilities.rs index f9b58f60..0635d3d6 100644 --- a/src/storage/bigquery/bigquery_sensors_utilities.rs +++ b/src/storage/bigquery/bigquery_sensors_utilities.rs @@ -1,8 +1,9 @@ use crate::storage::StorageError; -use anyhow::{Result, anyhow}; +use anyhow::Result; use gcp_bigquery_client::model::{ query_parameter::QueryParameter, query_parameter_type::QueryParameterType, query_parameter_value::QueryParameterValue, query_request::QueryRequest, + query_response::ResultSet, }; use hybridmap::HybridMap; use once_cell::sync::Lazy; @@ -139,13 +140,14 @@ async fn get_existing_sensors_ids_from_uuids( query_request.query_parameters = Some(vec![query_parameter]); - let mut result = bqs + let result = bqs .client() .read() .await .job() .query(bqs.project_id(), query_request) .await?; + let mut result = ResultSet::new_from_query_response(result); let mut results_map = HybridMap::with_capacity(result.row_count()); @@ -209,15 +211,15 @@ async fn create_sensors( .unit .as_ref() .and_then(|unit| units_map.get(&unit.name).copied()); - ProstSensor { + Ok(ProstSensor { sensor_id: *sensor_id, uuid: sensor.uuid.to_string(), name: sensor.name.clone(), r#type: sensor.sensor_type.to_string(), unit, - } + }) }) - .collect::>(); + .collect::>>()?; publish_rows(bqs, "sensors", &SENSORS_DESCRIPTOR, rows).await?; diff --git a/src/storage/bigquery/bigquery_string_values_utilities.rs b/src/storage/bigquery/bigquery_string_values_utilities.rs index 4c0a81a2..243957c2 100644 --- a/src/storage/bigquery/bigquery_string_values_utilities.rs +++ b/src/storage/bigquery/bigquery_string_values_utilities.rs @@ -1,11 +1,12 @@ use std::{collections::HashSet, num::NonZeroUsize}; use crate::storage::StorageError; -use anyhow::{Result, anyhow}; +use anyhow::Result; use clru::CLruCache; use gcp_bigquery_client::model::{ query_parameter::QueryParameter, query_parameter_type::QueryParameterType, query_parameter_value::QueryParameterValue, query_request::QueryRequest, + query_response::ResultSet, }; use hybridmap::HybridMap; use once_cell::sync::Lazy; @@ -136,13 +137,14 @@ async fn get_existing_string_values_ids( }; query_request.query_parameters = Some(vec![query_parameter]); - let mut result = bqs + let result = bqs .client() .read() .await .job() .query(bqs.project_id(), query_request) .await?; + let mut result = ResultSet::new_from_query_response(result); let mut results_map = HybridMap::with_capacity(result.row_count()); diff --git a/src/storage/bigquery/bigquery_table_descriptors.rs b/src/storage/bigquery/bigquery_table_descriptors.rs index 904dd56a..2d5876db 100644 --- a/src/storage/bigquery/bigquery_table_descriptors.rs +++ b/src/storage/bigquery/bigquery_table_descriptors.rs @@ -1,285 +1,126 @@ -use gcp_bigquery_client::storage::{ColumnType, FieldDescriptor, TableDescriptor}; +use gcp_bigquery_client::storage::{ColumnMode, ColumnType, FieldDescriptor, TableDescriptor}; use once_cell::sync::Lazy; +fn field(name: &str, number: u32, typ: ColumnType) -> FieldDescriptor { + FieldDescriptor { + name: name.to_string(), + number, + typ, + mode: ColumnMode::Nullable, + } +} + pub static UNITS_DESCRIPTOR: Lazy = Lazy::new(|| TableDescriptor { field_descriptors: vec![ - FieldDescriptor { - name: "id".to_string(), - number: 1, - typ: ColumnType::Int64, - }, - FieldDescriptor { - name: "name".to_string(), - number: 2, - typ: ColumnType::String, - }, - FieldDescriptor { - name: "description".to_string(), - number: 3, - typ: ColumnType::String, - }, + field("id", 1, ColumnType::Int64), + field("name", 2, ColumnType::String), + field("description", 3, ColumnType::String), ], }); pub static SENSORS_DESCRIPTOR: Lazy = Lazy::new(|| TableDescriptor { field_descriptors: vec![ - FieldDescriptor { - name: "sensor_id".to_string(), - number: 1, - typ: ColumnType::Int64, - }, - FieldDescriptor { - name: "uuid".to_string(), - number: 2, - typ: ColumnType::String, - }, - FieldDescriptor { - name: "name".to_string(), - number: 3, - typ: ColumnType::String, - }, - FieldDescriptor { - name: "type".to_string(), - number: 4, - typ: ColumnType::String, - }, - FieldDescriptor { - name: "unit".to_string(), - number: 5, - typ: ColumnType::Int64, - }, + field("sensor_id", 1, ColumnType::Int64), + field("uuid", 2, ColumnType::String), + field("name", 3, ColumnType::String), + field("type", 4, ColumnType::String), + field("unit", 5, ColumnType::Int64), ], }); pub static LABELS_NAME_DICTIONARY_DESCRIPTOR: Lazy = Lazy::new(|| TableDescriptor { field_descriptors: vec![ - FieldDescriptor { - name: "id".to_string(), - number: 1, - typ: ColumnType::Int64, - }, - FieldDescriptor { - name: "name".to_string(), - number: 2, - typ: ColumnType::String, - }, + field("id", 1, ColumnType::Int64), + field("name", 2, ColumnType::String), ], }); pub static LABELS_DESCRIPTION_DICTIONARY_DESCRIPTOR: Lazy = Lazy::new(|| TableDescriptor { field_descriptors: vec![ - FieldDescriptor { - name: "id".to_string(), - number: 1, - typ: ColumnType::Int64, - }, - FieldDescriptor { - name: "description".to_string(), - number: 2, - typ: ColumnType::String, - }, + field("id", 1, ColumnType::Int64), + field("description", 2, ColumnType::String), ], }); pub static LABELS_DESCRIPTOR: Lazy = Lazy::new(|| TableDescriptor { field_descriptors: vec![ - FieldDescriptor { - name: "sensor_id".to_string(), - number: 1, - typ: ColumnType::Int64, - }, - FieldDescriptor { - name: "name".to_string(), - number: 2, - typ: ColumnType::Int64, - }, - FieldDescriptor { - name: "description".to_string(), - number: 3, - typ: ColumnType::Int64, - }, + field("sensor_id", 1, ColumnType::Int64), + field("name", 2, ColumnType::Int64), + field("description", 3, ColumnType::Int64), ], }); pub static STRINGS_VALUES_DICTIONARY_DESCRIPTOR: Lazy = Lazy::new(|| TableDescriptor { field_descriptors: vec![ - FieldDescriptor { - name: "id".to_string(), - number: 1, - typ: ColumnType::Int64, - }, - FieldDescriptor { - name: "value".to_string(), - number: 2, - typ: ColumnType::String, - }, + field("id", 1, ColumnType::Int64), + field("value", 2, ColumnType::String), ], }); pub static INTEGER_VALUES_DESCRIPTOR: Lazy = Lazy::new(|| TableDescriptor { field_descriptors: vec![ - FieldDescriptor { - name: "sensor_id".to_string(), - number: 1, - typ: ColumnType::Int64, - }, - FieldDescriptor { - name: "timestamp".to_string(), - number: 2, - typ: ColumnType::Timestamp, - }, - FieldDescriptor { - name: "value".to_string(), - number: 3, - typ: ColumnType::Int64, - }, + field("sensor_id", 1, ColumnType::Int64), + field("timestamp", 2, ColumnType::String), + field("value", 3, ColumnType::Int64), ], }); pub static NUMERIC_VALUES_DESCRIPTOR: Lazy = Lazy::new(|| TableDescriptor { field_descriptors: vec![ - FieldDescriptor { - name: "sensor_id".to_string(), - number: 1, - typ: ColumnType::Int64, - }, - FieldDescriptor { - name: "timestamp".to_string(), - number: 2, - typ: ColumnType::Timestamp, - }, - FieldDescriptor { - name: "value".to_string(), - number: 3, - typ: ColumnType::Bytes, - }, + field("sensor_id", 1, ColumnType::Int64), + field("timestamp", 2, ColumnType::String), + field("value", 3, ColumnType::Bytes), ], }); pub static FLOAT_VALUES_DESCRIPTOR: Lazy = Lazy::new(|| TableDescriptor { field_descriptors: vec![ - FieldDescriptor { - name: "sensor_id".to_string(), - number: 1, - typ: ColumnType::Int64, - }, - FieldDescriptor { - name: "timestamp".to_string(), - number: 2, - typ: ColumnType::Timestamp, - }, - FieldDescriptor { - name: "value".to_string(), - number: 3, - typ: ColumnType::Float64, - }, + field("sensor_id", 1, ColumnType::Int64), + field("timestamp", 2, ColumnType::String), + field("value", 3, ColumnType::Double), ], }); pub static STRING_VALUES_DESCRIPTOR: Lazy = Lazy::new(|| TableDescriptor { field_descriptors: vec![ - FieldDescriptor { - name: "sensor_id".to_string(), - number: 1, - typ: ColumnType::Int64, - }, - FieldDescriptor { - name: "timestamp".to_string(), - number: 2, - typ: ColumnType::Timestamp, - }, - FieldDescriptor { - name: "value".to_string(), - number: 3, - typ: ColumnType::Int64, - }, + field("sensor_id", 1, ColumnType::Int64), + field("timestamp", 2, ColumnType::String), + field("value", 3, ColumnType::Int64), ], }); pub static BOOLEAN_VALUES_DESCRIPTOR: Lazy = Lazy::new(|| TableDescriptor { field_descriptors: vec![ - FieldDescriptor { - name: "sensor_id".to_string(), - number: 1, - typ: ColumnType::Int64, - }, - FieldDescriptor { - name: "timestamp".to_string(), - number: 2, - typ: ColumnType::Timestamp, - }, - FieldDescriptor { - name: "value".to_string(), - number: 3, - typ: ColumnType::Bool, - }, + field("sensor_id", 1, ColumnType::Int64), + field("timestamp", 2, ColumnType::String), + field("value", 3, ColumnType::Bool), ], }); pub static LOCATION_VALUES_DESCRIPTOR: Lazy = Lazy::new(|| TableDescriptor { field_descriptors: vec![ - FieldDescriptor { - name: "sensor_id".to_string(), - number: 1, - typ: ColumnType::Int64, - }, - FieldDescriptor { - name: "timestamp".to_string(), - number: 2, - typ: ColumnType::Timestamp, - }, - FieldDescriptor { - name: "latitude".to_string(), - number: 3, - typ: ColumnType::Float64, - }, - FieldDescriptor { - name: "longitude".to_string(), - number: 4, - typ: ColumnType::Float64, - }, + field("sensor_id", 1, ColumnType::Int64), + field("timestamp", 2, ColumnType::String), + field("latitude", 3, ColumnType::Double), + field("longitude", 4, ColumnType::Double), ], }); pub static JSON_VALUES_DESCRIPTOR: Lazy = Lazy::new(|| TableDescriptor { field_descriptors: vec![ - FieldDescriptor { - name: "sensor_id".to_string(), - number: 1, - typ: ColumnType::Int64, - }, - FieldDescriptor { - name: "timestamp".to_string(), - number: 2, - typ: ColumnType::Timestamp, - }, - FieldDescriptor { - name: "value".to_string(), - number: 3, - typ: ColumnType::Json, - }, + field("sensor_id", 1, ColumnType::Int64), + field("timestamp", 2, ColumnType::String), + field("value", 3, ColumnType::String), ], }); pub static BLOB_VALUES_DESCRIPTOR: Lazy = Lazy::new(|| TableDescriptor { field_descriptors: vec![ - FieldDescriptor { - name: "sensor_id".to_string(), - number: 1, - typ: ColumnType::Int64, - }, - FieldDescriptor { - name: "timestamp".to_string(), - number: 2, - typ: ColumnType::Timestamp, - }, - FieldDescriptor { - name: "value".to_string(), - number: 3, - typ: ColumnType::Bytes, - }, + field("sensor_id", 1, ColumnType::Int64), + field("timestamp", 2, ColumnType::String), + field("value", 3, ColumnType::Bytes), ], }); diff --git a/src/storage/bigquery/bigquery_units_utilities.rs b/src/storage/bigquery/bigquery_units_utilities.rs index 4bd38a94..b9a824e1 100644 --- a/src/storage/bigquery/bigquery_units_utilities.rs +++ b/src/storage/bigquery/bigquery_units_utilities.rs @@ -1,11 +1,12 @@ use std::{collections::HashMap, num::NonZeroUsize}; use crate::storage::StorageError; -use anyhow::{Result, anyhow}; +use anyhow::Result; use clru::CLruCache; use gcp_bigquery_client::model::{ query_parameter::QueryParameter, query_parameter_type::QueryParameterType, query_parameter_value::QueryParameterValue, query_request::QueryRequest, + query_response::ResultSet, }; use hybridmap::HybridMap; use once_cell::sync::Lazy; @@ -133,13 +134,14 @@ async fn get_existing_units_ids( }; query_request.query_parameters = Some(vec![query_parameter]); - let mut result = bqs + let result = bqs .client() .read() .await .job() .query(bqs.project_id(), query_request) .await?; + let mut result = ResultSet::new_from_query_response(result); let mut results_map = HybridMap::with_capacity(result.row_count()); diff --git a/src/storage/bigquery/bigquery_utilities.rs b/src/storage/bigquery/bigquery_utilities.rs index eda40a90..7bc64184 100644 --- a/src/storage/bigquery/bigquery_utilities.rs +++ b/src/storage/bigquery/bigquery_utilities.rs @@ -1,6 +1,7 @@ use anyhow::{Result, bail}; use gcp_bigquery_client::{ - google::cloud::bigquery::storage::v1::AppendRowsResponse, storage::TableDescriptor, + google::cloud::bigquery::storage::v1::AppendRowsResponse, + storage::{StorageApi, TableDescriptor}, }; use tokio_stream::StreamExt; use tonic::Streaming; @@ -23,13 +24,14 @@ pub async fn publish_rows( let stream_name = bqs.new_stream_name(table_name.to_string()); let trace_id = create_trace_id(table_name); + let (rows, _) = StorageApi::create_rows(table_descriptor, &rows, 9 * 1024 * 1024); let streaming = bqs .client() .write() .await .storage_mut() - .append_rows(&stream_name, table_descriptor, &rows, trace_id) + .append_rows(&stream_name, rows, trace_id) .await?; check_streaming(streaming).await?; diff --git a/src/storage/bigquery/mod.rs b/src/storage/bigquery/mod.rs index 31ddeb54..2b855c2d 100644 --- a/src/storage/bigquery/mod.rs +++ b/src/storage/bigquery/mod.rs @@ -1,4 +1,7 @@ -use crate::storage::StorageInstance; +use crate::{ + datamodel::{SensAppDateTime, SensorData}, + storage::StorageInstance, +}; use anyhow::{Context, Result, bail}; use async_trait::async_trait; use bigquery_publishers::{ @@ -9,14 +12,14 @@ use bigquery_sensors_utilities::get_sensor_ids_or_create_sensors; use futures::future::try_join_all; use gcp_bigquery_client::{ error::BQError, - model::{dataset::Dataset, query_request::QueryRequest}, + model::{dataset::Dataset, query_request::QueryRequest, query_response::ResultSet}, storage::StreamName, }; use once_cell::sync::Lazy; use regex::Regex; -use std::{future::Future, pin::Pin, sync::Arc}; +use std::{future::Future, pin::Pin, str::FromStr, sync::Arc}; use tokio::sync::RwLock; -use tracing::{debug, error, info}; +use tracing::{debug, info}; use url::Url; mod bigquery_labels_utilities; @@ -116,6 +119,10 @@ impl BigQueryStorage { pub fn new_stream_name(&self, table: String) -> StreamName { StreamName::new_default(self.project_id.clone(), self.dataset_id.clone(), table) } + + fn parse_sensor_type(sensor_type: &str) -> Result { + crate::datamodel::SensorType::from_str(sensor_type).map_err(anyhow::Error::msg) + } } #[async_trait] @@ -158,10 +165,13 @@ impl StorageInstance for BigQueryStorage { .query(&self.project_id, QueryRequest::new(parametrized_init_sql)) .await?; - if let Some(total_rows) = rs.total_rows() { - if total_rows > 0 { - bail!("BigQuery should not return any rows on the schema creation query"); - } + if let Some(total_rows) = rs + .total_rows + .as_deref() + .and_then(|value| value.parse::().ok()) + && total_rows > 0 + { + bail!("BigQuery should not return any rows on the schema creation query"); } Ok(()) @@ -223,8 +233,13 @@ impl StorageInstance for BigQueryStorage { async fn list_series( &self, _metric_filter: Option<&str>, - ) -> Result> { - use crate::datamodel::{Sensor, SensorType, sensapp_vec::SensAppLabels, unit::Unit}; + _limit: Option, + _bookmark: Option<&str>, + ) -> Result { + // TODO: Implement pagination for BigQuery backend + // TODO: Implement metric_filter support for BigQuery backend + // For now, ignore limit, bookmark, and metric_filter parameters and return all results + use crate::datamodel::{Sensor, sensapp_vec::SensAppLabels, unit::Unit}; use gcp_bigquery_client::model::query_request::QueryRequest; use smallvec::smallvec; use std::str::FromStr; @@ -232,7 +247,7 @@ impl StorageInstance for BigQueryStorage { let query = format!( r#" - SELECT s.sensor_id, s.uuid, s.name, s.type, u.name as unit_name, u.description as unit_description + SELECT s.sensor_id, s.uuid AS sensor_uuid, s.name AS sensor_name, s.type AS sensor_type, u.name AS unit_name, u.description AS unit_description FROM `{}.{}.sensors` s LEFT JOIN `{}.{}.units` u ON s.unit = u.id ORDER BY s.uuid ASC @@ -247,22 +262,28 @@ impl StorageInstance for BigQueryStorage { .job() .query(&self.project_id, QueryRequest::new(query)) .await?; + let mut rs = ResultSet::new_from_query_response(rs); let mut sensors = Vec::new(); - for row in rs.rows.unwrap_or_default() { - let sensor_id: i64 = row.columns[0].value.as_ref().unwrap().parse().unwrap(); - let sensor_uuid: Uuid = Uuid::from_str(row.columns[1].value.as_ref().unwrap()).unwrap(); - let sensor_name = row.columns[2].value.as_ref().unwrap().to_string(); - let sensor_type_str = row.columns[3].value.as_ref().unwrap(); - let sensor_type = - SensorType::from_str(sensor_type_str).context("Failed to parse sensor type")?; - let unit = row.columns[4].value.as_ref().map(|name| { - Unit::new( - name.to_string(), - row.columns[5].value.as_ref().map(|d| d.to_string()), - ) - }); + while rs.next_row() { + let sensor_id = rs + .get_i64_by_name("sensor_id")? + .context("BigQuery row missing sensor_id")?; + let sensor_uuid = Uuid::from_str( + &rs.get_string_by_name("sensor_uuid")? + .context("BigQuery row missing sensor_uuid")?, + )?; + let sensor_name = rs + .get_string_by_name("sensor_name")? + .context("BigQuery row missing sensor_name")?; + let sensor_type = Self::parse_sensor_type( + &rs.get_string_by_name("sensor_type")? + .context("BigQuery row missing sensor_type")?, + )?; + let unit_name = rs.get_string_by_name("unit_name")?; + let unit_description = rs.get_string_by_name("unit_description")?; + let unit = unit_name.map(|name| Unit::new(name, unit_description)); // Query labels for this sensor let labels_query = format!( @@ -289,11 +310,16 @@ impl StorageInstance for BigQueryStorage { .job() .query(&self.project_id, QueryRequest::new(labels_query)) .await?; + let mut labels_rs = ResultSet::new_from_query_response(labels_rs); let mut labels: SensAppLabels = smallvec![]; - for label_row in labels_rs.rows.unwrap_or_default() { - let label_name = label_row.columns[0].value.as_ref().unwrap().to_string(); - let label_value = label_row.columns[1].value.as_ref().unwrap().to_string(); + while labels_rs.next_row() { + let label_name = labels_rs + .get_string_by_name("label_name")? + .context("BigQuery row missing label_name")?; + let label_value = labels_rs + .get_string_by_name("label_value")? + .context("BigQuery row missing label_value")?; labels.push((label_name, label_value)); } @@ -302,7 +328,61 @@ impl StorageInstance for BigQueryStorage { sensors.push(sensor); } - Ok(sensors) + Ok(crate::storage::ListSeriesResult { + series: sensors, + bookmark: None, + }) + } + + async fn list_metrics(&self) -> Result> { + use crate::datamodel::{Metric, unit::Unit}; + + let query = format!( + r#" + SELECT s.name AS metric_name, s.type AS sensor_type, u.name AS unit_name, u.description AS unit_description, COUNT(*) AS series_count + FROM `{}.{}.sensors` s + LEFT JOIN `{}.{}.units` u ON s.unit = u.id + GROUP BY s.name, s.type, u.name, u.description + ORDER BY s.name ASC + "#, + self.project_id, self.dataset_id, self.project_id, self.dataset_id + ); + + let rs = self + .client + .read() + .await + .job() + .query(&self.project_id, QueryRequest::new(query)) + .await?; + let mut rs = ResultSet::new_from_query_response(rs); + let mut metrics = Vec::new(); + + while rs.next_row() { + let metric_name = rs + .get_string_by_name("metric_name")? + .context("BigQuery row missing metric_name")?; + let sensor_type = Self::parse_sensor_type( + &rs.get_string_by_name("sensor_type")? + .context("BigQuery row missing sensor_type")?, + )?; + let unit_name = rs.get_string_by_name("unit_name")?; + let unit_description = rs.get_string_by_name("unit_description")?; + let unit = unit_name.map(|name| Unit::new(name, unit_description)); + let series_count = rs + .get_i64_by_name("series_count")? + .context("BigQuery row missing series_count")?; + + metrics.push(Metric::new( + metric_name, + sensor_type, + unit, + series_count, + Vec::new(), + )); + } + + Ok(metrics) } async fn query_sensor_data( @@ -312,17 +392,14 @@ impl StorageInstance for BigQueryStorage { _end_time: Option, _limit: Option, ) -> Result> { - use crate::datamodel::{ - Sensor, SensorData, SensorType, sensapp_vec::SensAppLabels, unit::Unit, - }; + use crate::datamodel::{Sensor, SensorData, sensapp_vec::SensAppLabels, unit::Unit}; use gcp_bigquery_client::model::query_request::QueryRequest; use smallvec::smallvec; - use std::str::FromStr; // Query sensor metadata by UUID let sensor_query = format!( r#" - SELECT s.sensor_id, s.uuid, s.name, s.type, u.name as unit_name, u.description as unit_description + SELECT s.sensor_id, s.uuid AS sensor_uuid, s.name AS sensor_name, s.type AS sensor_type, u.name AS unit_name, u.description AS unit_description FROM `{}.{}.sensors` s LEFT JOIN `{}.{}.units` u ON s.unit = u.id WHERE s.uuid = '{}' @@ -337,28 +414,30 @@ impl StorageInstance for BigQueryStorage { .job() .query(&self.project_id, QueryRequest::new(sensor_query)) .await?; + let mut sensor_rs = ResultSet::new_from_query_response(sensor_rs); + if !sensor_rs.next_row() { + return Ok(None); + } - let sensor_row = match sensor_rs.rows.and_then(|rows| rows.into_iter().next()) { - Some(row) => row, - None => return Ok(None), - }; - - let sensor_id: i64 = sensor_row.columns[0] - .value - .as_ref() - .unwrap() - .parse() - .unwrap(); - let sensor_uuid = - uuid::Uuid::from_str(sensor_row.columns[1].value.as_ref().unwrap()).unwrap(); - let sensor_type = SensorType::from_str(sensor_row.columns[3].value.as_ref().unwrap()) - .context("Failed to parse sensor type")?; - let unit = sensor_row.columns[4].value.as_ref().map(|name| { - Unit::new( - name.to_string(), - sensor_row.columns[5].value.as_ref().map(|d| d.to_string()), - ) - }); + let sensor_id = sensor_rs + .get_i64_by_name("sensor_id")? + .context("BigQuery row missing sensor_id")?; + let sensor_uuid = uuid::Uuid::parse_str( + &sensor_rs + .get_string_by_name("sensor_uuid")? + .context("BigQuery row missing sensor_uuid")?, + )?; + let sensor_name = sensor_rs + .get_string_by_name("sensor_name")? + .context("BigQuery row missing sensor_name")?; + let sensor_type = Self::parse_sensor_type( + &sensor_rs + .get_string_by_name("sensor_type")? + .context("BigQuery row missing sensor_type")?, + )?; + let unit_name = sensor_rs.get_string_by_name("unit_name")?; + let unit_description = sensor_rs.get_string_by_name("unit_description")?; + let unit = unit_name.map(|name| Unit::new(name, unit_description)); // Query labels let labels_query = format!( @@ -385,11 +464,16 @@ impl StorageInstance for BigQueryStorage { .job() .query(&self.project_id, QueryRequest::new(labels_query)) .await?; + let mut labels_rs = ResultSet::new_from_query_response(labels_rs); let mut labels: SensAppLabels = smallvec![]; - for label_row in labels_rs.rows.unwrap_or_default() { - let label_name = label_row.columns[0].value.as_ref().unwrap().to_string(); - let label_value = label_row.columns[1].value.as_ref().unwrap().to_string(); + while labels_rs.next_row() { + let label_name = labels_rs + .get_string_by_name("label_name")? + .context("BigQuery row missing label_name")?; + let label_value = labels_rs + .get_string_by_name("label_value")? + .context("BigQuery row missing label_value")?; labels.push((label_name, label_value)); } diff --git a/src/storage/clickhouse/clickhouse_publishers.rs b/src/storage/clickhouse/clickhouse_publishers.rs index 563ee848..52061ab1 100644 --- a/src/storage/clickhouse/clickhouse_publishers.rs +++ b/src/storage/clickhouse/clickhouse_publishers.rs @@ -1,6 +1,7 @@ use crate::datamodel::{Sample, TypedSamples, batch::SingleSensorBatch}; use crate::storage::clickhouse::clickhouse_utilities::{ - datetime_to_micros, get_sensor_id_or_create_sensor, map_clickhouse_error, + datetime_to_micros, decimal_to_clickhouse_raw, get_sensor_id_or_create_sensor, + map_clickhouse_error, }; use anyhow::Result; use base64::prelude::*; @@ -19,7 +20,7 @@ struct IntegerValueRow { struct NumericValueRow { sensor_id: u64, timestamp_us: i64, - value: rust_decimal::Decimal, + value: i128, } #[derive(Serialize, clickhouse::Row)] @@ -140,11 +141,7 @@ impl<'a> ClickHousePublisher<'a> { // Get or create the label inserter if self.label_inserter.is_none() { - self.label_inserter = Some( - self.client - .inserter("labels") - .map_err(|e| map_clickhouse_error(e, None, None))?, - ); + self.label_inserter = Some(self.client.inserter("labels")); } let inserter = self.label_inserter.as_mut().unwrap(); @@ -157,6 +154,7 @@ impl<'a> ClickHousePublisher<'a> { }; inserter .write(&row) + .await .map_err(|e| map_clickhouse_error(e, None, None))?; } @@ -207,11 +205,7 @@ impl<'a> ClickHousePublisher<'a> { // Get or create the integer inserter if self.integer_inserter.is_none() { - self.integer_inserter = Some( - self.client - .inserter("integer_values") - .map_err(|e| map_clickhouse_error(e, None, None))?, - ); + self.integer_inserter = Some(self.client.inserter("integer_values")); } let inserter = self.integer_inserter.as_mut().unwrap(); @@ -224,6 +218,7 @@ impl<'a> ClickHousePublisher<'a> { }; inserter .write(&row) + .await .map_err(|e| map_clickhouse_error(e, None, None))?; } @@ -242,11 +237,7 @@ impl<'a> ClickHousePublisher<'a> { // Get or create the numeric inserter if self.numeric_inserter.is_none() { - self.numeric_inserter = Some( - self.client - .inserter("numeric_values") - .map_err(|e| map_clickhouse_error(e, None, None))?, - ); + self.numeric_inserter = Some(self.client.inserter("numeric_values")); } let inserter = self.numeric_inserter.as_mut().unwrap(); @@ -255,10 +246,11 @@ impl<'a> ClickHousePublisher<'a> { let row = NumericValueRow { sensor_id, timestamp_us: datetime_to_micros(&sample.datetime), - value: sample.value, + value: decimal_to_clickhouse_raw(&sample.value)?, }; inserter .write(&row) + .await .map_err(|e| map_clickhouse_error(e, None, None))?; } @@ -277,11 +269,7 @@ impl<'a> ClickHousePublisher<'a> { // Get or create the float inserter if self.float_inserter.is_none() { - self.float_inserter = Some( - self.client - .inserter("float_values") - .map_err(|e| map_clickhouse_error(e, None, None))?, - ); + self.float_inserter = Some(self.client.inserter("float_values")); } let inserter = self.float_inserter.as_mut().unwrap(); @@ -294,6 +282,7 @@ impl<'a> ClickHousePublisher<'a> { }; inserter .write(&row) + .await .map_err(|e| map_clickhouse_error(e, None, None))?; } @@ -312,11 +301,7 @@ impl<'a> ClickHousePublisher<'a> { // Get or create the string inserter if self.string_inserter.is_none() { - self.string_inserter = Some( - self.client - .inserter("string_values") - .map_err(|e| map_clickhouse_error(e, None, None))?, - ); + self.string_inserter = Some(self.client.inserter("string_values")); } let inserter = self.string_inserter.as_mut().unwrap(); @@ -329,6 +314,7 @@ impl<'a> ClickHousePublisher<'a> { }; inserter .write(&row) + .await .map_err(|e| map_clickhouse_error(e, None, None))?; } @@ -347,11 +333,7 @@ impl<'a> ClickHousePublisher<'a> { // Get or create the boolean inserter if self.boolean_inserter.is_none() { - self.boolean_inserter = Some( - self.client - .inserter("boolean_values") - .map_err(|e| map_clickhouse_error(e, None, None))?, - ); + self.boolean_inserter = Some(self.client.inserter("boolean_values")); } let inserter = self.boolean_inserter.as_mut().unwrap(); @@ -364,6 +346,7 @@ impl<'a> ClickHousePublisher<'a> { }; inserter .write(&row) + .await .map_err(|e| map_clickhouse_error(e, None, None))?; } @@ -382,11 +365,7 @@ impl<'a> ClickHousePublisher<'a> { // Get or create the location inserter if self.location_inserter.is_none() { - self.location_inserter = Some( - self.client - .inserter("location_values") - .map_err(|e| map_clickhouse_error(e, None, None))?, - ); + self.location_inserter = Some(self.client.inserter("location_values")); } let inserter = self.location_inserter.as_mut().unwrap(); @@ -400,6 +379,7 @@ impl<'a> ClickHousePublisher<'a> { }; inserter .write(&row) + .await .map_err(|e| map_clickhouse_error(e, None, None))?; } @@ -418,11 +398,7 @@ impl<'a> ClickHousePublisher<'a> { // Get or create the json inserter if self.json_inserter.is_none() { - self.json_inserter = Some( - self.client - .inserter("json_values") - .map_err(|e| map_clickhouse_error(e, None, None))?, - ); + self.json_inserter = Some(self.client.inserter("json_values")); } let inserter = self.json_inserter.as_mut().unwrap(); @@ -435,6 +411,7 @@ impl<'a> ClickHousePublisher<'a> { }; inserter .write(&row) + .await .map_err(|e| map_clickhouse_error(e, None, None))?; } @@ -453,11 +430,7 @@ impl<'a> ClickHousePublisher<'a> { // Get or create the blob inserter if self.blob_inserter.is_none() { - self.blob_inserter = Some( - self.client - .inserter("blob_values") - .map_err(|e| map_clickhouse_error(e, None, None))?, - ); + self.blob_inserter = Some(self.client.inserter("blob_values")); } let inserter = self.blob_inserter.as_mut().unwrap(); @@ -470,6 +443,7 @@ impl<'a> ClickHousePublisher<'a> { }; inserter .write(&row) + .await .map_err(|e| map_clickhouse_error(e, None, None))?; } diff --git a/src/storage/clickhouse/clickhouse_utilities.rs b/src/storage/clickhouse/clickhouse_utilities.rs index 9fb11424..5d53bcf8 100644 --- a/src/storage/clickhouse/clickhouse_utilities.rs +++ b/src/storage/clickhouse/clickhouse_utilities.rs @@ -3,11 +3,14 @@ use crate::datamodel::{SensorType, sensapp_datetime::SensAppDateTimeExt, unit::U use crate::storage::StorageError; use anyhow::Result; use clickhouse::Row; +use rust_decimal::{Decimal, RoundingStrategy}; use serde::Serialize; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; use uuid::Uuid; +pub const CLICKHOUSE_NUMERIC_SCALE: u32 = 8; + /// Convert UUID to UInt64 using a deterministic hash function /// We use the standard library's DefaultHasher which is typically xxHash64 pub fn uuid_to_sensor_id(uuid: &Uuid) -> u64 { @@ -24,6 +27,23 @@ pub fn micros_to_datetime(micros: i64) -> SensAppDateTime { SensAppDateTime::from_unix_microseconds_i64(micros) } +pub fn decimal_to_clickhouse_raw(value: &Decimal) -> Result { + let mut normalized = if value.scale() > CLICKHOUSE_NUMERIC_SCALE { + value.round_dp_with_strategy( + CLICKHOUSE_NUMERIC_SCALE, + RoundingStrategy::MidpointNearestEven, + ) + } else { + *value + }; + normalized.rescale(CLICKHOUSE_NUMERIC_SCALE); + Ok(normalized.mantissa()) +} + +pub fn decimal_from_clickhouse_raw(raw: i128) -> Decimal { + Decimal::from_i128_with_scale(raw, CLICKHOUSE_NUMERIC_SCALE) +} + /// Get sensor_id for a given UUID, creating the sensor if it doesn't exist pub async fn get_sensor_id_or_create_sensor( client: &clickhouse::Client, @@ -74,7 +94,8 @@ pub async fn get_sensor_id_or_create_sensor( }; let mut insert = client - .insert("sensors") + .insert::("sensors") + .await .map_err(|e| StorageError::invalid_data_format(&e.to_string(), Some(*uuid), Some(name)))?; insert @@ -127,7 +148,8 @@ async fn get_or_create_unit(client: &clickhouse::Client, unit: &Unit) -> Result< // Unit doesn't exist, create it let mut insert = client - .insert("units") + .insert::("units") + .await .map_err(|e| StorageError::invalid_data_format(&e.to_string(), None, None))?; insert diff --git a/src/storage/clickhouse/matchers.rs b/src/storage/clickhouse/matchers.rs new file mode 100644 index 00000000..eb428d3e --- /dev/null +++ b/src/storage/clickhouse/matchers.rs @@ -0,0 +1,190 @@ +use super::ClickHouseStorage; +use crate::datamodel::sensapp_vec::SensAppLabels; +use crate::datamodel::unit::Unit; +use crate::datamodel::{Sensor, SensorType}; +use crate::storage::{LabelMatcher, MatcherType, StorageError}; +use anyhow::Result; +use smallvec::smallvec; +use std::collections::HashMap; +use std::str::FromStr; +use uuid::Uuid; + +impl ClickHouseStorage { + pub(super) async fn find_sensors_by_matchers( + &self, + name_matchers: &[&LabelMatcher], + label_matchers: &[&LabelMatcher], + numeric_only: bool, + ) -> Result> { + let mut sql = String::from( + r#"SELECT DISTINCT s.sensor_id, s.uuid, s.name, s.type, + COALESCE(u.name, '') AS unit_name, + COALESCE(u.description, '') AS unit_description + FROM sensors s + LEFT JOIN units u ON s.unit = u.id"#, + ); + let mut where_clauses: Vec = Vec::new(); + let mut params: Vec = Vec::new(); + + if numeric_only { + where_clauses.push("s.type IN ('Integer', 'Numeric', 'Float')".to_string()); + } + + for matcher in name_matchers { + let clause = match matcher.matcher_type { + MatcherType::Equal => { + params.push(matcher.value.clone()); + "s.name = ?".to_string() + } + MatcherType::NotEqual => { + params.push(matcher.value.clone()); + "s.name != ?".to_string() + } + MatcherType::RegexMatch => { + params.push(matcher.value.clone()); + "match(s.name, ?)".to_string() + } + MatcherType::RegexNotMatch => { + params.push(matcher.value.clone()); + "NOT match(s.name, ?)".to_string() + } + }; + where_clauses.push(clause); + } + + for matcher in label_matchers { + let subquery = match matcher.matcher_type { + MatcherType::Equal => { + params.push(matcher.name.clone()); + params.push(matcher.value.clone()); + r#"s.sensor_id IN ( + SELECT l.sensor_id FROM labels l + WHERE l.name = ? AND COALESCE(l.description, '') = ? + )"# + .to_string() + } + MatcherType::NotEqual => { + params.push(matcher.name.clone()); + params.push(matcher.value.clone()); + r#"s.sensor_id NOT IN ( + SELECT l.sensor_id FROM labels l + WHERE l.name = ? AND COALESCE(l.description, '') = ? + )"# + .to_string() + } + MatcherType::RegexMatch => { + params.push(matcher.name.clone()); + params.push(matcher.value.clone()); + r#"s.sensor_id IN ( + SELECT l.sensor_id FROM labels l + WHERE l.name = ? AND match(COALESCE(l.description, ''), ?) + )"# + .to_string() + } + MatcherType::RegexNotMatch => { + params.push(matcher.name.clone()); + params.push(matcher.value.clone()); + r#"s.sensor_id NOT IN ( + SELECT l.sensor_id FROM labels l + WHERE l.name = ? AND match(COALESCE(l.description, ''), ?) + )"# + .to_string() + } + }; + where_clauses.push(subquery); + } + + if !where_clauses.is_empty() { + sql.push_str(" WHERE "); + sql.push_str(&where_clauses.join(" AND ")); + } + sql.push_str(" ORDER BY s.sensor_id ASC"); + + #[derive(clickhouse::Row, serde::Deserialize)] + struct SensorRow { + sensor_id: u64, + #[serde(with = "clickhouse::serde::uuid")] + uuid: Uuid, + name: String, + r#type: String, + unit_name: String, + unit_description: String, + } + + let mut query = self.client.query(&sql); + for param in ¶ms { + query = query.bind(param.as_str()); + } + + let mut cursor = query + .fetch::() + .map_err(|e| StorageError::invalid_data_format(&e.to_string(), None, None))?; + + let mut sensor_rows = Vec::new(); + while let Some(row) = cursor.next().await? { + sensor_rows.push(row); + } + + if sensor_rows.is_empty() { + return Ok(Vec::new()); + } + + let sensor_ids: Vec = sensor_rows.iter().map(|row| row.sensor_id).collect(); + let mut labels_map: HashMap = HashMap::new(); + + #[derive(clickhouse::Row, serde::Deserialize)] + struct LabelRow { + sensor_id: u64, + name: String, + description: String, + } + + for sensor_id in sensor_ids { + let mut labels_cursor = self + .client + .query( + "SELECT sensor_id, name, COALESCE(description, '') AS description FROM labels WHERE sensor_id = ? ORDER BY name ASC", + ) + .bind(sensor_id) + .fetch::() + .map_err(|e| StorageError::invalid_data_format(&e.to_string(), None, None))?; + + while let Some(row) = labels_cursor.next().await? { + labels_map + .entry(row.sensor_id) + .or_insert_with(|| smallvec![]) + .push((row.name, row.description)); + } + } + + let mut results = Vec::with_capacity(sensor_rows.len()); + for row in sensor_rows { + let sensor_type = SensorType::from_str(&row.r#type).map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Failed to parse sensor type '{}': {}", row.r#type, e), + Some(row.uuid), + Some(&row.name), + )) + })?; + + let unit = if row.unit_name.is_empty() { + None + } else { + Some(Unit::new( + row.unit_name, + if row.unit_description.is_empty() { + None + } else { + Some(row.unit_description) + }, + )) + }; + + let labels = labels_map.remove(&row.sensor_id).unwrap_or_default(); + let sensor = Sensor::new(row.uuid, row.name, sensor_type, unit, Some(labels)); + results.push((row.sensor_id, sensor)); + } + + Ok(results) + } +} diff --git a/src/storage/clickhouse/migrations/20240223133248_init.sql b/src/storage/clickhouse/migrations/20240223133248_init.sql index dfd6715c..5c428a91 100644 --- a/src/storage/clickhouse/migrations/20240223133248_init.sql +++ b/src/storage/clickhouse/migrations/20240223133248_init.sql @@ -44,12 +44,14 @@ SETTINGS index_granularity_bytes = 10485760; CREATE TABLE IF NOT EXISTS numeric_values ( sensor_id UInt64, timestamp_us Int64 CODEC(DoubleDelta, LZ4), - value Decimal128(38) CODEC(ZSTD(1)) + value Decimal(38, 8) CODEC(ZSTD(1)) ) ENGINE = MergeTree() PARTITION BY toYYYYMM(toDateTime64(timestamp_us / 1000000, 6)) ORDER BY (sensor_id, timestamp_us) SETTINGS index_granularity_bytes = 10485760; +ALTER TABLE numeric_values MODIFY COLUMN value Decimal(38, 8); + -- Create the 'float_values' table CREATE TABLE IF NOT EXISTS float_values ( sensor_id UInt64, @@ -95,7 +97,7 @@ SETTINGS index_granularity_bytes = 10485760; CREATE TABLE IF NOT EXISTS json_values ( sensor_id UInt64, timestamp_us Int64 CODEC(DoubleDelta, LZ4), - value JSON -- Native JSON type (ClickHouse 24.8+) for better performance and compression + value String CODEC(ZSTD(3)) ) ENGINE = MergeTree() PARTITION BY toYYYYMM(toDateTime64(timestamp_us / 1000000, 6)) ORDER BY (sensor_id, timestamp_us) diff --git a/src/storage/clickhouse/mod.rs b/src/storage/clickhouse/mod.rs index 65300cfb..f29e835a 100644 --- a/src/storage/clickhouse/mod.rs +++ b/src/storage/clickhouse/mod.rs @@ -1,4 +1,7 @@ -use super::{DEFAULT_QUERY_LIMIT, StorageError, StorageInstance}; +use super::{ + Aggregation, DEFAULT_QUERY_LIMIT, SensorAvailabilitySummary, SensorDataQueryOptions, + StorageError, StorageInstance, +}; use crate::datamodel::sensapp_vec::SensAppLabels; use crate::datamodel::{ Metric, Sample, SensAppDateTime, Sensor, SensorData, SensorType, TypedSamples, batch::Batch, @@ -15,10 +18,13 @@ use uuid::Uuid; pub mod clickhouse_publishers; pub mod clickhouse_utilities; +mod matchers; +use crate::storage::{DEFAULT_LIST_SERIES_LIMIT, MAX_LIST_SERIES_LIMIT}; use clickhouse_publishers::ClickHousePublisher; use clickhouse_utilities::{ - datetime_to_micros, map_clickhouse_error, micros_to_datetime, uuid_to_sensor_id, + datetime_to_micros, decimal_from_clickhouse_raw, map_clickhouse_error, micros_to_datetime, + uuid_to_sensor_id, }; pub struct ClickHouseStorage { @@ -31,6 +37,26 @@ pub struct ClickHouseStorage { password: Option, } +#[derive(clickhouse::Row, serde::Deserialize)] +struct LatestTimestampRow { + timestamp_us: i64, +} + +#[derive(clickhouse::Row, serde::Deserialize)] +struct AvailabilitySummaryWithStepRow { + sample_count: u64, + first_sample_at: Option, + last_sample_at: Option, + covered_buckets: u64, +} + +#[derive(clickhouse::Row, serde::Deserialize)] +struct AvailabilitySummaryRow { + sample_count: u64, + first_sample_at: Option, + last_sample_at: Option, +} + impl std::fmt::Debug for ClickHouseStorage { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ClickHouseStorage") @@ -40,6 +66,13 @@ impl std::fmt::Debug for ClickHouseStorage { } impl ClickHouseStorage { + fn aggregated_bucket_expr(query: &AggregatedSamplesQuery) -> String { + format!( + "{} + intDiv(timestamp_us - {}, {}) * {}", + query.origin_us, query.origin_us, query.step_us, query.step_us + ) + } + pub async fn connect(connection_string: &str) -> Result { // Parse ClickHouse connection string // Format: clickhouse://user:password@host:port/database @@ -164,6 +197,235 @@ impl ClickHouseStorage { Ok(()) } + + async fn get_sensor_metadata(&self, sensor_uuid: &str) -> Result> { + let uuid = Uuid::from_str(sensor_uuid).map_err(|e| { + StorageError::invalid_data_format( + &format!("Invalid UUID '{}': {}", sensor_uuid, e), + None, + None, + ) + })?; + + let sensor_id = uuid_to_sensor_id(&uuid); + + let sensor_query = r#" + SELECT + s.uuid, + s.name, + s.type, + u.name AS unit_name, + u.description AS unit_description + FROM sensors s + LEFT JOIN units u ON s.unit = u.id + WHERE s.sensor_id = ? + LIMIT 1 + "#; + + #[derive(clickhouse::Row, serde::Deserialize)] + struct SensorMetadataRow { + #[serde(with = "clickhouse::serde::uuid")] + uuid: Uuid, + name: String, + r#type: String, + unit_name: String, + unit_description: Option, + } + + let mut sensor_cursor = self + .client + .query(sensor_query) + .bind(sensor_id) + .fetch::() + .map_err(|e| map_clickhouse_error(e, Some(uuid), None))?; + + let Some(row) = sensor_cursor.next().await? else { + return Ok(None); + }; + + let sensor_type = SensorType::from_str(&row.r#type).map_err(|e| { + StorageError::invalid_data_format( + &format!("Failed to parse sensor type '{}': {}", row.r#type, e), + Some(row.uuid), + Some(&row.name), + ) + })?; + + let unit = if !row.unit_name.is_empty() { + Some(Unit { + name: row.unit_name, + description: row.unit_description, + }) + } else { + None + }; + + let labels_query = "SELECT name, COALESCE(description, '') FROM labels WHERE sensor_id = ?"; + let mut labels_cursor = self + .client + .query(labels_query) + .bind(sensor_id) + .fetch::<(String, String)>() + .map_err(|e| map_clickhouse_error(e, Some(row.uuid), Some(&row.name)))?; + + let mut labels = Vec::new(); + while let Some((label_name, label_description)) = labels_cursor.next().await? { + labels.push((label_name, label_description)); + } + + Ok(Some(( + sensor_id, + Sensor { + uuid: row.uuid, + name: row.name, + sensor_type, + unit, + labels: SensAppLabels::from(labels), + }, + ))) + } + + fn sensor_table_name(sensor_type: SensorType) -> &'static str { + match sensor_type { + SensorType::Integer => "integer_values", + SensorType::Numeric => "numeric_values", + SensorType::Float => "float_values", + SensorType::String => "string_values", + SensorType::Boolean => "boolean_values", + SensorType::Location => "location_values", + SensorType::Json => "json_values", + SensorType::Blob => "blob_values", + } + } + + async fn query_latest_timestamp_us( + &self, + table_name: &str, + sensor_id: u64, + start_time_us: Option, + end_time_us: Option, + ) -> Result> { + let query = format!( + "SELECT timestamp_us FROM {table_name} WHERE sensor_id = ?{} ORDER BY timestamp_us DESC LIMIT 1", + clickhouse_time_where(start_time_us, end_time_us) + ); + + let mut cursor = self.client.query(&query).bind(sensor_id); + if let Some(start_time_us) = start_time_us { + cursor = cursor.bind(start_time_us); + } + if let Some(end_time_us) = end_time_us { + cursor = cursor.bind(end_time_us); + } + + let mut rows = cursor + .fetch::() + .map_err(|e| map_clickhouse_error(e, None, None))?; + + Ok(rows.next().await?.map(|row| row.timestamp_us)) + } + + async fn query_availability_summary_native( + &self, + table_name: &str, + sensor_id: u64, + sensor: Sensor, + start_time_us: i64, + end_time_us: i64, + step_ms: Option, + ) -> Result { + if let Some(step_ms) = step_ms { + let step_us = step_ms.checked_mul(1000).context("step is too large")?; + let query = format!( + r#" + SELECT + count() AS sample_count, + minOrNull(timestamp_us) AS first_sample_at, + maxOrNull(timestamp_us) AS last_sample_at, + countDistinct(intDiv(timestamp_us - ?, ?)) AS covered_buckets + FROM {table_name} + WHERE sensor_id = ? + AND timestamp_us >= ? + AND timestamp_us <= ? + "# + ); + + let mut rows = self + .client + .query(&query) + .bind(start_time_us) + .bind(step_us) + .bind(sensor_id) + .bind(start_time_us) + .bind(end_time_us) + .fetch::() + .map_err(|e| map_clickhouse_error(e, Some(sensor.uuid), Some(&sensor.name)))?; + + let row = rows + .next() + .await? + .unwrap_or(AvailabilitySummaryWithStepRow { + sample_count: 0, + first_sample_at: None, + last_sample_at: None, + covered_buckets: 0, + }); + + return Ok(SensorAvailabilitySummary { + sensor, + sample_count: row.sample_count as usize, + first_sample_at: row.first_sample_at.map(micros_to_datetime), + last_sample_at: row.last_sample_at.map(micros_to_datetime), + covered_buckets: Some(row.covered_buckets as usize), + }); + } + + let query = format!( + r#" + SELECT + count() AS sample_count, + minOrNull(timestamp_us) AS first_sample_at, + maxOrNull(timestamp_us) AS last_sample_at + FROM {table_name} + WHERE sensor_id = ? + AND timestamp_us >= ? + AND timestamp_us <= ? + "# + ); + + let mut rows = self + .client + .query(&query) + .bind(sensor_id) + .bind(start_time_us) + .bind(end_time_us) + .fetch::() + .map_err(|e| map_clickhouse_error(e, Some(sensor.uuid), Some(&sensor.name)))?; + + let row = rows.next().await?.unwrap_or(AvailabilitySummaryRow { + sample_count: 0, + first_sample_at: None, + last_sample_at: None, + }); + + Ok(SensorAvailabilitySummary { + sensor, + sample_count: row.sample_count as usize, + first_sample_at: row.first_sample_at.map(micros_to_datetime), + last_sample_at: row.last_sample_at.map(micros_to_datetime), + covered_buckets: None, + }) + } +} + +struct AggregatedSamplesQuery { + sensor_id: u64, + start_time_us: Option, + end_time_us: Option, + step_us: i64, + origin_us: i64, + aggregation: Aggregation, + limit: Option, } #[async_trait] @@ -219,31 +481,84 @@ impl StorageInstance for ClickHouseStorage { async fn list_series( &self, metric_filter: Option<&str>, - ) -> Result> { - let (query, use_filter) = match metric_filter { - Some(_) => ( - r#" + limit: Option, + bookmark: Option<&str>, + ) -> Result { + let bookmark_id = if let Some(bookmark_str) = bookmark { + Some(bookmark_str.parse::().map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Invalid bookmark format: {}", e), + None, + None, + )) + })?) + } else { + None + }; + + let effective_limit = limit + .unwrap_or(DEFAULT_LIST_SERIES_LIMIT) + .min(MAX_LIST_SERIES_LIMIT); + let fetch_limit = effective_limit.saturating_add(1); + + let (query, use_filter, use_bookmark) = + match (metric_filter.is_some(), bookmark_id.is_some()) { + (true, true) => ( + r#" SELECT s.sensor_id, s.uuid, s.name, s.type, COALESCE(u.name, '') as unit_name, COALESCE(u.description, '') as unit_description FROM sensors s LEFT JOIN units u ON s.unit = u.id - WHERE s.name = ? ORDER BY s.uuid ASC + WHERE s.name = ? AND s.sensor_id > ? + ORDER BY s.sensor_id ASC + LIMIT ? "#, - true, - ), - None => ( - r#" + true, + true, + ), + (true, false) => ( + r#" SELECT s.sensor_id, s.uuid, s.name, s.type, COALESCE(u.name, '') as unit_name, COALESCE(u.description, '') as unit_description FROM sensors s LEFT JOIN units u ON s.unit = u.id - ORDER BY s.uuid ASC + WHERE s.name = ? + ORDER BY s.sensor_id ASC + LIMIT ? "#, - false, - ), - }; + true, + false, + ), + (false, true) => ( + r#" + SELECT s.sensor_id, s.uuid, s.name, s.type, + COALESCE(u.name, '') as unit_name, + COALESCE(u.description, '') as unit_description + FROM sensors s + LEFT JOIN units u ON s.unit = u.id + WHERE s.sensor_id > ? + ORDER BY s.sensor_id ASC + LIMIT ? + "#, + false, + true, + ), + (false, false) => ( + r#" + SELECT s.sensor_id, s.uuid, s.name, s.type, + COALESCE(u.name, '') as unit_name, + COALESCE(u.description, '') as unit_description + FROM sensors s + LEFT JOIN units u ON s.unit = u.id + ORDER BY s.sensor_id ASC + LIMIT ? + "#, + false, + false, + ), + }; #[derive(clickhouse::Row, serde::Deserialize)] struct SensorRow { @@ -256,24 +571,31 @@ impl StorageInstance for ClickHouseStorage { unit_description: String, } - let mut cursor = if use_filter { - self.client - .query(query) - .bind(metric_filter.unwrap()) - .fetch::() - .map_err(|e| map_clickhouse_error(e, None, None))? - } else { - self.client - .query(query) - .fetch::() - .map_err(|e| map_clickhouse_error(e, None, None))? - }; + let mut query_builder = self.client.query(query); + if use_filter { + query_builder = query_builder.bind(metric_filter.unwrap()); + } + if use_bookmark { + query_builder = query_builder.bind(bookmark_id.unwrap()); + } + let mut cursor = query_builder + .bind(fetch_limit as u64) + .fetch::() + .map_err(|e| map_clickhouse_error(e, None, None))?; // Process sensors directly from cursor let mut sensors = Vec::new(); + let mut last_sensor_id = None; + let mut has_more = false; while let Some(row) = cursor.next().await? { + if sensors.len() == effective_limit { + has_more = true; + break; + } + let sensor_id = row.sensor_id; + last_sensor_id = Some(sensor_id); let uuid = row.uuid; let name = row.name; let sensor_type_str = row.r#type; @@ -326,7 +648,14 @@ impl StorageInstance for ClickHouseStorage { sensors.push(sensor); } - Ok(sensors) + Ok(crate::storage::ListSeriesResult { + series: sensors, + bookmark: if has_more { + last_sensor_id.map(|sensor_id| sensor_id.to_string()) + } else { + None + }, + }) } async fn list_metrics(&self) -> Result> { @@ -486,16 +815,194 @@ impl StorageInstance for ClickHouseStorage { Ok(Some(sensor_data)) } + async fn query_sensor_data_advanced( + &self, + sensor_uuid: &str, + options: &SensorDataQueryOptions, + ) -> Result> { + options.validate()?; + + if let (Some(step_ms), Some(aggregation)) = (options.step_ms, options.aggregation) { + if let Some((sensor_id, mut sensor)) = self.get_sensor_metadata(sensor_uuid).await? { + let start_time_us = options.start_time.as_ref().map(datetime_to_micros); + let end_time_us = options.end_time.as_ref().map(datetime_to_micros); + let aggregation_query = AggregatedSamplesQuery { + sensor_id, + start_time_us, + end_time_us, + step_us: step_ms * 1000, + origin_us: start_time_us.unwrap_or(0), + aggregation, + limit: options.limit, + }; + + let samples = match sensor.sensor_type { + SensorType::Integer => { + self.query_integer_samples_aggregated(&aggregation_query) + .await? + } + SensorType::Numeric => { + self.query_numeric_samples_aggregated(&aggregation_query) + .await? + } + SensorType::Float => { + self.query_float_samples_aggregated(&aggregation_query) + .await? + } + _ => { + let raw = self + .query_sensor_data( + sensor_uuid, + options.start_time, + options.end_time, + options.limit, + ) + .await?; + return raw + .map(|sensor_data| { + crate::storage::common::apply_query_options(sensor_data, options) + }) + .transpose(); + } + }; + + sensor.sensor_type = match &samples { + TypedSamples::Integer(_) => SensorType::Integer, + TypedSamples::Numeric(_) => SensorType::Numeric, + TypedSamples::Float(_) => SensorType::Float, + _ => sensor.sensor_type, + }; + if aggregation.output_is_count() { + sensor.unit = None; + } + + let post_query_options = SensorDataQueryOptions { + start_time: options.start_time, + end_time: options.end_time, + limit: options.limit, + step_ms: None, + aggregation: None, + simplify: options.simplify, + }; + + return crate::storage::common::apply_query_options( + SensorData::new(sensor, samples), + &post_query_options, + ) + .map(Some); + } + + return Ok(None); + } + + let raw = self + .query_sensor_data( + sensor_uuid, + options.start_time, + options.end_time, + options.limit, + ) + .await?; + + raw.map(|sensor_data| crate::storage::common::apply_query_options(sensor_data, options)) + .transpose() + } + + async fn query_sensor_data_latest( + &self, + sensor_uuid: &str, + start_time: Option, + end_time: Option, + ) -> Result> { + let Some((sensor_id, sensor)) = self.get_sensor_metadata(sensor_uuid).await? else { + return Ok(None); + }; + + let start_time_us = start_time.as_ref().map(datetime_to_micros); + let end_time_us = end_time.as_ref().map(datetime_to_micros); + let table_name = Self::sensor_table_name(sensor.sensor_type); + + let Some(latest_timestamp_us) = self + .query_latest_timestamp_us(table_name, sensor_id, start_time_us, end_time_us) + .await? + else { + return Ok(None); + }; + + let samples = self + .query_samples_by_type( + sensor_id, + &sensor.sensor_type, + Some(micros_to_datetime(latest_timestamp_us)), + Some(micros_to_datetime(latest_timestamp_us)), + 1, + ) + .await?; + + Ok(Some(SensorData::new(sensor, samples))) + } + + async fn query_sensor_data_availability( + &self, + sensor_uuid: &str, + start_time: SensAppDateTime, + end_time: SensAppDateTime, + step_ms: Option, + ) -> Result> { + let Some((sensor_id, sensor)) = self.get_sensor_metadata(sensor_uuid).await? else { + return Ok(None); + }; + + let summary = self + .query_availability_summary_native( + Self::sensor_table_name(sensor.sensor_type), + sensor_id, + sensor, + datetime_to_micros(&start_time), + datetime_to_micros(&end_time), + step_ms, + ) + .await?; + + Ok(Some(summary)) + } + async fn query_sensors_by_labels( &self, - _matchers: &[super::LabelMatcher], - _start_time: Option, - _end_time: Option, - _limit: Option, - _numeric_only: bool, + matchers: &[super::LabelMatcher], + start_time: Option, + end_time: Option, + limit: Option, + numeric_only: bool, ) -> Result> { - // TODO: Implement label-based query for ClickHouse - anyhow::bail!("query_sensors_by_labels not yet implemented for ClickHouse") + if matchers.is_empty() { + return Ok(Vec::new()); + } + + let (name_matchers, label_matchers): (Vec<_>, Vec<_>) = matchers + .iter() + .partition(|matcher| matcher.is_name_matcher()); + + let sensors = self + .find_sensors_by_matchers(&name_matchers, &label_matchers, numeric_only) + .await?; + + if sensors.is_empty() { + return Ok(Vec::new()); + } + + let limit = limit.unwrap_or(DEFAULT_QUERY_LIMIT); + let mut results = Vec::with_capacity(sensors.len()); + + for (sensor_id, sensor) in sensors { + let samples = self + .query_samples_by_type(sensor_id, &sensor.sensor_type, start_time, end_time, limit) + .await?; + + results.push(SensorData::new(sensor, samples)); + } + + Ok(results) } /// Health check for ClickHouse storage @@ -516,6 +1023,248 @@ impl StorageInstance for ClickHouseStorage { } impl ClickHouseStorage { + async fn query_integer_samples_aggregated( + &self, + query: &AggregatedSamplesQuery, + ) -> Result { + let limit = query.limit.unwrap_or(DEFAULT_QUERY_LIMIT); + let bucket_expr = Self::aggregated_bucket_expr(query); + let where_clause = clickhouse_time_where(query.start_time_us, query.end_time_us); + + match query.aggregation { + Aggregation::Avg => { + #[derive(clickhouse::Row, serde::Deserialize)] + struct Row { + timestamp_us: i64, + value: f64, + } + + let sql = format!( + "SELECT {bucket_expr} AS timestamp_us, avg(value) AS value FROM integer_values WHERE sensor_id = ?{where_clause} GROUP BY timestamp_us ORDER BY timestamp_us ASC LIMIT {limit}" + ); + let mut cursor = self.client.query(&sql).bind(query.sensor_id); + if let Some(start_time_us) = query.start_time_us { + cursor = cursor.bind(start_time_us); + } + if let Some(end_time_us) = query.end_time_us { + cursor = cursor.bind(end_time_us); + } + let mut rows_cursor = cursor + .fetch::() + .map_err(|e| map_clickhouse_error(e, None, None))?; + let mut samples = smallvec::smallvec![]; + while let Some(row) = rows_cursor.next().await? { + samples.push(Sample { + datetime: micros_to_datetime(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Float(samples)) + } + Aggregation::Count => { + #[derive(clickhouse::Row, serde::Deserialize)] + struct Row { + timestamp_us: i64, + value: i64, + } + + let sql = format!( + "SELECT {bucket_expr} AS timestamp_us, toInt64(count()) AS value FROM integer_values WHERE sensor_id = ?{where_clause} GROUP BY timestamp_us ORDER BY timestamp_us ASC LIMIT {limit}" + ); + let mut cursor = self.client.query(&sql).bind(query.sensor_id); + if let Some(start_time_us) = query.start_time_us { + cursor = cursor.bind(start_time_us); + } + if let Some(end_time_us) = query.end_time_us { + cursor = cursor.bind(end_time_us); + } + let mut rows_cursor = cursor + .fetch::() + .map_err(|e| map_clickhouse_error(e, None, None))?; + let mut samples = smallvec::smallvec![]; + while let Some(row) = rows_cursor.next().await? { + samples.push(Sample { + datetime: micros_to_datetime(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + _ => { + #[derive(clickhouse::Row, serde::Deserialize)] + struct Row { + timestamp_us: i64, + value: i64, + } + + let expression = clickhouse_integer_expression(query.aggregation); + let sql = format!( + "SELECT {bucket_expr} AS timestamp_us, {expression} AS value FROM integer_values WHERE sensor_id = ?{where_clause} GROUP BY timestamp_us ORDER BY timestamp_us ASC LIMIT {limit}" + ); + let mut cursor = self.client.query(&sql).bind(query.sensor_id); + if let Some(start_time_us) = query.start_time_us { + cursor = cursor.bind(start_time_us); + } + if let Some(end_time_us) = query.end_time_us { + cursor = cursor.bind(end_time_us); + } + let mut rows_cursor = cursor + .fetch::() + .map_err(|e| map_clickhouse_error(e, None, None))?; + let mut samples = smallvec::smallvec![]; + while let Some(row) = rows_cursor.next().await? { + samples.push(Sample { + datetime: micros_to_datetime(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + } + } + + async fn query_float_samples_aggregated( + &self, + query: &AggregatedSamplesQuery, + ) -> Result { + let limit = query.limit.unwrap_or(DEFAULT_QUERY_LIMIT); + let bucket_expr = Self::aggregated_bucket_expr(query); + let where_clause = clickhouse_time_where(query.start_time_us, query.end_time_us); + + match query.aggregation { + Aggregation::Count => { + #[derive(clickhouse::Row, serde::Deserialize)] + struct Row { + timestamp_us: i64, + value: i64, + } + + let sql = format!( + "SELECT {bucket_expr} AS timestamp_us, toInt64(count()) AS value FROM float_values WHERE sensor_id = ?{where_clause} GROUP BY timestamp_us ORDER BY timestamp_us ASC LIMIT {limit}" + ); + let mut cursor = self.client.query(&sql).bind(query.sensor_id); + if let Some(start_time_us) = query.start_time_us { + cursor = cursor.bind(start_time_us); + } + if let Some(end_time_us) = query.end_time_us { + cursor = cursor.bind(end_time_us); + } + let mut rows_cursor = cursor + .fetch::() + .map_err(|e| map_clickhouse_error(e, None, None))?; + let mut samples = smallvec::smallvec![]; + while let Some(row) = rows_cursor.next().await? { + samples.push(Sample { + datetime: micros_to_datetime(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + _ => { + #[derive(clickhouse::Row, serde::Deserialize)] + struct Row { + timestamp_us: i64, + value: f64, + } + + let expression = clickhouse_float_expression(query.aggregation); + let sql = format!( + "SELECT {bucket_expr} AS timestamp_us, {expression} AS value FROM float_values WHERE sensor_id = ?{where_clause} GROUP BY timestamp_us ORDER BY timestamp_us ASC LIMIT {limit}" + ); + let mut cursor = self.client.query(&sql).bind(query.sensor_id); + if let Some(start_time_us) = query.start_time_us { + cursor = cursor.bind(start_time_us); + } + if let Some(end_time_us) = query.end_time_us { + cursor = cursor.bind(end_time_us); + } + let mut rows_cursor = cursor + .fetch::() + .map_err(|e| map_clickhouse_error(e, None, None))?; + let mut samples = smallvec::smallvec![]; + while let Some(row) = rows_cursor.next().await? { + samples.push(Sample { + datetime: micros_to_datetime(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Float(samples)) + } + } + } + + async fn query_numeric_samples_aggregated( + &self, + query: &AggregatedSamplesQuery, + ) -> Result { + let limit = query.limit.unwrap_or(DEFAULT_QUERY_LIMIT); + let bucket_expr = Self::aggregated_bucket_expr(query); + let where_clause = clickhouse_time_where(query.start_time_us, query.end_time_us); + + match query.aggregation { + Aggregation::Count => { + #[derive(clickhouse::Row, serde::Deserialize)] + struct Row { + timestamp_us: i64, + value: i64, + } + + let sql = format!( + "SELECT {bucket_expr} AS timestamp_us, toInt64(count()) AS value FROM numeric_values WHERE sensor_id = ?{where_clause} GROUP BY timestamp_us ORDER BY timestamp_us ASC LIMIT {limit}" + ); + let mut cursor = self.client.query(&sql).bind(query.sensor_id); + if let Some(start_time_us) = query.start_time_us { + cursor = cursor.bind(start_time_us); + } + if let Some(end_time_us) = query.end_time_us { + cursor = cursor.bind(end_time_us); + } + let mut rows_cursor = cursor + .fetch::() + .map_err(|e| map_clickhouse_error(e, None, None))?; + let mut samples = smallvec::smallvec![]; + while let Some(row) = rows_cursor.next().await? { + samples.push(Sample { + datetime: micros_to_datetime(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + _ => { + #[derive(clickhouse::Row, serde::Deserialize)] + struct Row { + timestamp_us: i64, + value: i128, + } + + let expression = clickhouse_numeric_expression(query.aggregation); + let sql = format!( + "SELECT {bucket_expr} AS timestamp_us, {expression} AS value FROM numeric_values WHERE sensor_id = ?{where_clause} GROUP BY timestamp_us ORDER BY timestamp_us ASC LIMIT {limit}" + ); + let mut cursor = self.client.query(&sql).bind(query.sensor_id); + if let Some(start_time_us) = query.start_time_us { + cursor = cursor.bind(start_time_us); + } + if let Some(end_time_us) = query.end_time_us { + cursor = cursor.bind(end_time_us); + } + let mut rows_cursor = cursor + .fetch::() + .map_err(|e| map_clickhouse_error(e, None, None))?; + let mut samples = smallvec::smallvec![]; + while let Some(row) = rows_cursor.next().await? { + samples.push(Sample { + datetime: micros_to_datetime(row.timestamp_us), + value: decimal_from_clickhouse_raw(row.value), + }); + } + Ok(TypedSamples::Numeric(samples)) + } + } + } + /// Query samples by type with time range filtering async fn query_samples_by_type( &self, @@ -598,21 +1347,14 @@ impl ClickHouseStorage { } let mut result_cursor = cursor - .fetch::<(i64, String)>() + .fetch::<(i64, i128)>() .map_err(|e| map_clickhouse_error(e, None, None))?; if let TypedSamples::Numeric(ref mut samples) = typed_samples { - while let Some((timestamp_us, value_str)) = result_cursor.next().await? { - let value = rust_decimal::Decimal::from_str(&value_str).map_err(|e| { - StorageError::invalid_data_format( - &format!("Failed to parse decimal value: {}", e), - None, - None, - ) - })?; + while let Some((timestamp_us, value_raw)) = result_cursor.next().await? { samples.push(Sample { datetime: micros_to_datetime(timestamp_us), - value, + value: decimal_from_clickhouse_raw(value_raw), }); } } @@ -792,3 +1534,49 @@ impl ClickHouseStorage { Ok(typed_samples) } } + +fn clickhouse_time_where(start_time_us: Option, end_time_us: Option) -> String { + let mut conditions = String::new(); + if start_time_us.is_some() { + conditions.push_str(" AND timestamp_us >= ?"); + } + if end_time_us.is_some() { + conditions.push_str(" AND timestamp_us <= ?"); + } + conditions +} + +fn clickhouse_integer_expression(aggregation: Aggregation) -> &'static str { + match aggregation { + Aggregation::Min => "min(value)", + Aggregation::Max => "max(value)", + Aggregation::Sum => "sum(value)", + Aggregation::First => "argMin(value, timestamp_us)", + Aggregation::Last => "argMax(value, timestamp_us)", + Aggregation::Avg | Aggregation::Count => unreachable!("handled separately"), + } +} + +fn clickhouse_float_expression(aggregation: Aggregation) -> &'static str { + match aggregation { + Aggregation::Avg => "avg(value)", + Aggregation::Min => "min(value)", + Aggregation::Max => "max(value)", + Aggregation::Sum => "sum(value)", + Aggregation::First => "argMin(value, timestamp_us)", + Aggregation::Last => "argMax(value, timestamp_us)", + Aggregation::Count => unreachable!("handled separately"), + } +} + +fn clickhouse_numeric_expression(aggregation: Aggregation) -> &'static str { + match aggregation { + Aggregation::Avg => "toDecimal128(avg(value), 8)", + Aggregation::Min => "min(value)", + Aggregation::Max => "max(value)", + Aggregation::Sum => "sum(value)", + Aggregation::First => "argMin(value, timestamp_us)", + Aggregation::Last => "argMax(value, timestamp_us)", + Aggregation::Count => unreachable!("handled separately"), + } +} diff --git a/src/storage/common.rs b/src/storage/common.rs index 57691504..8decedb6 100644 --- a/src/storage/common.rs +++ b/src/storage/common.rs @@ -1,5 +1,14 @@ -use crate::datamodel::SensAppDateTime; +use crate::datamodel::sensapp_datetime::SensAppDateTimeExt; +use crate::datamodel::{Sample, SensAppDateTime, SensorData, SensorType, TypedSamples}; +use crate::storage::{ + Aggregation, SensorAvailabilitySummary, SensorDataQueryOptions, SimplifyOptions, +}; +use anyhow::{Result, anyhow}; use hifitime::Unit; +use rust_decimal::{Decimal, prelude::ToPrimitive}; +use simplify_polyline::{Point, simplify}; +use smallvec::smallvec; +use std::collections::BTreeSet; /// Convert SensAppDateTime to Unix microseconds for database storage #[allow(dead_code)] // Used by SQLite backend when enabled @@ -9,6 +18,519 @@ pub fn datetime_to_micros(datetime: &SensAppDateTime) -> i64 { datetime.to_unix(Unit::Microsecond).floor() as i64 } +pub fn apply_query_options( + mut sensor_data: SensorData, + options: &SensorDataQueryOptions, +) -> Result { + if let (Some(step_ms), Some(aggregation)) = (options.step_ms, options.aggregation) { + sensor_data = aggregate_sensor_data(sensor_data, options.start_time, step_ms, aggregation)?; + } + + if let Some(simplify_options) = options.simplify { + sensor_data = simplify_sensor_data(sensor_data, simplify_options)?; + } + + Ok(sensor_data) +} + +pub fn keep_only_last_sample(mut sensor_data: SensorData) -> Option { + sensor_data.samples = match sensor_data.samples { + TypedSamples::Integer(mut samples) => TypedSamples::Integer(smallvec![samples.pop()?]), + TypedSamples::Numeric(mut samples) => TypedSamples::Numeric(smallvec![samples.pop()?]), + TypedSamples::Float(mut samples) => TypedSamples::Float(smallvec![samples.pop()?]), + TypedSamples::String(mut samples) => TypedSamples::String(smallvec![samples.pop()?]), + TypedSamples::Boolean(mut samples) => TypedSamples::Boolean(smallvec![samples.pop()?]), + TypedSamples::Location(mut samples) => TypedSamples::Location(smallvec![samples.pop()?]), + TypedSamples::Blob(mut samples) => TypedSamples::Blob(smallvec![samples.pop()?]), + TypedSamples::Json(mut samples) => TypedSamples::Json(smallvec![samples.pop()?]), + }; + + Some(sensor_data) +} + +pub fn summarize_sensor_data_availability( + sensor_data: SensorData, + start_time: SensAppDateTime, + step_ms: Option, +) -> Result { + let start_us = datetime_to_micros(&start_time); + let step_us = step_ms + .map(|value| { + value + .checked_mul(1000) + .ok_or_else(|| anyhow!("step is too large")) + }) + .transpose()?; + + let (sample_count, first_sample_at, last_sample_at, covered_buckets) = match &sensor_data + .samples + { + TypedSamples::Integer(samples) => { + availability_stats_for_samples(samples, start_us, step_us) + } + TypedSamples::Numeric(samples) => { + availability_stats_for_samples(samples, start_us, step_us) + } + TypedSamples::Float(samples) => availability_stats_for_samples(samples, start_us, step_us), + TypedSamples::String(samples) => availability_stats_for_samples(samples, start_us, step_us), + TypedSamples::Boolean(samples) => { + availability_stats_for_samples(samples, start_us, step_us) + } + TypedSamples::Location(samples) => { + availability_stats_for_samples(samples, start_us, step_us) + } + TypedSamples::Blob(samples) => availability_stats_for_samples(samples, start_us, step_us), + TypedSamples::Json(samples) => availability_stats_for_samples(samples, start_us, step_us), + }; + + Ok(SensorAvailabilitySummary { + sensor: sensor_data.sensor, + sample_count, + first_sample_at, + last_sample_at, + covered_buckets, + }) +} + +fn availability_stats_for_samples( + samples: &[Sample], + start_us: i64, + step_us: Option, +) -> ( + usize, + Option, + Option, + Option, +) { + let covered_buckets = step_us.map(|step_us| { + samples + .iter() + .map(|sample| (datetime_to_micros(&sample.datetime) - start_us).div_euclid(step_us)) + .collect::>() + .len() + }); + + ( + samples.len(), + samples.first().map(|sample| sample.datetime), + samples.last().map(|sample| sample.datetime), + covered_buckets, + ) +} + +fn aggregate_sensor_data( + mut sensor_data: SensorData, + start_time: Option, + step_ms: i64, + aggregation: Aggregation, +) -> Result { + let origin_us = start_time.as_ref().map(datetime_to_micros).unwrap_or(0); + let step_us = step_ms + .checked_mul(1000) + .ok_or_else(|| anyhow!("step is too large"))?; + + sensor_data.samples = match sensor_data.samples { + TypedSamples::Float(samples) => { + aggregate_float_samples(samples.as_slice(), origin_us, step_us, aggregation)? + } + TypedSamples::Integer(samples) => { + aggregate_integer_samples(samples.as_slice(), origin_us, step_us, aggregation)? + } + TypedSamples::Numeric(samples) => { + aggregate_numeric_samples(samples.as_slice(), origin_us, step_us, aggregation)? + } + _ => { + return Err(anyhow!("aggregation is only supported for numeric series")); + } + }; + + sensor_data.sensor.sensor_type = match &sensor_data.samples { + TypedSamples::Float(_) => SensorType::Float, + TypedSamples::Integer(_) => SensorType::Integer, + TypedSamples::Numeric(_) => SensorType::Numeric, + _ => sensor_data.sensor.sensor_type, + }; + + if aggregation.output_is_count() { + sensor_data.sensor.sensor_type = SensorType::Integer; + sensor_data.sensor.unit = None; + } + + Ok(sensor_data) +} + +fn simplify_sensor_data( + mut sensor_data: SensorData, + options: SimplifyOptions, +) -> Result { + sensor_data.samples = match sensor_data.samples { + TypedSamples::Float(samples) => { + let keep_indices = simplify_indices_f64( + &samples, + |sample| sample.value, + options.tolerance, + options.high_quality, + )?; + TypedSamples::Float(filter_samples_by_indices(samples, keep_indices)) + } + TypedSamples::Integer(samples) => { + let keep_indices = simplify_indices_f64( + &samples, + |sample| sample.value as f64, + options.tolerance, + options.high_quality, + )?; + TypedSamples::Integer(filter_samples_by_indices(samples, keep_indices)) + } + TypedSamples::Numeric(samples) => { + let keep_indices = simplify_indices_f64( + &samples, + |sample| sample.value.to_f64().unwrap_or(0.0), + options.tolerance, + options.high_quality, + )?; + TypedSamples::Numeric(filter_samples_by_indices(samples, keep_indices)) + } + _ => { + return Err(anyhow!("simplify is only supported for numeric series")); + } + }; + + Ok(sensor_data) +} + +fn filter_samples_by_indices( + samples: crate::datamodel::sensapp_vec::SensAppVec>, + keep_indices: Vec, +) -> crate::datamodel::sensapp_vec::SensAppVec> { + let mut keep_iter = keep_indices.into_iter().peekable(); + let mut filtered = smallvec![]; + + for (index, sample) in samples.into_iter().enumerate() { + if keep_iter.peek().copied() == Some(index) { + filtered.push(sample); + keep_iter.next(); + } + } + + filtered +} + +fn simplify_indices_f64( + samples: &[Sample], + value_fn: F, + tolerance: f64, + high_quality: bool, +) -> Result> +where + F: Fn(&Sample) -> f64, +{ + if samples.len() <= 2 { + return Ok((0..samples.len()).collect()); + } + + let min_ts = datetime_to_micros(&samples[0].datetime); + let max_ts = datetime_to_micros(&samples[samples.len() - 1].datetime); + let mut min_val = f64::INFINITY; + let mut max_val = f64::NEG_INFINITY; + for sample in samples { + let value = value_fn(sample); + min_val = min_val.min(value); + max_val = max_val.max(value); + } + + let ts_span = (max_ts - min_ts).max(1) as f64; + let value_span = (max_val - min_val).abs().max(f64::EPSILON); + + let points: Vec> = samples + .iter() + .map(|sample| Point { + vec: [ + (datetime_to_micros(&sample.datetime) - min_ts) as f64 / ts_span, + (value_fn(sample) - min_val) / value_span, + ], + }) + .collect(); + + let simplified = simplify(&points, tolerance, high_quality); + let mut keep_indices = Vec::with_capacity(simplified.len()); + let mut search_start = 0usize; + + for point in simplified { + let relative_index = points[search_start..] + .iter() + .position(|candidate| *candidate == point) + .ok_or_else(|| anyhow!("failed to map simplified points back to samples"))?; + let absolute_index = search_start + relative_index; + keep_indices.push(absolute_index); + search_start = absolute_index.saturating_add(1); + } + + Ok(keep_indices) +} + +fn bucket_start(timestamp_us: i64, origin_us: i64, step_us: i64) -> i64 { + origin_us + (timestamp_us - origin_us).div_euclid(step_us) * step_us +} + +fn aggregate_float_samples( + samples: &[Sample], + origin_us: i64, + step_us: i64, + aggregation: Aggregation, +) -> Result { + if samples.is_empty() { + return Ok(TypedSamples::Float(smallvec![])); + } + + let mut output_float = smallvec![]; + let mut output_int = smallvec![]; + let mut index = 0usize; + + while index < samples.len() { + let bucket_us = bucket_start( + datetime_to_micros(&samples[index].datetime), + origin_us, + step_us, + ); + let mut end = index; + let mut sum = 0.0; + let mut min = samples[index].value; + let mut max = samples[index].value; + let first = samples[index].value; + let mut last = samples[index].value; + let mut count = 0i64; + + while end < samples.len() + && bucket_start( + datetime_to_micros(&samples[end].datetime), + origin_us, + step_us, + ) == bucket_us + { + let value = samples[end].value; + sum += value; + min = min.min(value); + max = max.max(value); + last = value; + count += 1; + end += 1; + } + + let datetime = SensAppDateTime::from_unix_microseconds_i64(bucket_us); + match aggregation { + Aggregation::Avg => output_float.push(Sample { + datetime, + value: sum / count as f64, + }), + Aggregation::Min => output_float.push(Sample { + datetime, + value: min, + }), + Aggregation::Max => output_float.push(Sample { + datetime, + value: max, + }), + Aggregation::Sum => output_float.push(Sample { + datetime, + value: sum, + }), + Aggregation::First => output_float.push(Sample { + datetime, + value: first, + }), + Aggregation::Last => output_float.push(Sample { + datetime, + value: last, + }), + Aggregation::Count => output_int.push(Sample { + datetime, + value: count, + }), + } + + index = end; + } + + Ok(if aggregation.output_is_count() { + TypedSamples::Integer(output_int) + } else { + TypedSamples::Float(output_float) + }) +} + +fn aggregate_integer_samples( + samples: &[Sample], + origin_us: i64, + step_us: i64, + aggregation: Aggregation, +) -> Result { + if samples.is_empty() { + return Ok(TypedSamples::Integer(smallvec![])); + } + + let mut output_int = smallvec![]; + let mut output_float = smallvec![]; + let mut index = 0usize; + + while index < samples.len() { + let bucket_us = bucket_start( + datetime_to_micros(&samples[index].datetime), + origin_us, + step_us, + ); + let mut end = index; + let mut sum: i128 = 0; + let mut min = samples[index].value; + let mut max = samples[index].value; + let first = samples[index].value; + let mut last = samples[index].value; + let mut count = 0i64; + + while end < samples.len() + && bucket_start( + datetime_to_micros(&samples[end].datetime), + origin_us, + step_us, + ) == bucket_us + { + let value = samples[end].value; + sum += value as i128; + min = min.min(value); + max = max.max(value); + last = value; + count += 1; + end += 1; + } + + let datetime = SensAppDateTime::from_unix_microseconds_i64(bucket_us); + match aggregation { + Aggregation::Avg => output_float.push(Sample { + datetime, + value: sum as f64 / count as f64, + }), + Aggregation::Min => output_int.push(Sample { + datetime, + value: min, + }), + Aggregation::Max => output_int.push(Sample { + datetime, + value: max, + }), + Aggregation::Sum => output_int.push(Sample { + datetime, + value: i64::try_from(sum).map_err(|_| anyhow!("integer aggregation overflowed"))?, + }), + Aggregation::First => output_int.push(Sample { + datetime, + value: first, + }), + Aggregation::Last => output_int.push(Sample { + datetime, + value: last, + }), + Aggregation::Count => output_int.push(Sample { + datetime, + value: count, + }), + } + + index = end; + } + + Ok(if matches!(aggregation, Aggregation::Avg) { + TypedSamples::Float(output_float) + } else { + TypedSamples::Integer(output_int) + }) +} + +fn aggregate_numeric_samples( + samples: &[Sample], + origin_us: i64, + step_us: i64, + aggregation: Aggregation, +) -> Result { + if samples.is_empty() { + return Ok(TypedSamples::Numeric(smallvec![])); + } + + let mut output_numeric = smallvec![]; + let mut output_int = smallvec![]; + let mut index = 0usize; + + while index < samples.len() { + let bucket_us = bucket_start( + datetime_to_micros(&samples[index].datetime), + origin_us, + step_us, + ); + let mut end = index; + let mut sum = Decimal::ZERO; + let mut min = samples[index].value; + let mut max = samples[index].value; + let first = samples[index].value; + let mut last = samples[index].value; + let mut count = 0i64; + + while end < samples.len() + && bucket_start( + datetime_to_micros(&samples[end].datetime), + origin_us, + step_us, + ) == bucket_us + { + let value = samples[end].value; + sum += value; + min = min.min(value); + max = max.max(value); + last = value; + count += 1; + end += 1; + } + + let datetime = SensAppDateTime::from_unix_microseconds_i64(bucket_us); + match aggregation { + Aggregation::Avg => output_numeric.push(Sample { + datetime, + value: sum / Decimal::from(count), + }), + Aggregation::Min => output_numeric.push(Sample { + datetime, + value: min, + }), + Aggregation::Max => output_numeric.push(Sample { + datetime, + value: max, + }), + Aggregation::Sum => output_numeric.push(Sample { + datetime, + value: sum, + }), + Aggregation::First => output_numeric.push(Sample { + datetime, + value: first, + }), + Aggregation::Last => output_numeric.push(Sample { + datetime, + value: last, + }), + Aggregation::Count => output_int.push(Sample { + datetime, + value: count, + }), + } + + index = end; + } + + Ok(if aggregation.output_is_count() { + TypedSamples::Integer(output_int) + } else { + TypedSamples::Numeric(output_numeric) + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/storage/data_query.rs b/src/storage/data_query.rs new file mode 100644 index 00000000..250d0dd3 --- /dev/null +++ b/src/storage/data_query.rs @@ -0,0 +1,82 @@ +use crate::datamodel::SensAppDateTime; +use anyhow::{Result, anyhow}; +use std::str::FromStr; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Aggregation { + Avg, + Min, + Max, + Sum, + Count, + First, + Last, +} + +impl Aggregation { + pub fn output_is_count(self) -> bool { + matches!(self, Self::Count) + } +} + +impl FromStr for Aggregation { + type Err = anyhow::Error; + + fn from_str(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "avg" => Ok(Self::Avg), + "min" => Ok(Self::Min), + "max" => Ok(Self::Max), + "sum" => Ok(Self::Sum), + "count" => Ok(Self::Count), + "first" => Ok(Self::First), + "last" => Ok(Self::Last), + _ => Err(anyhow!( + "Unsupported aggregation '{}'. Supported values: avg, min, max, sum, count, first, last", + value + )), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct SimplifyOptions { + pub tolerance: f64, + pub high_quality: bool, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct SensorDataQueryOptions { + pub start_time: Option, + pub end_time: Option, + pub limit: Option, + pub step_ms: Option, + pub aggregation: Option, + pub simplify: Option, +} + +impl SensorDataQueryOptions { + pub fn validate(&self) -> Result<()> { + if self.step_ms.is_some() != self.aggregation.is_some() { + return Err(anyhow!( + "'step' and 'aggregation' must be provided together" + )); + } + + if let Some(step_ms) = self.step_ms + && step_ms <= 0 + { + return Err(anyhow!("'step' must be greater than zero")); + } + + if let Some(simplify) = self.simplify + && (!simplify.tolerance.is_finite() || simplify.tolerance <= 0.0) + { + return Err(anyhow!( + "'simplify_tolerance' must be a finite number greater than zero" + )); + } + + Ok(()) + } +} diff --git a/src/storage/duckdb/duckdb_utilities.rs b/src/storage/duckdb/duckdb_utilities.rs index 62cf5964..f69b411c 100644 --- a/src/storage/duckdb/duckdb_utilities.rs +++ b/src/storage/duckdb/duckdb_utilities.rs @@ -3,7 +3,6 @@ use crate::datamodel::unit::Unit; use anyhow::Result; use cached::proc_macro::cached; use duckdb::{CachedStatement, OptionalExt, Transaction, params}; -use std::time::Duration; use uuid::Uuid; #[cached( diff --git a/src/storage/duckdb/migrations/20240223133248_init.sql b/src/storage/duckdb/migrations/20240223133248_init.sql index 7713fb15..21a2b51e 100644 --- a/src/storage/duckdb/migrations/20240223133248_init.sql +++ b/src/storage/duckdb/migrations/20240223133248_init.sql @@ -65,7 +65,7 @@ CREATE TABLE IF NOT EXISTS integer_values ( CREATE TABLE IF NOT EXISTS numeric_values ( sensor_id BIGINT NOT NULL, timestamp_ms TIMESTAMP_MS NOT NULL, - value DECIMAL(18,6) NOT NULL, + value DECIMAL(38,9) NOT NULL, --FOREIGN KEY (sensor_id) REFERENCES sensors(sensor_id) ); diff --git a/src/storage/duckdb/mod.rs b/src/storage/duckdb/mod.rs index e9b4a6c5..d873ca0f 100644 --- a/src/storage/duckdb/mod.rs +++ b/src/storage/duckdb/mod.rs @@ -1,15 +1,29 @@ -use crate::datamodel::TypedSamples; use crate::datamodel::batch::{Batch, SingleSensorBatch}; +use crate::datamodel::sensapp_datetime::SensAppDateTimeExt; +use crate::datamodel::sensapp_vec::SensAppLabels; +use crate::datamodel::unit::Unit; +use crate::datamodel::{ + Metric, Sample, SensAppDateTime, Sensor, SensorData, SensorType, TypedSamples, +}; use anyhow::{Context, Result, bail}; use async_trait::async_trait; use duckdb::Connection; use duckdb_publishers::*; use duckdb_utilities::get_sensor_id_or_create_sensor; +use geo::Point; +use regex::Regex; +use rust_decimal::Decimal; +use serde_json::Value as JsonValue; +use smallvec::smallvec; +use std::str::FromStr; use std::sync::Arc; use tokio::sync::Mutex; use tokio::task::spawn_blocking; +use uuid::Uuid; -use super::StorageInstance; +use super::{ + Aggregation, SensorAvailabilitySummary, SensorDataQueryOptions, StorageError, StorageInstance, +}; mod duckdb_publishers; mod duckdb_utilities; @@ -81,31 +95,831 @@ impl StorageInstance for DuckDBStorage { async fn list_series( &self, - _metric_filter: Option<&str>, - ) -> Result> { - unimplemented!(); + metric_filter: Option<&str>, + limit: Option, + bookmark: Option<&str>, + ) -> Result { + let connection = Arc::clone(&self.connection); + let metric_filter = metric_filter.map(str::to_string); + let bookmark_id = if let Some(bookmark) = bookmark { + Some(bookmark.parse::().map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Invalid bookmark format: {}", e), + None, + None, + )) + })?) + } else { + None + }; + let effective_limit = limit + .unwrap_or(crate::storage::DEFAULT_LIST_SERIES_LIMIT) + .min(crate::storage::MAX_LIST_SERIES_LIMIT); + let fetch_limit = effective_limit.saturating_add(1) as i64; + + spawn_blocking(move || -> Result { + let connection = connection.blocking_lock(); + let (sql, use_filter, use_bookmark) = match (metric_filter.is_some(), bookmark_id.is_some()) { + (true, true) => ( + r#" + SELECT s.sensor_id, CAST(s.uuid AS TEXT) AS uuid, s.name, s.type, u.name, u.description + FROM sensors s + LEFT JOIN units u ON s.unit = u.id + WHERE s.name = ? AND s.sensor_id > ? + ORDER BY s.sensor_id ASC + LIMIT ? + "#, + true, + true, + ), + (true, false) => ( + r#" + SELECT s.sensor_id, CAST(s.uuid AS TEXT) AS uuid, s.name, s.type, u.name, u.description + FROM sensors s + LEFT JOIN units u ON s.unit = u.id + WHERE s.name = ? + ORDER BY s.sensor_id ASC + LIMIT ? + "#, + true, + false, + ), + (false, true) => ( + r#" + SELECT s.sensor_id, CAST(s.uuid AS TEXT) AS uuid, s.name, s.type, u.name, u.description + FROM sensors s + LEFT JOIN units u ON s.unit = u.id + WHERE s.sensor_id > ? + ORDER BY s.sensor_id ASC + LIMIT ? + "#, + false, + true, + ), + (false, false) => ( + r#" + SELECT s.sensor_id, CAST(s.uuid AS TEXT) AS uuid, s.name, s.type, u.name, u.description + FROM sensors s + LEFT JOIN units u ON s.unit = u.id + ORDER BY s.sensor_id ASC + LIMIT ? + "#, + false, + false, + ), + }; + + let mut statement = connection.prepare(sql)?; + let mut rows = match (use_filter, use_bookmark) { + (true, true) => statement.query(duckdb::params![ + metric_filter.as_deref().unwrap(), + bookmark_id.unwrap(), + fetch_limit, + ])?, + (true, false) => statement.query(duckdb::params![ + metric_filter.as_deref().unwrap(), + fetch_limit, + ])?, + (false, true) => { + statement.query(duckdb::params![bookmark_id.unwrap(), fetch_limit])? + } + (false, false) => statement.query(duckdb::params![fetch_limit])?, + }; + + let mut label_statement = connection.prepare( + r#" + SELECT lnd.name, ldd.description + FROM labels l + JOIN labels_name_dictionary lnd ON l.name = lnd.id + JOIN labels_description_dictionary ldd ON l.description = ldd.id + WHERE l.sensor_id = ? + "#, + )?; + + let mut sensors = Vec::new(); + let mut last_sensor_id = None; + let mut has_more = false; + + while let Some(row) = rows.next()? { + if sensors.len() == effective_limit { + has_more = true; + break; + } + + let sensor_id: i64 = row.get(0)?; + + let sensor_uuid: String = row.get(1)?; + let sensor_name: String = row.get(2)?; + let sensor_type_str: String = row.get(3)?; + let unit_name: Option = row.get(4)?; + let unit_description: Option = row.get(5)?; + + let sensor_uuid = Uuid::parse_str(&sensor_uuid).map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Failed to parse sensor UUID '{}': {}", sensor_uuid, e), + None, + Some(&sensor_name), + )) + })?; + + let sensor_type = SensorType::from_str(&sensor_type_str).map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Failed to parse sensor type '{}': {}", sensor_type_str, e), + Some(sensor_uuid), + Some(&sensor_name), + )) + })?; + + let unit = unit_name.map(|unit_name| Unit::new(unit_name, unit_description.clone())); + + let mut label_rows = label_statement.query([sensor_id])?; + let mut labels: SensAppLabels = smallvec![]; + while let Some(label_row) = label_rows.next()? { + let label_name: String = label_row.get(0)?; + let label_value: String = label_row.get(1)?; + labels.push((label_name, label_value)); + } + + sensors.push(Sensor::new( + sensor_uuid, + sensor_name, + sensor_type, + unit, + Some(labels), + )); + + last_sensor_id = Some(sensor_id); + } + + let next_bookmark = if has_more { + last_sensor_id.map(|value| value.to_string()) + } else { + None + }; + + Ok(crate::storage::ListSeriesResult { + series: sensors, + bookmark: next_bookmark, + }) + }) + .await? + } + + async fn list_metrics(&self) -> Result> { + let connection = Arc::clone(&self.connection); + + spawn_blocking(move || -> Result> { + let connection = connection.blocking_lock(); + let mut statement = connection.prepare( + r#" + SELECT s.name, s.type, u.name, u.description, COUNT(*) + FROM sensors s + LEFT JOIN units u ON s.unit = u.id + GROUP BY s.name, s.type, u.name, u.description + ORDER BY s.name ASC + "#, + )?; + + let mut rows = statement.query([])?; + let mut metrics = Vec::new(); + + while let Some(row) = rows.next()? { + let metric_name: String = row.get(0)?; + let sensor_type_str: String = row.get(1)?; + let unit_name: Option = row.get(2)?; + let unit_description: Option = row.get(3)?; + let series_count: i64 = row.get(4)?; + + let sensor_type = SensorType::from_str(&sensor_type_str).map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Failed to parse sensor type '{}': {}", sensor_type_str, e), + None, + Some(&metric_name), + )) + })?; + + let unit = + unit_name.map(|unit_name| Unit::new(unit_name, unit_description.clone())); + + metrics.push(Metric::new( + metric_name, + sensor_type, + unit, + series_count, + Vec::new(), + )); + } + + Ok(metrics) + }) + .await? } async fn query_sensor_data( &self, - _sensor_uuid: &str, - _start_time: Option, - _end_time: Option, - _limit: Option, + sensor_uuid: &str, + start_time: Option, + end_time: Option, + limit: Option, ) -> Result> { - unimplemented!("DuckDB sensor data querying not yet implemented"); + let connection = Arc::clone(&self.connection); + let sensor_uuid = sensor_uuid.to_string(); + let start_time_ms = start_time.map(|time| time.to_unix_milliseconds().floor() as i64); + let end_time_ms = end_time.map(|time| time.to_unix_milliseconds().floor() as i64); + + spawn_blocking(move || -> Result> { + let connection = connection.blocking_lock(); + + let mut sensor_statement = connection.prepare( + r#" + SELECT s.sensor_id, CAST(s.uuid AS TEXT) AS uuid, s.name, s.type, u.name, u.description + FROM sensors s + LEFT JOIN units u ON s.unit = u.id + WHERE CAST(s.uuid AS TEXT) = ? + "#, + )?; + + let mut sensor_rows = sensor_statement.query([sensor_uuid.as_str()])?; + let Some(sensor_row) = sensor_rows.next()? else { + return Ok(None); + }; + + let sensor_id: i64 = sensor_row.get(0)?; + let sensor_uuid_str: String = sensor_row.get(1)?; + let sensor_name: String = sensor_row.get(2)?; + let sensor_type_str: String = sensor_row.get(3)?; + let unit_name: Option = sensor_row.get(4)?; + let unit_description: Option = sensor_row.get(5)?; + + let sensor_uuid = Uuid::parse_str(&sensor_uuid_str).map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Failed to parse sensor UUID '{}': {}", sensor_uuid_str, e), + None, + Some(&sensor_name), + )) + })?; + + let sensor_type = SensorType::from_str(&sensor_type_str).map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Failed to parse sensor type '{}': {}", sensor_type_str, e), + Some(sensor_uuid), + Some(&sensor_name), + )) + })?; + + let unit = unit_name.map(|name| Unit::new(name, unit_description.clone())); + + let mut label_statement = connection.prepare( + r#" + SELECT lnd.name, ldd.description + FROM labels l + JOIN labels_name_dictionary lnd ON l.name = lnd.id + JOIN labels_description_dictionary ldd ON l.description = ldd.id + WHERE l.sensor_id = ? + "#, + )?; + let mut label_rows = label_statement.query([sensor_id])?; + let mut labels: SensAppLabels = smallvec![]; + while let Some(label_row) = label_rows.next()? { + let label_name: String = label_row.get(0)?; + let label_value: String = label_row.get(1)?; + labels.push((label_name, label_value)); + } + + let sensor = Sensor::new(sensor_uuid, sensor_name, sensor_type, unit, Some(labels)); + + fn is_in_range(timestamp_ms: i64, start: Option, end: Option) -> bool { + start.is_none_or(|value| timestamp_ms >= value) + && end.is_none_or(|value| timestamp_ms <= value) + } + + let samples = match sensor.sensor_type { + SensorType::Integer => { + let mut statement = connection.prepare( + r#" + SELECT epoch_ms(timestamp_ms), value + FROM integer_values + WHERE sensor_id = ? + ORDER BY timestamp_ms ASC + "#, + )?; + let mut rows = statement.query([sensor_id])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + if !is_in_range(timestamp_ms, start_time_ms, end_time_ms) { + continue; + } + let value: i64 = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64(timestamp_ms), + value, + }); + if limit.is_some_and(|max| samples.len() >= max) { + break; + } + } + TypedSamples::Integer(samples) + } + SensorType::Numeric => { + let mut statement = connection.prepare( + r#" + SELECT epoch_ms(timestamp_ms), CAST(value AS VARCHAR) + FROM numeric_values + WHERE sensor_id = ? + ORDER BY timestamp_ms ASC + "#, + )?; + let mut rows = statement.query([sensor_id])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + if !is_in_range(timestamp_ms, start_time_ms, end_time_ms) { + continue; + } + let value: String = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64(timestamp_ms), + value: Decimal::from_str(&value) + .context("Failed to parse DuckDB numeric value")?, + }); + if limit.is_some_and(|max| samples.len() >= max) { + break; + } + } + TypedSamples::Numeric(samples) + } + SensorType::Float => { + let mut statement = connection.prepare( + r#" + SELECT epoch_ms(timestamp_ms), value + FROM float_values + WHERE sensor_id = ? + ORDER BY timestamp_ms ASC + "#, + )?; + let mut rows = statement.query([sensor_id])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + if !is_in_range(timestamp_ms, start_time_ms, end_time_ms) { + continue; + } + let value: f64 = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64(timestamp_ms), + value, + }); + if limit.is_some_and(|max| samples.len() >= max) { + break; + } + } + TypedSamples::Float(samples) + } + SensorType::String => { + let mut statement = connection.prepare( + r#" + SELECT epoch_ms(sv.timestamp_ms), svd.value + FROM string_values sv + JOIN strings_values_dictionary svd ON sv.value = svd.id + WHERE sv.sensor_id = ? + ORDER BY sv.timestamp_ms ASC + "#, + )?; + let mut rows = statement.query([sensor_id])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + if !is_in_range(timestamp_ms, start_time_ms, end_time_ms) { + continue; + } + let value: String = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64(timestamp_ms), + value, + }); + if limit.is_some_and(|max| samples.len() >= max) { + break; + } + } + TypedSamples::String(samples) + } + SensorType::Boolean => { + let mut statement = connection.prepare( + r#" + SELECT epoch_ms(timestamp_ms), value + FROM boolean_values + WHERE sensor_id = ? + ORDER BY timestamp_ms ASC + "#, + )?; + let mut rows = statement.query([sensor_id])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + if !is_in_range(timestamp_ms, start_time_ms, end_time_ms) { + continue; + } + let value: bool = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64(timestamp_ms), + value, + }); + if limit.is_some_and(|max| samples.len() >= max) { + break; + } + } + TypedSamples::Boolean(samples) + } + SensorType::Location => { + let mut statement = connection.prepare( + r#" + SELECT epoch_ms(timestamp_ms), latitude, longitude + FROM location_values + WHERE sensor_id = ? + ORDER BY timestamp_ms ASC + "#, + )?; + let mut rows = statement.query([sensor_id])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + if !is_in_range(timestamp_ms, start_time_ms, end_time_ms) { + continue; + } + let latitude: f64 = row.get(1)?; + let longitude: f64 = row.get(2)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64(timestamp_ms), + value: Point::new(longitude, latitude), + }); + if limit.is_some_and(|max| samples.len() >= max) { + break; + } + } + TypedSamples::Location(samples) + } + SensorType::Json => { + let mut statement = connection.prepare( + r#" + SELECT epoch_ms(timestamp_ms), CAST(value AS VARCHAR) + FROM json_values + WHERE sensor_id = ? + ORDER BY timestamp_ms ASC + "#, + )?; + let mut rows = statement.query([sensor_id])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + if !is_in_range(timestamp_ms, start_time_ms, end_time_ms) { + continue; + } + let value: String = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64(timestamp_ms), + value: serde_json::from_str::(&value) + .context("Failed to parse DuckDB JSON value")?, + }); + if limit.is_some_and(|max| samples.len() >= max) { + break; + } + } + TypedSamples::Json(samples) + } + SensorType::Blob => { + let mut statement = connection.prepare( + r#" + SELECT epoch_ms(timestamp_ms), value + FROM blob_values + WHERE sensor_id = ? + ORDER BY timestamp_ms ASC + "#, + )?; + let mut rows = statement.query([sensor_id])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + if !is_in_range(timestamp_ms, start_time_ms, end_time_ms) { + continue; + } + let value: Vec = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64(timestamp_ms), + value, + }); + if limit.is_some_and(|max| samples.len() >= max) { + break; + } + } + TypedSamples::Blob(samples) + } + }; + + Ok(Some(SensorData::new(sensor, samples))) + }) + .await? + } + + async fn query_sensor_data_advanced( + &self, + sensor_uuid: &str, + options: &SensorDataQueryOptions, + ) -> Result> { + options.validate()?; + + if let (Some(step_ms), Some(aggregation)) = (options.step_ms, options.aggregation) { + let connection = Arc::clone(&self.connection); + let sensor_uuid = sensor_uuid.to_string(); + let options = options.clone(); + + let aggregated = spawn_blocking(move || -> Result> { + let connection = connection.blocking_lock(); + + let Some((sensor_id, mut sensor)) = + duckdb_get_sensor_metadata(&connection, &sensor_uuid)? + else { + return Ok(None); + }; + + let start_time_ms = options + .start_time + .map(|time| time.to_unix_milliseconds().floor() as i64); + let end_time_ms = options + .end_time + .map(|time| time.to_unix_milliseconds().floor() as i64); + let origin_ms = start_time_ms.unwrap_or(0); + + let samples = match sensor.sensor_type { + SensorType::Integer => duckdb_query_integer_samples_aggregated( + &connection, + sensor_id, + start_time_ms, + end_time_ms, + step_ms, + origin_ms, + aggregation, + options.limit, + )?, + SensorType::Numeric => duckdb_query_numeric_samples_aggregated( + &connection, + sensor_id, + start_time_ms, + end_time_ms, + step_ms, + origin_ms, + aggregation, + options.limit, + )?, + SensorType::Float => duckdb_query_float_samples_aggregated( + &connection, + sensor_id, + start_time_ms, + end_time_ms, + step_ms, + origin_ms, + aggregation, + options.limit, + )?, + _ => return Ok(None), + }; + + sensor.sensor_type = match &samples { + TypedSamples::Integer(_) => SensorType::Integer, + TypedSamples::Numeric(_) => SensorType::Numeric, + TypedSamples::Float(_) => SensorType::Float, + _ => sensor.sensor_type, + }; + + if aggregation.output_is_count() { + sensor.unit = None; + } + + Ok(Some(SensorData::new(sensor, samples))) + }) + .await??; + + if let Some(sensor_data) = aggregated { + let post_query_options = SensorDataQueryOptions { + start_time: options.start_time, + end_time: options.end_time, + limit: options.limit, + step_ms: None, + aggregation: None, + simplify: options.simplify, + }; + + return crate::storage::common::apply_query_options( + sensor_data, + &post_query_options, + ) + .map(Some); + } + } + + let raw = self + .query_sensor_data( + sensor_uuid, + options.start_time, + options.end_time, + options.limit, + ) + .await?; + + raw.map(|sensor_data| crate::storage::common::apply_query_options(sensor_data, options)) + .transpose() + } + + async fn query_sensor_data_latest( + &self, + sensor_uuid: &str, + start_time: Option, + end_time: Option, + ) -> Result> { + let connection = Arc::clone(&self.connection); + let sensor_uuid_owned = sensor_uuid.to_string(); + let start_time_ms = start_time.map(|time| time.to_unix_milliseconds().floor() as i64); + let end_time_ms = end_time.map(|time| time.to_unix_milliseconds().floor() as i64); + + let latest_timestamp_ms = spawn_blocking(move || -> Result> { + let connection = connection.blocking_lock(); + let Some((sensor_id, sensor)) = + duckdb_get_sensor_metadata(&connection, &sensor_uuid_owned)? + else { + return Ok(None); + }; + + duckdb_query_latest_timestamp_ms( + &connection, + duckdb_sensor_table_name(sensor.sensor_type), + sensor_id, + start_time_ms, + end_time_ms, + ) + }) + .await??; + + let Some(latest_timestamp_ms) = latest_timestamp_ms else { + return Ok(None); + }; + + self.query_sensor_data( + sensor_uuid, + Some(SensAppDateTime::from_unix_milliseconds_i64( + latest_timestamp_ms, + )), + Some(SensAppDateTime::from_unix_milliseconds_i64( + latest_timestamp_ms, + )), + Some(1), + ) + .await + } + + async fn query_sensor_data_availability( + &self, + sensor_uuid: &str, + start_time: SensAppDateTime, + end_time: SensAppDateTime, + step_ms: Option, + ) -> Result> { + let connection = Arc::clone(&self.connection); + let sensor_uuid_owned = sensor_uuid.to_string(); + let start_time_ms = start_time.to_unix_milliseconds().floor() as i64; + let end_time_ms = end_time.to_unix_milliseconds().floor() as i64; + + spawn_blocking(move || -> Result> { + let connection = connection.blocking_lock(); + let Some((sensor_id, sensor)) = + duckdb_get_sensor_metadata(&connection, &sensor_uuid_owned)? + else { + return Ok(None); + }; + + let summary = duckdb_query_availability_summary( + &connection, + duckdb_sensor_table_name(sensor.sensor_type), + sensor_id, + sensor, + start_time_ms, + end_time_ms, + step_ms, + )?; + + Ok(Some(summary)) + }) + .await? } async fn query_sensors_by_labels( &self, - _matchers: &[super::LabelMatcher], - _start_time: Option, - _end_time: Option, - _limit: Option, - _numeric_only: bool, + matchers: &[super::LabelMatcher], + start_time: Option, + end_time: Option, + limit: Option, + numeric_only: bool, ) -> Result> { - // TODO: Implement label-based query for DuckDB - anyhow::bail!("query_sensors_by_labels not yet implemented for DuckDB") + use crate::storage::MatcherType; + + if matchers.is_empty() { + return Ok(Vec::new()); + } + + fn matches_value( + actual: Option<&str>, + expected: &str, + matcher_type: MatcherType, + ) -> Result { + Ok(match matcher_type { + MatcherType::Equal => actual == Some(expected), + MatcherType::NotEqual => actual != Some(expected), + MatcherType::RegexMatch => { + if let Some(value) = actual { + Regex::new(expected)?.is_match(value) + } else { + false + } + } + MatcherType::RegexNotMatch => { + if let Some(value) = actual { + !Regex::new(expected)?.is_match(value) + } else { + true + } + } + }) + } + + fn sensor_matches(sensor: &Sensor, matchers: &[super::LabelMatcher]) -> Result { + for matcher in matchers { + if matcher.is_name_matcher() { + if !matches_value(Some(&sensor.name), &matcher.value, matcher.matcher_type)? { + return Ok(false); + } + continue; + } + + let actual = sensor + .labels + .iter() + .find(|(name, _)| name == &matcher.name) + .map(|(_, value)| value.as_str()); + + if !matches_value(actual, &matcher.value, matcher.matcher_type)? { + return Ok(false); + } + } + + Ok(true) + } + + fn is_numeric_sensor_type(sensor_type: SensorType) -> bool { + matches!( + sensor_type, + SensorType::Integer | SensorType::Numeric | SensorType::Float + ) + } + + let mut matching_sensors = Vec::new(); + let mut bookmark = None; + + loop { + let page = self + .list_series( + None, + Some(crate::storage::MAX_LIST_SERIES_LIMIT), + bookmark.as_deref(), + ) + .await?; + + for sensor in page.series { + if numeric_only && !is_numeric_sensor_type(sensor.sensor_type) { + continue; + } + + if sensor_matches(&sensor, matchers)? { + matching_sensors.push(sensor); + } + } + + if page.bookmark.is_none() { + break; + } + + bookmark = page.bookmark; + } + + let mut results = Vec::new(); + for sensor in matching_sensors { + if let Some(sensor_data) = self + .query_sensor_data(&sensor.uuid.to_string(), start_time, end_time, limit) + .await? + { + results.push(sensor_data); + } + } + + Ok(results) } /// Health check for DuckDB storage @@ -151,11 +965,19 @@ impl StorageInstance for DuckDBStorage { // Step 2: Clear all cached function caches // The cached macro generates cache variables named after the function in uppercase use cached::Cached; - duckdb_utilities::GET_LABEL_NAME_ID_OR_CREATE.cache_clear(); - duckdb_utilities::GET_LABEL_DESCRIPTION_ID_OR_CREATE.cache_clear(); - duckdb_utilities::GET_UNIT_ID_OR_CREATE.cache_clear(); - duckdb_utilities::GET_SENSOR_ID_OR_CREATE_SENSOR.cache_clear(); - duckdb_utilities::GET_STRING_VALUE_ID_OR_CREATE.cache_clear(); + duckdb_utilities::GET_LABEL_NAME_ID_OR_CREATE + .lock() + .cache_clear(); + duckdb_utilities::GET_LABEL_DESCRIPTION_ID_OR_CREATE + .lock() + .cache_clear(); + duckdb_utilities::GET_UNIT_ID_OR_CREATE.lock().cache_clear(); + duckdb_utilities::GET_SENSOR_ID_OR_CREATE_SENSOR + .lock() + .cache_clear(); + duckdb_utilities::GET_STRING_VALUE_ID_OR_CREATE + .lock() + .cache_clear(); Ok(()) } @@ -197,3 +1019,574 @@ fn publish_single_sensor_batch( } Ok(()) } + +fn duckdb_get_sensor_metadata( + connection: &Connection, + sensor_uuid: &str, +) -> Result> { + let mut sensor_statement = connection.prepare( + r#" + SELECT s.sensor_id, CAST(s.uuid AS TEXT) AS uuid, s.name, s.type, u.name, u.description + FROM sensors s + LEFT JOIN units u ON s.unit = u.id + WHERE CAST(s.uuid AS TEXT) = ? + "#, + )?; + + let mut sensor_rows = sensor_statement.query([sensor_uuid])?; + let Some(sensor_row) = sensor_rows.next()? else { + return Ok(None); + }; + + let sensor_id: i64 = sensor_row.get(0)?; + let sensor_uuid_str: String = sensor_row.get(1)?; + let sensor_name: String = sensor_row.get(2)?; + let sensor_type_str: String = sensor_row.get(3)?; + let unit_name: Option = sensor_row.get(4)?; + let unit_description: Option = sensor_row.get(5)?; + + let sensor_uuid = Uuid::parse_str(&sensor_uuid_str).map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Failed to parse sensor UUID '{}': {}", sensor_uuid_str, e), + None, + Some(&sensor_name), + )) + })?; + + let sensor_type = SensorType::from_str(&sensor_type_str).map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Failed to parse sensor type '{}': {}", sensor_type_str, e), + Some(sensor_uuid), + Some(&sensor_name), + )) + })?; + + let unit = unit_name.map(|name| Unit::new(name, unit_description.clone())); + + let mut label_statement = connection.prepare( + r#" + SELECT lnd.name, ldd.description + FROM labels l + JOIN labels_name_dictionary lnd ON l.name = lnd.id + JOIN labels_description_dictionary ldd ON l.description = ldd.id + WHERE l.sensor_id = ? + "#, + )?; + let mut label_rows = label_statement.query([sensor_id])?; + let mut labels: SensAppLabels = smallvec![]; + while let Some(label_row) = label_rows.next()? { + let label_name: String = label_row.get(0)?; + let label_value: String = label_row.get(1)?; + labels.push((label_name, label_value)); + } + + Ok(Some(( + sensor_id, + Sensor::new(sensor_uuid, sensor_name, sensor_type, unit, Some(labels)), + ))) +} + +fn duckdb_sensor_table_name(sensor_type: SensorType) -> &'static str { + match sensor_type { + SensorType::Integer => "integer_values", + SensorType::Numeric => "numeric_values", + SensorType::Float => "float_values", + SensorType::String => "string_values", + SensorType::Boolean => "boolean_values", + SensorType::Location => "location_values", + SensorType::Json => "json_values", + SensorType::Blob => "blob_values", + } +} + +fn duckdb_query_latest_timestamp_ms( + connection: &Connection, + table_name: &str, + sensor_id: i64, + start_time_ms: Option, + end_time_ms: Option, +) -> Result> { + let sql = format!( + r#" + SELECT epoch_ms(MAX(timestamp_ms)) + FROM {table_name} + WHERE sensor_id = ?1 + AND (?2 IS NULL OR timestamp_ms >= epoch_ms(?2)) + AND (?3 IS NULL OR timestamp_ms <= epoch_ms(?3)) + "# + ); + + let mut statement = connection.prepare(&sql)?; + let value: Option = statement.query_row( + duckdb::params![sensor_id, start_time_ms, end_time_ms], + |row| row.get(0), + )?; + + Ok(value) +} + +fn duckdb_query_availability_summary( + connection: &Connection, + table_name: &str, + sensor_id: i64, + sensor: Sensor, + start_time_ms: i64, + end_time_ms: i64, + step_ms: Option, +) -> Result { + let (sql, params): (String, Vec) = if let Some(step_ms) = step_ms { + ( + format!( + r#" + SELECT + COUNT(*) AS sample_count, + epoch_ms(MIN(timestamp_ms)) AS first_sample_at, + epoch_ms(MAX(timestamp_ms)) AS last_sample_at, + COUNT(DISTINCT time_bucket(INTERVAL '{step_ms} milliseconds', timestamp_ms, epoch_ms(?2))) AS covered_buckets + FROM {table_name} + WHERE sensor_id = ?1 + AND timestamp_ms >= epoch_ms(?2) + AND timestamp_ms <= epoch_ms(?3) + "# + ), + vec![sensor_id, start_time_ms, end_time_ms], + ) + } else { + ( + format!( + r#" + SELECT + COUNT(*) AS sample_count, + epoch_ms(MIN(timestamp_ms)) AS first_sample_at, + epoch_ms(MAX(timestamp_ms)) AS last_sample_at, + NULL AS covered_buckets + FROM {table_name} + WHERE sensor_id = ?1 + AND timestamp_ms >= epoch_ms(?2) + AND timestamp_ms <= epoch_ms(?3) + "# + ), + vec![sensor_id, start_time_ms, end_time_ms], + ) + }; + + let mut statement = connection.prepare(&sql)?; + let row = statement.query_row(duckdb::params_from_iter(params.iter()), |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + )) + })?; + + Ok(SensorAvailabilitySummary { + sensor, + sample_count: row.0.max(0) as usize, + first_sample_at: row.1.map(SensAppDateTime::from_unix_milliseconds_i64), + last_sample_at: row.2.map(SensAppDateTime::from_unix_milliseconds_i64), + covered_buckets: row.3.map(|value| value.max(0) as usize), + }) +} + +fn duckdb_bucketed_cte(table_name: &str, step_ms: i64) -> String { + format!( + r#" + WITH bucketed AS ( + SELECT + time_bucket(INTERVAL '{step_ms} milliseconds', timestamp_ms, epoch_ms(?4)) AS bucket_ts, + timestamp_ms, + value + FROM {table_name} + WHERE sensor_id = ?1 + AND (?2 IS NULL OR timestamp_ms >= epoch_ms(?2)) + AND (?3 IS NULL OR timestamp_ms <= epoch_ms(?3)) + ) + "# + ) +} + +fn duckdb_group_by_clause() -> &'static str { + "FROM bucketed GROUP BY 1 ORDER BY 1 ASC LIMIT ?5" +} + +fn duckdb_first_last_query( + table_name: &str, + step_ms: i64, + aggregation: Aggregation, + value_expression: &str, +) -> String { + let direction = match aggregation { + Aggregation::First => "ASC", + Aggregation::Last => "DESC", + _ => unreachable!("only first/last use duckdb_first_last_query"), + }; + + format!( + r#" + WITH bucketed AS ( + SELECT + time_bucket(INTERVAL '{step_ms} milliseconds', timestamp_ms, epoch_ms(?4)) AS bucket_ts, + timestamp_ms, + {value_expression} AS value + FROM {table_name} + WHERE sensor_id = ?1 + AND (?2 IS NULL OR timestamp_ms >= epoch_ms(?2)) + AND (?3 IS NULL OR timestamp_ms <= epoch_ms(?3)) + ), + ranked AS ( + SELECT + bucket_ts, + value, + ROW_NUMBER() OVER (PARTITION BY bucket_ts ORDER BY timestamp_ms {direction}) AS row_num + FROM bucketed + ) + SELECT epoch_ms(bucket_ts) AS timestamp_ms, value + FROM ranked + WHERE row_num = 1 + ORDER BY bucket_ts ASC + LIMIT ?5 + "# + ) +} + +#[allow(clippy::too_many_arguments)] +fn duckdb_query_integer_samples_aggregated( + connection: &Connection, + sensor_id: i64, + start_time_ms: Option, + end_time_ms: Option, + step_ms: i64, + origin_ms: i64, + aggregation: Aggregation, + limit: Option, +) -> Result { + let limit = limit.unwrap_or(super::DEFAULT_QUERY_LIMIT) as i64; + + match aggregation { + Aggregation::Avg => { + let sql = format!( + "{} SELECT epoch_ms(bucket_ts) AS timestamp_ms, AVG(value) AS value {}", + duckdb_bucketed_cte("integer_values", step_ms), + duckdb_group_by_clause() + ); + let mut statement = connection.prepare(&sql)?; + let mut rows = statement.query(duckdb::params![ + sensor_id, + start_time_ms, + end_time_ms, + origin_ms, + limit + ])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + let value: f64 = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64( + timestamp_ms, + ), + value, + }); + } + Ok(TypedSamples::Float(samples)) + } + Aggregation::Count => { + let sql = format!( + "{} SELECT epoch_ms(bucket_ts) AS timestamp_ms, COUNT(*) AS value {}", + duckdb_bucketed_cte("integer_values", step_ms), + duckdb_group_by_clause() + ); + let mut statement = connection.prepare(&sql)?; + let mut rows = statement.query(duckdb::params![ + sensor_id, + start_time_ms, + end_time_ms, + origin_ms, + limit + ])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + let value: i64 = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64( + timestamp_ms, + ), + value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + Aggregation::First | Aggregation::Last => { + let sql = duckdb_first_last_query("integer_values", step_ms, aggregation, "value"); + let mut statement = connection.prepare(&sql)?; + let mut rows = statement.query(duckdb::params![ + sensor_id, + start_time_ms, + end_time_ms, + origin_ms, + limit + ])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + let value: i64 = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64( + timestamp_ms, + ), + value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + _ => { + let expression = match aggregation { + Aggregation::Min => "MIN(value)", + Aggregation::Max => "MAX(value)", + Aggregation::Sum => "SUM(value)", + _ => unreachable!("handled separately"), + }; + let sql = format!( + "{} SELECT epoch_ms(bucket_ts) AS timestamp_ms, {} AS value {}", + duckdb_bucketed_cte("integer_values", step_ms), + expression, + duckdb_group_by_clause() + ); + let mut statement = connection.prepare(&sql)?; + let mut rows = statement.query(duckdb::params![ + sensor_id, + start_time_ms, + end_time_ms, + origin_ms, + limit + ])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + let value: i64 = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64( + timestamp_ms, + ), + value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + } +} + +#[allow(clippy::too_many_arguments)] +fn duckdb_query_float_samples_aggregated( + connection: &Connection, + sensor_id: i64, + start_time_ms: Option, + end_time_ms: Option, + step_ms: i64, + origin_ms: i64, + aggregation: Aggregation, + limit: Option, +) -> Result { + let limit = limit.unwrap_or(super::DEFAULT_QUERY_LIMIT) as i64; + + match aggregation { + Aggregation::Count => { + let sql = format!( + "{} SELECT epoch_ms(bucket_ts) AS timestamp_ms, COUNT(*) AS value {}", + duckdb_bucketed_cte("float_values", step_ms), + duckdb_group_by_clause() + ); + let mut statement = connection.prepare(&sql)?; + let mut rows = statement.query(duckdb::params![ + sensor_id, + start_time_ms, + end_time_ms, + origin_ms, + limit + ])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + let value: i64 = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64( + timestamp_ms, + ), + value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + Aggregation::First | Aggregation::Last => { + let sql = duckdb_first_last_query("float_values", step_ms, aggregation, "value"); + let mut statement = connection.prepare(&sql)?; + let mut rows = statement.query(duckdb::params![ + sensor_id, + start_time_ms, + end_time_ms, + origin_ms, + limit + ])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + let value: f64 = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64( + timestamp_ms, + ), + value, + }); + } + Ok(TypedSamples::Float(samples)) + } + _ => { + let expression = match aggregation { + Aggregation::Avg => "AVG(value)", + Aggregation::Min => "MIN(value)", + Aggregation::Max => "MAX(value)", + Aggregation::Sum => "SUM(value)", + _ => unreachable!("handled separately"), + }; + let sql = format!( + "{} SELECT epoch_ms(bucket_ts) AS timestamp_ms, {} AS value {}", + duckdb_bucketed_cte("float_values", step_ms), + expression, + duckdb_group_by_clause() + ); + let mut statement = connection.prepare(&sql)?; + let mut rows = statement.query(duckdb::params![ + sensor_id, + start_time_ms, + end_time_ms, + origin_ms, + limit + ])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + let value: f64 = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64( + timestamp_ms, + ), + value, + }); + } + Ok(TypedSamples::Float(samples)) + } + } +} + +#[allow(clippy::too_many_arguments)] +fn duckdb_query_numeric_samples_aggregated( + connection: &Connection, + sensor_id: i64, + start_time_ms: Option, + end_time_ms: Option, + step_ms: i64, + origin_ms: i64, + aggregation: Aggregation, + limit: Option, +) -> Result { + let limit = limit.unwrap_or(super::DEFAULT_QUERY_LIMIT) as i64; + + match aggregation { + Aggregation::Count => { + let sql = format!( + "{} SELECT epoch_ms(bucket_ts) AS timestamp_ms, COUNT(*) AS value {}", + duckdb_bucketed_cte("numeric_values", step_ms), + duckdb_group_by_clause() + ); + let mut statement = connection.prepare(&sql)?; + let mut rows = statement.query(duckdb::params![ + sensor_id, + start_time_ms, + end_time_ms, + origin_ms, + limit + ])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + let value: i64 = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64( + timestamp_ms, + ), + value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + Aggregation::First | Aggregation::Last => { + let sql = duckdb_first_last_query( + "numeric_values", + step_ms, + aggregation, + "CAST(value AS VARCHAR)", + ); + let mut statement = connection.prepare(&sql)?; + let mut rows = statement.query(duckdb::params![ + sensor_id, + start_time_ms, + end_time_ms, + step_ms, + origin_ms, + limit + ])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + let value: String = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64( + timestamp_ms, + ), + value: Decimal::from_str(&value) + .context("Failed to parse DuckDB aggregated numeric value")?, + }); + } + Ok(TypedSamples::Numeric(samples)) + } + _ => { + let expression = match aggregation { + Aggregation::Avg => "CAST(AVG(value) AS VARCHAR)", + Aggregation::Min => "CAST(MIN(value) AS VARCHAR)", + Aggregation::Max => "CAST(MAX(value) AS VARCHAR)", + Aggregation::Sum => "CAST(SUM(value) AS VARCHAR)", + _ => unreachable!("handled separately"), + }; + let sql = format!( + "{} SELECT epoch_ms(bucket_ts) AS timestamp_ms, {} AS value {}", + duckdb_bucketed_cte("numeric_values", step_ms), + expression, + duckdb_group_by_clause() + ); + let mut statement = connection.prepare(&sql)?; + let mut rows = statement.query(duckdb::params![ + sensor_id, + start_time_ms, + end_time_ms, + step_ms, + origin_ms, + limit + ])?; + let mut samples = smallvec![]; + while let Some(row) = rows.next()? { + let timestamp_ms: i64 = row.get(0)?; + let value: String = row.get(1)?; + samples.push(Sample { + datetime: crate::datamodel::SensAppDateTime::from_unix_milliseconds_i64( + timestamp_ms, + ), + value: Decimal::from_str(&value) + .context("Failed to parse DuckDB aggregated numeric value")?, + }); + } + Ok(TypedSamples::Numeric(samples)) + } + } +} diff --git a/src/storage/error.rs b/src/storage/error.rs index 0d498812..6935db5f 100644 --- a/src/storage/error.rs +++ b/src/storage/error.rs @@ -20,6 +20,7 @@ pub enum StorageError { /// Invalid data format in database #[error("Invalid data format: {message} for sensor {sensor_context}")] + #[allow(dead_code)] // Part of the shared storage error API across backend feature sets InvalidDataFormat { message: String, sensor_context: String, @@ -73,6 +74,7 @@ impl StorageError { } /// Create an invalid data format error with sensor context + #[allow(dead_code)] // Part of the shared storage error API across backend feature sets pub fn invalid_data_format( message: &str, sensor_uuid: Option, diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 051b1e07..570dabc0 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,5 +1,5 @@ // Core storage traits and factory - always available -use crate::datamodel::SensAppDateTime; +use crate::datamodel::{SensAppDateTime, Sensor}; use anyhow::Result; use async_trait::async_trait; use std::fmt::Debug; @@ -8,14 +8,43 @@ pub mod error; pub use error::StorageError; pub mod common; +pub mod data_query; pub mod query; +pub use data_query::{Aggregation, SensorDataQueryOptions, SimplifyOptions}; +#[allow(unused_imports)] pub use query::{LabelMatcher, MatcherType}; /// Default limit for timeseries queries when no limit is specified /// Set to 10 million records - appropriate for timeseries data +#[allow(dead_code)] pub const DEFAULT_QUERY_LIMIT: usize = 10_000_000; +/// Default limit for list_series when no limit is specified +pub const DEFAULT_LIST_SERIES_LIMIT: usize = 256; + +/// Maximum limit for list_series to prevent excessive memory usage +pub const MAX_LIST_SERIES_LIMIT: usize = 16384; + +/// Result type for list_series with pagination support +#[derive(Debug)] +pub struct ListSeriesResult { + /// The series matching the query + pub series: Vec, + /// Bookmark for the next page (sensor_id as string) + /// None if this is the last page + pub bookmark: Option, +} + +#[derive(Debug)] +pub struct SensorAvailabilitySummary { + pub sensor: Sensor, + pub sample_count: usize, + pub first_sample_at: Option, + pub last_sample_at: Option, + pub covered_buckets: Option, +} + #[async_trait] pub trait StorageInstance: Send + Sync + Debug { async fn create_or_migrate(&self) -> Result<()>; @@ -26,7 +55,9 @@ pub trait StorageInstance: Send + Sync + Debug { async fn list_series( &self, metric_filter: Option<&str>, - ) -> Result>; + limit: Option, + bookmark: Option<&str>, + ) -> Result; async fn list_metrics(&self) -> Result>; @@ -39,6 +70,60 @@ pub trait StorageInstance: Send + Sync + Debug { limit: Option, ) -> Result>; + async fn query_sensor_data_advanced( + &self, + sensor_uuid: &str, + options: &crate::storage::SensorDataQueryOptions, + ) -> Result> { + options.validate()?; + + let raw = self + .query_sensor_data( + sensor_uuid, + options.start_time, + options.end_time, + options.limit, + ) + .await?; + + raw.map(|sensor_data| crate::storage::common::apply_query_options(sensor_data, options)) + .transpose() + } + + async fn query_sensor_data_latest( + &self, + sensor_uuid: &str, + start_time: Option, + end_time: Option, + ) -> Result> { + let Some(sensor_data) = self + .query_sensor_data(sensor_uuid, start_time, end_time, None) + .await? + else { + return Ok(None); + }; + + Ok(crate::storage::common::keep_only_last_sample(sensor_data)) + } + + async fn query_sensor_data_availability( + &self, + sensor_uuid: &str, + start_time: SensAppDateTime, + end_time: SensAppDateTime, + step_ms: Option, + ) -> Result> { + let Some(sensor_data) = self + .query_sensor_data(sensor_uuid, Some(start_time), Some(end_time), None) + .await? + else { + return Ok(None); + }; + + crate::storage::common::summarize_sensor_data_availability(sensor_data, start_time, step_ms) + .map(Some) + } + /// Query sensors and their data by label matchers. /// /// This method finds all sensors matching the given label matchers and returns diff --git a/src/storage/postgresql/migrations/20250819143000_add_sensor_catalog_view.sql b/src/storage/postgresql/migrations/20250819143000_add_sensor_catalog_view.sql index 12cf6ca1..843632c7 100644 --- a/src/storage/postgresql/migrations/20250819143000_add_sensor_catalog_view.sql +++ b/src/storage/postgresql/migrations/20250819143000_add_sensor_catalog_view.sql @@ -6,6 +6,15 @@ SELECT s.name, s.type, u.name as unit_name, - u.description as unit_description + u.description as unit_description, + label_data.labels FROM sensors s -LEFT JOIN units u ON s.unit = u.id; +LEFT JOIN units u ON s.unit = u.id +LEFT JOIN LATERAL ( + SELECT jsonb_object_agg(lnd.name, ldd.description) as labels + FROM labels l + JOIN labels_name_dictionary lnd ON l."name" = lnd.id + JOIN labels_description_dictionary ldd ON l.description = ldd.id + WHERE l.sensor_id = s.sensor_id +) label_data ON true +ORDER BY s.sensor_id; diff --git a/src/storage/postgresql/migrations/20250819144000_add_metrics_summary_view.sql b/src/storage/postgresql/migrations/20250819144000_add_metrics_summary_view.sql index 6da1fbb0..8bb39482 100644 --- a/src/storage/postgresql/migrations/20250819144000_add_metrics_summary_view.sql +++ b/src/storage/postgresql/migrations/20250819144000_add_metrics_summary_view.sql @@ -11,4 +11,5 @@ FROM sensors s LEFT JOIN units u ON s.unit = u.id LEFT JOIN labels l ON s.sensor_id = l.sensor_id LEFT JOIN labels_name_dictionary lnd ON l.name = lnd.id -GROUP BY s.name, s.type; +GROUP BY s.name, s.type +ORDER BY s.name; diff --git a/src/storage/postgresql/mod.rs b/src/storage/postgresql/mod.rs index 0ecef107..458e3d89 100644 --- a/src/storage/postgresql/mod.rs +++ b/src/storage/postgresql/mod.rs @@ -8,7 +8,11 @@ //! - `postgresql_publishers.rs`: Value publishing functions //! - `postgresql_utilities.rs`: Helper functions for sensor/label/unit creation -use super::{DEFAULT_QUERY_LIMIT, StorageError, StorageInstance, common::datetime_to_micros}; +use super::{ + DEFAULT_QUERY_LIMIT, SensorAvailabilitySummary, StorageError, StorageInstance, + common::datetime_to_micros, +}; +use crate::datamodel::sensapp_datetime::SensAppDateTimeExt; use crate::datamodel::{ Metric, SensAppDateTime, Sensor, SensorData, SensorType, TypedSamples, batch::Batch, }; @@ -16,7 +20,11 @@ use crate::datamodel::{sensapp_vec::SensAppLabels, unit::Unit}; use anyhow::{Context, Result}; use async_trait::async_trait; use smallvec::smallvec; -use sqlx::{PgPool, postgres::PgConnectOptions}; +use sqlx::{ + PgPool, + postgres::{PgConnectOptions, PgPoolOptions}, +}; +use std::collections::HashMap; use std::{str::FromStr, sync::Arc}; use uuid::Uuid; @@ -41,12 +49,236 @@ impl PostgresStorage { let connect_options = PgConnectOptions::from_str(connection_string) .context("Failed to create postgres connection options")?; - let pool = PgPool::connect_with(connect_options) + let max_connections = std::env::var("SENSAPP_PG_POOL_MAX_CONNECTIONS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(10); + + let pool = PgPoolOptions::new() + .max_connections(max_connections) + .connect_with(connect_options) .await .context("Failed to create postgres pool")?; Ok(Self { pool }) } + + async fn get_sensor_metadata(&self, sensor_uuid: &str) -> Result> { + let parsed_uuid = Uuid::from_str(sensor_uuid).context("Failed to parse sensor UUID")?; + + #[derive(sqlx::FromRow)] + struct SensorRow { + sensor_id: Option, + uuid: Option, + name: Option, + r#type: Option, + unit_name: Option, + unit_description: Option, + } + + let sensor_row = sqlx::query_as::<_, SensorRow>( + r#" + SELECT s.sensor_id AS sensor_id, s.uuid, s.name, s.type, u.name AS unit_name, u.description AS unit_description + FROM sensors s + LEFT JOIN units u ON s.unit = u.id + WHERE s.uuid = $1 + "#, + ) + .bind(parsed_uuid) + .fetch_optional(&self.pool) + .await?; + + let Some(sensor_row) = sensor_row else { + return Ok(None); + }; + + let sensor_uuid = sensor_row.uuid.ok_or_else(|| { + anyhow::Error::from(StorageError::missing_field( + "UUID", + None, + sensor_row.name.as_deref(), + )) + })?; + + let sensor_name = sensor_row.name.ok_or_else(|| { + anyhow::Error::from(StorageError::missing_field("name", Some(sensor_uuid), None)) + })?; + + let sensor_type_str = sensor_row.r#type.ok_or_else(|| { + anyhow::Error::from(StorageError::missing_field( + "type", + Some(sensor_uuid), + Some(&sensor_name), + )) + })?; + + let sensor_type = SensorType::from_str(&sensor_type_str).map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Failed to parse sensor type '{}': {}", sensor_type_str, e), + Some(sensor_uuid), + Some(&sensor_name), + )) + })?; + + let unit = match (sensor_row.unit_name, sensor_row.unit_description) { + (Some(name), description) => Some(Unit::new(name, description)), + _ => None, + }; + + let sensor_id = sensor_row.sensor_id.ok_or_else(|| { + anyhow::Error::from(StorageError::missing_field( + "sensor_id", + Some(sensor_uuid), + Some(&sensor_name), + )) + })?; + + #[derive(sqlx::FromRow)] + struct LabelRow { + label_name: String, + label_value: String, + } + + let labels_rows: Vec = sqlx::query_as( + r#" + SELECT lnd.name as label_name, ldd.description as label_value + FROM labels l + JOIN labels_name_dictionary lnd ON l.name = lnd.id + JOIN labels_description_dictionary ldd ON l.description = ldd.id + WHERE l.sensor_id = $1 + "#, + ) + .bind(sensor_id) + .fetch_all(&self.pool) + .await?; + + let mut labels: SensAppLabels = smallvec![]; + for label_row in labels_rows { + labels.push((label_row.label_name, label_row.label_value)); + } + + let sensor = Sensor::new(sensor_uuid, sensor_name, sensor_type, unit, Some(labels)); + + Ok(Some((sensor_id, sensor))) + } + + fn sensor_table_name(sensor_type: SensorType) -> &'static str { + match sensor_type { + SensorType::Integer => "integer_values", + SensorType::Numeric => "numeric_values", + SensorType::Float => "float_values", + SensorType::String => "string_values", + SensorType::Boolean => "boolean_values", + SensorType::Location => "location_values", + SensorType::Json => "json_values", + SensorType::Blob => "blob_values", + } + } + + async fn query_latest_timestamp_us( + &self, + table_name: &str, + sensor_id: i64, + start_time: Option, + end_time: Option, + ) -> Result> { + let sql = format!( + r#" + SELECT MAX(timestamp_us) AS timestamp_us + FROM {table_name} + WHERE sensor_id = $1 + AND ($2::BIGINT IS NULL OR timestamp_us >= $2) + AND ($3::BIGINT IS NULL OR timestamp_us <= $3) + "# + ); + + let timestamp_us: Option = sqlx::query_scalar(&sql) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .fetch_one(&self.pool) + .await?; + + Ok(timestamp_us) + } + + async fn query_availability_summary_native( + &self, + table_name: &str, + sensor_id: i64, + sensor: Sensor, + start_time: i64, + end_time: i64, + step_ms: Option, + ) -> Result { + #[derive(sqlx::FromRow)] + struct Row { + sample_count: i64, + first_sample_at: Option, + last_sample_at: Option, + covered_buckets: Option, + } + + let row: Row = if let Some(step_ms) = step_ms { + let step_us = step_ms.checked_mul(1000).context("step is too large")?; + let sql = format!( + r#" + SELECT + COUNT(*)::bigint AS sample_count, + MIN(timestamp_us) AS first_sample_at, + MAX(timestamp_us) AS last_sample_at, + COUNT(DISTINCT ((timestamp_us - $1) / $2))::bigint AS covered_buckets + FROM {table_name} + WHERE sensor_id = $3 + AND timestamp_us >= $4 + AND timestamp_us <= $5 + "# + ); + + sqlx::query_as(&sql) + .bind(start_time) + .bind(step_us) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .fetch_one(&self.pool) + .await? + } else { + let sql = format!( + r#" + SELECT + COUNT(*)::bigint AS sample_count, + MIN(timestamp_us) AS first_sample_at, + MAX(timestamp_us) AS last_sample_at, + NULL::bigint AS covered_buckets + FROM {table_name} + WHERE sensor_id = $1 + AND timestamp_us >= $2 + AND timestamp_us <= $3 + "# + ); + + sqlx::query_as(&sql) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .fetch_one(&self.pool) + .await? + }; + + Ok(SensorAvailabilitySummary { + sensor, + sample_count: row.sample_count.max(0) as usize, + first_sample_at: row + .first_sample_at + .map(SensAppDateTime::from_unix_microseconds_i64), + last_sample_at: row + .last_sample_at + .map(SensAppDateTime::from_unix_microseconds_i64), + covered_buckets: row.covered_buckets.map(|value| value.max(0) as usize), + }) + } } #[async_trait] @@ -81,7 +313,9 @@ impl StorageInstance for PostgresStorage { async fn list_series( &self, metric_filter: Option<&str>, - ) -> Result> { + limit: Option, + bookmark: Option<&str>, + ) -> Result { #[derive(sqlx::FromRow)] struct SensorRow { sensor_id: Option, @@ -90,25 +324,54 @@ impl StorageInstance for PostgresStorage { r#type: Option, unit_name: Option, unit_description: Option, + labels: Option>>, } + // Parse bookmark as sensor_id + let bookmark_id: Option = if let Some(bookmark_str) = bookmark { + Some(bookmark_str.parse::().map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Invalid bookmark format: {}", e), + None, + None, + )) + })?) + } else { + None + }; + + // Validate and apply limit + let effective_limit = limit + .unwrap_or(crate::storage::DEFAULT_LIST_SERIES_LIMIT) + .min(crate::storage::MAX_LIST_SERIES_LIMIT); + let fetch_limit = effective_limit.saturating_add(1); + // Query sensors with their metadata using the catalog view, optionally filtered by metric name + // Use bookmark for cursor-based pagination (sensor_id > bookmark) let sensor_rows: Vec = sqlx::query_as( r#" - SELECT sensor_id, uuid, name, type, unit_name, unit_description + SELECT sensor_id, uuid, name, type, unit_name, unit_description, labels FROM sensor_catalog_view WHERE ($1::TEXT IS NULL OR name = $1) - ORDER BY uuid ASC + AND ($2::BIGINT IS NULL OR sensor_id > $2) + ORDER BY sensor_id ASC + LIMIT $3 "#, ) .bind(metric_filter) + .bind(bookmark_id) + .bind(fetch_limit as i64) .fetch_all(&self.pool) .await?; + let has_more = sensor_rows.len() > effective_limit; let mut sensors = Vec::new(); + let mut last_sensor_id: Option = None; + + for sensor_row in sensor_rows.into_iter().take(effective_limit) { + // Keep track of the last sensor_id for bookmark + last_sensor_id = sensor_row.sensor_id; - for sensor_row in sensor_rows { - // Parse sensor metadata with improved error handling let sensor_uuid = sensor_row .uuid .ok_or_else(|| { @@ -141,51 +404,34 @@ impl StorageInstance for PostgresStorage { _ => None, }; - // Query labels for this sensor with proper error context - let sensor_id = sensor_row.sensor_id.ok_or_else(|| { - anyhow::Error::from(StorageError::missing_field( - "sensor_id", - Some(sensor_uuid), - Some(&sensor_name), - )) - })?; - - #[derive(sqlx::FromRow)] - struct LabelRow { - label_name: String, - label_value: String, - } + let labels: Option = if let Some(labels_json) = sensor_row.labels { + let mut labels: SensAppLabels = smallvec![]; + for (label_name, label_value) in labels_json.0 { + labels.push((label_name, label_value)); + } - let labels_rows: Vec = sqlx::query_as( - r#" - SELECT lnd.name as label_name, ldd.description as label_value - FROM labels l - JOIN labels_name_dictionary lnd ON l.name = lnd.id - JOIN labels_description_dictionary ldd ON l.description = ldd.id - WHERE l.sensor_id = $1 - "#, - ) - .bind(sensor_id) - .fetch_all(&self.pool) - .await - .with_context(|| { - format!( - "Failed to query labels for sensor UUID={} name='{}'", - sensor_uuid, sensor_name - ) - })?; - - let mut labels: SensAppLabels = smallvec![]; - for label_row in labels_rows { - labels.push((label_row.label_name, label_row.label_value)); - } + Some(labels) + } else { + None + }; - let sensor = Sensor::new(sensor_uuid, sensor_name, sensor_type, unit, Some(labels)); + let sensor = Sensor::new(sensor_uuid, sensor_name, sensor_type, unit, labels); sensors.push(sensor); } - Ok(sensors) + // Determine if there's a next page based on whether we got the full limit + // If we got exactly the limit, there might be more pages, so return the last sensor_id as bookmark + let next_bookmark = if has_more { + last_sensor_id.map(|id| id.to_string()) + } else { + None + }; + + Ok(crate::storage::ListSeriesResult { + series: sensors, + bookmark: next_bookmark, + }) } async fn list_metrics(&self) -> Result> { @@ -264,105 +510,10 @@ impl StorageInstance for PostgresStorage { end_time: Option, limit: Option, ) -> Result> { - // Parse UUID - let parsed_uuid = Uuid::from_str(sensor_uuid).context("Failed to parse sensor UUID")?; - - #[derive(sqlx::FromRow)] - struct SensorMetadataRow { - sensor_id: Option, - uuid: Option, - name: Option, - r#type: Option, - unit_name: Option, - unit_description: Option, - } - - // Query sensor metadata by UUID using the catalog view - let sensor_row: Option = sqlx::query_as( - r#" - SELECT sensor_id, uuid, name, type, unit_name, unit_description - FROM sensor_catalog_view - WHERE uuid = $1 - "#, - ) - .bind(parsed_uuid) - .fetch_optional(&self.pool) - .await?; - - let sensor_row = match sensor_row { - Some(row) => row, - None => return Ok(None), - }; - - // Parse sensor metadata with improved error handling - let sensor_uuid = sensor_row.uuid.ok_or_else(|| { - anyhow::Error::from(StorageError::missing_field( - "UUID", - None, - sensor_row.name.as_deref(), - )) - })?; - - let sensor_name = sensor_row.name.ok_or_else(|| { - anyhow::Error::from(StorageError::missing_field("name", Some(sensor_uuid), None)) - })?; - - let sensor_type_str = sensor_row.r#type.ok_or_else(|| { - anyhow::Error::from(StorageError::missing_field( - "type", - Some(sensor_uuid), - Some(&sensor_name), - )) - })?; - - let sensor_type = SensorType::from_str(&sensor_type_str).map_err(|e| { - anyhow::Error::from(StorageError::invalid_data_format( - &format!("Failed to parse sensor type '{}': {}", sensor_type_str, e), - Some(sensor_uuid), - Some(&sensor_name), - )) - })?; - - let unit = match (sensor_row.unit_name, sensor_row.unit_description) { - (Some(name), description) => Some(Unit::new(name, description)), - _ => None, + let Some((sensor_id, sensor)) = self.get_sensor_metadata(sensor_uuid).await? else { + return Ok(None); }; - // Query labels for this sensor with proper context - let sensor_id = sensor_row.sensor_id.ok_or_else(|| { - anyhow::Error::from(StorageError::missing_field( - "sensor_id", - Some(sensor_uuid), - Some(&sensor_name), - )) - })?; - #[derive(sqlx::FromRow)] - struct LabelRow { - label_name: String, - label_value: String, - } - - let labels_rows: Vec = sqlx::query_as( - r#" - SELECT lnd.name as label_name, ldd.description as label_value - FROM labels l - JOIN labels_name_dictionary lnd ON l.name = lnd.id - JOIN labels_description_dictionary ldd ON l.description = ldd.id - WHERE l.sensor_id = $1 - "#, - ) - .bind(sensor_id) - .fetch_all(&self.pool) - .await?; - - let mut labels: SensAppLabels = smallvec![]; - for label_row in labels_rows { - labels.push((label_row.label_name, label_row.label_value)); - } - - let sensor = Sensor::new(sensor_uuid, sensor_name, sensor_type, unit, Some(labels)); - - // Convert SensAppDateTime to microseconds for database queries using common utility let start_time_us = start_time.as_ref().map(datetime_to_micros); let end_time_us = end_time.as_ref().map(datetime_to_micros); @@ -405,6 +556,237 @@ impl StorageInstance for PostgresStorage { Ok(Some(SensorData::new(sensor, samples))) } + async fn query_sensor_data_advanced( + &self, + sensor_uuid: &str, + options: &crate::storage::SensorDataQueryOptions, + ) -> Result> { + options.validate()?; + + if let (Some(step_ms), Some(aggregation)) = (options.step_ms, options.aggregation) { + if let Some((sensor_id, mut sensor)) = self.get_sensor_metadata(sensor_uuid).await? { + let start_time_us = options.start_time.as_ref().map(datetime_to_micros); + let end_time_us = options.end_time.as_ref().map(datetime_to_micros); + let origin_us = start_time_us.unwrap_or(0); + + let samples = match sensor.sensor_type { + SensorType::Integer => { + self.query_integer_samples_aggregated( + sensor_id, + start_time_us, + end_time_us, + step_ms, + origin_us, + aggregation, + options.limit, + ) + .await? + } + SensorType::Numeric => { + self.query_numeric_samples_aggregated( + sensor_id, + start_time_us, + end_time_us, + step_ms, + origin_us, + aggregation, + options.limit, + ) + .await? + } + SensorType::Float => { + self.query_float_samples_aggregated( + sensor_id, + start_time_us, + end_time_us, + step_ms, + origin_us, + aggregation, + options.limit, + ) + .await? + } + _ => { + let raw = self + .query_sensor_data( + sensor_uuid, + options.start_time, + options.end_time, + options.limit, + ) + .await?; + return raw + .map(|sensor_data| { + crate::storage::common::apply_query_options(sensor_data, options) + }) + .transpose(); + } + }; + + sensor.sensor_type = match &samples { + TypedSamples::Integer(_) => SensorType::Integer, + TypedSamples::Numeric(_) => SensorType::Numeric, + TypedSamples::Float(_) => SensorType::Float, + _ => sensor.sensor_type, + }; + if aggregation.output_is_count() { + sensor.unit = None; + } + + let query_options = crate::storage::SensorDataQueryOptions { + start_time: options.start_time, + end_time: options.end_time, + limit: options.limit, + step_ms: None, + aggregation: None, + simplify: options.simplify, + }; + + let sensor_data = SensorData::new(sensor, samples); + return crate::storage::common::apply_query_options(sensor_data, &query_options) + .map(Some); + } + + return Ok(None); + } + + let raw = self + .query_sensor_data( + sensor_uuid, + options.start_time, + options.end_time, + options.limit, + ) + .await?; + + raw.map(|sensor_data| crate::storage::common::apply_query_options(sensor_data, options)) + .transpose() + } + + async fn query_sensor_data_latest( + &self, + sensor_uuid: &str, + start_time: Option, + end_time: Option, + ) -> Result> { + let Some((sensor_id, sensor)) = self.get_sensor_metadata(sensor_uuid).await? else { + return Ok(None); + }; + + let start_time_us = start_time.as_ref().map(datetime_to_micros); + let end_time_us = end_time.as_ref().map(datetime_to_micros); + let table_name = Self::sensor_table_name(sensor.sensor_type); + + let Some(latest_timestamp_us) = self + .query_latest_timestamp_us(table_name, sensor_id, start_time_us, end_time_us) + .await? + else { + return Ok(None); + }; + + let samples = match sensor.sensor_type { + SensorType::Integer => { + self.query_integer_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Numeric => { + self.query_numeric_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Float => { + self.query_float_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::String => { + self.query_string_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Boolean => { + self.query_boolean_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Location => { + self.query_location_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Json => { + self.query_json_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Blob => { + self.query_blob_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + }; + + Ok(Some(SensorData::new(sensor, samples))) + } + + async fn query_sensor_data_availability( + &self, + sensor_uuid: &str, + start_time: SensAppDateTime, + end_time: SensAppDateTime, + step_ms: Option, + ) -> Result> { + let Some((sensor_id, sensor)) = self.get_sensor_metadata(sensor_uuid).await? else { + return Ok(None); + }; + + let summary = self + .query_availability_summary_native( + Self::sensor_table_name(sensor.sensor_type), + sensor_id, + sensor, + datetime_to_micros(&start_time), + datetime_to_micros(&end_time), + step_ms, + ) + .await?; + + Ok(Some(summary)) + } + async fn query_sensors_by_labels( &self, matchers: &[super::LabelMatcher], diff --git a/src/storage/postgresql/postgresql_utilities.rs b/src/storage/postgresql/postgresql_utilities.rs index 7fbb5a4a..00902b51 100644 --- a/src/storage/postgresql/postgresql_utilities.rs +++ b/src/storage/postgresql/postgresql_utilities.rs @@ -3,7 +3,6 @@ use crate::datamodel::unit::Unit; use anyhow::Result; use cached::proc_macro::cached; use sqlx::{Executor, Postgres, Row, Transaction}; -use std::time::Duration; use uuid::Uuid; #[cached( diff --git a/src/storage/postgresql/queries.rs b/src/storage/postgresql/queries.rs index 41e1436c..ea3d592e 100644 --- a/src/storage/postgresql/queries.rs +++ b/src/storage/postgresql/queries.rs @@ -6,12 +6,298 @@ use super::{DEFAULT_QUERY_LIMIT, PostgresStorage}; use crate::datamodel::sensapp_datetime::SensAppDateTimeExt; use crate::datamodel::{Sample, SensAppDateTime, TypedSamples}; +use crate::storage::Aggregation; use anyhow::Result; use geo::Point; use serde_json::Value as JsonValue; use smallvec::smallvec; impl PostgresStorage { + #[allow(clippy::too_many_arguments)] + pub(super) async fn query_integer_samples_aggregated( + &self, + sensor_id: i64, + start_time: Option, + end_time: Option, + step_ms: i64, + origin_us: i64, + aggregation: Aggregation, + limit: Option, + ) -> Result { + let limit = limit.unwrap_or(DEFAULT_QUERY_LIMIT) as i64; + + match aggregation { + Aggregation::Avg => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: f64, + } + + let sql = format!( + "{} {} {}", + integer_bucketed_cte(), + "SELECT bucket_us AS timestamp_us, AVG(value)::double precision AS value", + integer_group_by_clause(limit, "bucket_us") + ); + + let rows: Vec = sqlx::query_as(&sql) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_ms) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Float(samples)) + } + Aggregation::Count => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: i64, + } + + let sql = format!( + "{} {} {}", + integer_bucketed_cte(), + "SELECT bucket_us AS timestamp_us, COUNT(*)::bigint AS value", + integer_group_by_clause(limit, "bucket_us") + ); + + let rows: Vec = sqlx::query_as(&sql) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_ms) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + _ => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: i64, + } + + let expression = integer_aggregate_expression(aggregation); + let sql = format!( + "{} SELECT bucket_us AS timestamp_us, {} AS value {}", + integer_bucketed_cte(), + expression, + integer_group_by_clause(limit, "bucket_us") + ); + + let rows: Vec = sqlx::query_as(&sql) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_ms) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + } + } + + #[allow(clippy::too_many_arguments)] + pub(super) async fn query_float_samples_aggregated( + &self, + sensor_id: i64, + start_time: Option, + end_time: Option, + step_ms: i64, + origin_us: i64, + aggregation: Aggregation, + limit: Option, + ) -> Result { + let limit = limit.unwrap_or(DEFAULT_QUERY_LIMIT) as i64; + + match aggregation { + Aggregation::Count => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: i64, + } + + let sql = format!( + "{} {} {}", + float_bucketed_cte(), + "SELECT bucket_us AS timestamp_us, COUNT(*)::bigint AS value", + integer_group_by_clause(limit, "bucket_us") + ); + + let rows: Vec = sqlx::query_as(&sql) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_ms) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + _ => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: f64, + } + + let expression = float_aggregate_expression(aggregation); + let sql = format!( + "{} SELECT bucket_us AS timestamp_us, {} AS value {}", + float_bucketed_cte(), + expression, + integer_group_by_clause(limit, "bucket_us") + ); + + let rows: Vec = sqlx::query_as(&sql) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_ms) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Float(samples)) + } + } + } + + #[allow(clippy::too_many_arguments)] + pub(super) async fn query_numeric_samples_aggregated( + &self, + sensor_id: i64, + start_time: Option, + end_time: Option, + step_ms: i64, + origin_us: i64, + aggregation: Aggregation, + limit: Option, + ) -> Result { + let limit = limit.unwrap_or(DEFAULT_QUERY_LIMIT) as i64; + + match aggregation { + Aggregation::Count => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: i64, + } + + let sql = format!( + "{} {} {}", + numeric_bucketed_cte(), + "SELECT bucket_us AS timestamp_us, COUNT(*)::bigint AS value", + integer_group_by_clause(limit, "bucket_us") + ); + + let rows: Vec = sqlx::query_as(&sql) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_ms) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + _ => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: rust_decimal::Decimal, + } + + let expression = numeric_aggregate_expression(aggregation); + let sql = format!( + "{} SELECT bucket_us AS timestamp_us, {} AS value {}", + numeric_bucketed_cte(), + expression, + integer_group_by_clause(limit, "bucket_us") + ); + + let rows: Vec = sqlx::query_as(&sql) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_ms) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Numeric(samples)) + } + } + } + pub(super) async fn query_integer_samples( &self, sensor_id: i64, @@ -335,3 +621,90 @@ impl PostgresStorage { Ok(TypedSamples::Blob(samples)) } } + +fn integer_bucketed_cte() -> &'static str { + r#" + WITH bucketed AS ( + SELECT + (EXTRACT(EPOCH FROM date_bin(($4::bigint * interval '1 millisecond'), to_timestamp(timestamp_us / 1000000.0), to_timestamp($5 / 1000000.0))) * 1000000)::bigint AS bucket_us, + timestamp_us, + value + FROM integer_values + WHERE sensor_id = $1 + AND ($2::BIGINT IS NULL OR timestamp_us >= $2) + AND ($3::BIGINT IS NULL OR timestamp_us <= $3) + ) + "# +} + +fn float_bucketed_cte() -> &'static str { + r#" + WITH bucketed AS ( + SELECT + (EXTRACT(EPOCH FROM date_bin(($4::bigint * interval '1 millisecond'), to_timestamp(timestamp_us / 1000000.0), to_timestamp($5 / 1000000.0))) * 1000000)::bigint AS bucket_us, + timestamp_us, + value + FROM float_values + WHERE sensor_id = $1 + AND ($2::BIGINT IS NULL OR timestamp_us >= $2) + AND ($3::BIGINT IS NULL OR timestamp_us <= $3) + ) + "# +} + +fn numeric_bucketed_cte() -> &'static str { + r#" + WITH bucketed AS ( + SELECT + (EXTRACT(EPOCH FROM date_bin(($4::bigint * interval '1 millisecond'), to_timestamp(timestamp_us / 1000000.0), to_timestamp($5 / 1000000.0))) * 1000000)::bigint AS bucket_us, + timestamp_us, + value + FROM numeric_values + WHERE sensor_id = $1 + AND ($2::BIGINT IS NULL OR timestamp_us >= $2) + AND ($3::BIGINT IS NULL OR timestamp_us <= $3) + ) + "# +} + +fn integer_group_by_clause(limit: i64, bucket_column: &str) -> String { + format!( + "FROM bucketed GROUP BY {} ORDER BY {} ASC LIMIT {}", + bucket_column, bucket_column, limit + ) +} + +fn integer_aggregate_expression(aggregation: Aggregation) -> &'static str { + match aggregation { + Aggregation::Min => "MIN(value)", + Aggregation::Max => "MAX(value)", + Aggregation::Sum => "SUM(value)::bigint", + Aggregation::First => "(array_agg(value ORDER BY timestamp_us ASC))[1]", + Aggregation::Last => "(array_agg(value ORDER BY timestamp_us DESC))[1]", + Aggregation::Avg | Aggregation::Count => unreachable!("handled separately"), + } +} + +fn float_aggregate_expression(aggregation: Aggregation) -> &'static str { + match aggregation { + Aggregation::Avg => "AVG(value)", + Aggregation::Min => "MIN(value)", + Aggregation::Max => "MAX(value)", + Aggregation::Sum => "SUM(value)", + Aggregation::First => "(array_agg(value ORDER BY timestamp_us ASC))[1]", + Aggregation::Last => "(array_agg(value ORDER BY timestamp_us DESC))[1]", + Aggregation::Count => unreachable!("handled separately"), + } +} + +fn numeric_aggregate_expression(aggregation: Aggregation) -> &'static str { + match aggregation { + Aggregation::Avg => "AVG(value)", + Aggregation::Min => "MIN(value)", + Aggregation::Max => "MAX(value)", + Aggregation::Sum => "SUM(value)", + Aggregation::First => "(array_agg(value ORDER BY timestamp_us ASC))[1]", + Aggregation::Last => "(array_agg(value ORDER BY timestamp_us DESC))[1]", + Aggregation::Count => unreachable!("handled separately"), + } +} diff --git a/src/storage/query.rs b/src/storage/query.rs index 836cff4a..a95de4db 100644 --- a/src/storage/query.rs +++ b/src/storage/query.rs @@ -136,6 +136,7 @@ impl LabelMatcher { /// Returns true if this matcher targets the metric name (`__name__`). #[inline] + #[allow(dead_code)] // Used by tests and feature-dependent call sites pub fn is_name_matcher(&self) -> bool { self.name == "__name__" } diff --git a/src/storage/rrdcached/mod.rs b/src/storage/rrdcached/mod.rs index f1c11650..c95ec2a3 100644 --- a/src/storage/rrdcached/mod.rs +++ b/src/storage/rrdcached/mod.rs @@ -3,8 +3,9 @@ use crate::{ datamodel::{Sensor, SensorType, TypedSamples}, storage::StorageInstance, }; -use anyhow::{Context, Result, anyhow, bail}; +use anyhow::{Context, Result, bail}; use async_trait::async_trait; +use futures::future::BoxFuture; use rrdcached_client::{ RRDCachedClient, batch_update::BatchUpdate, @@ -12,9 +13,10 @@ use rrdcached_client::{ create::{CreateArguments, CreateDataSource, CreateDataSourceType, CreateRoundRobinArchive}, errors::RRDCachedClientError, }; +use smallvec::SmallVec; use std::{collections::HashSet, sync::Arc}; use tokio::sync::RwLock; -use tracing::{debug, error}; +use tracing::{error, warn}; use url::Url; use uuid::Uuid; @@ -112,13 +114,170 @@ impl std::str::FromStr for Preset { #[derive(Debug)] pub struct RrdCachedStorage { - client: Arc>, - + client: Arc>>, + connector: Arc, created_sensors: Arc>>, - preset: Preset, } +#[derive(Debug, Clone)] +struct PreparedBatchUpdate { + path: String, + timestamp: Option, + data: Vec, +} + +impl PreparedBatchUpdate { + fn into_batch_update(self) -> Result { + BatchUpdate::new(&self.path, self.timestamp, self.data) + } +} + +// Trait to abstract over TCP and Unix socket clients +#[async_trait::async_trait] +trait RRDCachedClientTrait: Send + Sync + std::fmt::Debug { + async fn create( + &mut self, + args: rrdcached_client::create::CreateArguments, + ) -> Result<(), rrdcached_client::errors::RRDCachedClientError>; + async fn batch( + &mut self, + batch_updates: Vec, + ) -> Result<(), rrdcached_client::errors::RRDCachedClientError>; + async fn flush_all(&mut self) -> Result<(), rrdcached_client::errors::RRDCachedClientError>; + async fn list( + &mut self, + recursive: bool, + path: Option<&str>, + ) -> Result, rrdcached_client::errors::RRDCachedClientError>; + async fn fetch( + &mut self, + path: &str, + consolidation_function: ConsolidationFunction, + start: Option, + end: Option, + columns: Option>, + ) -> Result< + rrdcached_client::fetch::FetchResponse, + rrdcached_client::errors::RRDCachedClientError, + >; +} + +#[async_trait] +trait RRDCachedConnectorTrait: Send + Sync + std::fmt::Debug { + async fn connect(&self) -> Result, RRDCachedClientError>; +} + +#[derive(Debug, Clone)] +enum RRDCachedConnectionTarget { + Tcp { address: String }, + Unix { socket_path: String }, +} + +#[async_trait] +impl RRDCachedConnectorTrait for RRDCachedConnectionTarget { + async fn connect(&self) -> Result, RRDCachedClientError> { + match self { + RRDCachedConnectionTarget::Tcp { address } => { + let client = RRDCachedClient::connect_tcp(address).await?; + Ok(Box::new(client)) + } + RRDCachedConnectionTarget::Unix { socket_path } => { + let client = RRDCachedClient::connect_unix(socket_path).await?; + Ok(Box::new(client)) + } + } + } +} + +// Implement the trait for TCP client +#[async_trait::async_trait] +impl RRDCachedClientTrait for RRDCachedClient { + async fn create( + &mut self, + args: rrdcached_client::create::CreateArguments, + ) -> Result<(), rrdcached_client::errors::RRDCachedClientError> { + RRDCachedClient::create(self, args).await + } + + async fn batch( + &mut self, + batch_updates: Vec, + ) -> Result<(), rrdcached_client::errors::RRDCachedClientError> { + RRDCachedClient::batch(self, batch_updates).await + } + + async fn flush_all(&mut self) -> Result<(), rrdcached_client::errors::RRDCachedClientError> { + RRDCachedClient::flush_all(self).await + } + + async fn list( + &mut self, + recursive: bool, + path: Option<&str>, + ) -> Result, rrdcached_client::errors::RRDCachedClientError> { + RRDCachedClient::list(self, recursive, path).await + } + + async fn fetch( + &mut self, + path: &str, + consolidation_function: ConsolidationFunction, + start: Option, + end: Option, + columns: Option>, + ) -> Result< + rrdcached_client::fetch::FetchResponse, + rrdcached_client::errors::RRDCachedClientError, + > { + RRDCachedClient::fetch(self, path, consolidation_function, start, end, columns).await + } +} + +// Implement the trait for Unix socket client +#[async_trait::async_trait] +impl RRDCachedClientTrait for RRDCachedClient { + async fn create( + &mut self, + args: rrdcached_client::create::CreateArguments, + ) -> Result<(), rrdcached_client::errors::RRDCachedClientError> { + RRDCachedClient::create(self, args).await + } + + async fn batch( + &mut self, + batch_updates: Vec, + ) -> Result<(), rrdcached_client::errors::RRDCachedClientError> { + RRDCachedClient::batch(self, batch_updates).await + } + + async fn flush_all(&mut self) -> Result<(), rrdcached_client::errors::RRDCachedClientError> { + RRDCachedClient::flush_all(self).await + } + + async fn list( + &mut self, + recursive: bool, + path: Option<&str>, + ) -> Result, rrdcached_client::errors::RRDCachedClientError> { + RRDCachedClient::list(self, recursive, path).await + } + + async fn fetch( + &mut self, + path: &str, + consolidation_function: ConsolidationFunction, + start: Option, + end: Option, + columns: Option>, + ) -> Result< + rrdcached_client::fetch::FetchResponse, + rrdcached_client::errors::RRDCachedClientError, + > { + RRDCachedClient::fetch(self, path, consolidation_function, start, end, columns).await + } +} + impl RrdCachedStorage { pub async fn connect(connection_string: &str) -> Result { let url = Url::parse(connection_string)?; @@ -135,52 +294,163 @@ impl RrdCachedStorage { "rrdcached" | "rrdcached+tcp" => { // extract host and port let host = url.host_str().ok_or_else(|| { - anyhow::Error::from(StorageError::configuration( - "RRDCached connection URL missing host", + anyhow::Error::from(StorageError::Configuration( + "RRDCached connection URL missing host".to_string(), )) })?; let port = url.port().ok_or_else(|| { - anyhow::Error::from(StorageError::configuration( - "RRDCached connection URL missing port", + anyhow::Error::from(StorageError::Configuration( + "RRDCached connection URL missing port".to_string(), )) })?; - let client = RRDCachedClient::connect_tcp(&format!("{}:{}", host, port)).await?; - Ok(Self { - client: Arc::new(RwLock::new(client)), - created_sensors: Arc::new(RwLock::new(HashSet::new())), - preset, - }) + let connector = Arc::new(RRDCachedConnectionTarget::Tcp { + address: format!("{}:{}", host, port), + }); + + Self::connect_with_connector(connector, preset).await } "rrdcached+unix" => { - unimplemented!() + // Extract Unix socket path from the URL + let socket_path = url.path(); + if socket_path.is_empty() { + bail!("RRDCached Unix socket connection URL missing socket path"); + } + + let connector = Arc::new(RRDCachedConnectionTarget::Unix { + socket_path: socket_path.to_string(), + }); + + Self::connect_with_connector(connector, preset).await } _ => bail!("Invalid scheme in connection string: {}", scheme), } } + async fn connect_with_connector( + connector: Arc, + preset: Preset, + ) -> Result { + let client = connector.connect().await?; + Ok(Self::new_with_parts(client, connector, preset)) + } + + fn new_with_parts( + client: Box, + connector: Arc, + preset: Preset, + ) -> Self { + Self { + client: Arc::new(RwLock::new(client)), + connector, + created_sensors: Arc::new(RwLock::new(HashSet::new())), + preset, + } + } + + #[cfg(test)] + fn new_for_test( + client: Box, + connector: Arc, + ) -> Self { + Self::new_with_parts(client, connector, Preset::Hoarder) + } + + fn is_reconnectable_error(error: &RRDCachedClientError) -> bool { + matches!( + error, + RRDCachedClientError::Io(_) | RRDCachedClientError::Parsing(_) + ) + } + + fn log_client_error(operation: &str, error: &RRDCachedClientError) { + error!("RRDCached {} failed: {:?}", operation, error); + if let RRDCachedClientError::BatchUpdateErrorResponse(message, errors) = error { + error!("RRDCached batch update error response: {:?}", message); + for item in errors { + error!("RRDCached batch update error: {:?}", item); + } + } + } + + async fn reconnect(&self, operation: &str) -> Result<()> { + let client = + self.connector.connect().await.with_context(|| { + format!("Failed to reconnect RRDCached client after {}", operation) + })?; + let mut current_client = self.client.write().await; + *current_client = client; + Ok(()) + } + + async fn with_reconnect( + &self, + operation: &str, + mut action: impl for<'a> FnMut( + &'a mut dyn RRDCachedClientTrait, + ) -> BoxFuture<'a, Result>, + ) -> Result { + let mut retried = false; + + loop { + let result = { + let mut client = self.client.write().await; + action(client.as_mut()).await + }; + + match result { + Ok(value) => return Ok(value), + Err(error) if !retried && Self::is_reconnectable_error(&error) => { + Self::log_client_error(operation, &error); + warn!( + "RRDCached {} hit a reconnectable error; reconnecting and retrying once", + operation + ); + retried = true; + self.reconnect(operation).await?; + } + Err(error) => { + Self::log_client_error(operation, &error); + return Err(error).context(format!("RRDCached {} failed", operation)); + } + } + } + } + async fn create_sensors(&self, sensors: &[Arc], start_timestamp: u64) -> Result<()> { if sensors.is_empty() { return Ok(()); } - let mut client = self.client.write().await; - let mut created_sensors = self.created_sensors.write().await; + for sensor in sensors { - client - .create(CreateArguments { - path: sensor.uuid.to_string(), - data_sources: vec![CreateDataSource { - name: "sensapp".to_string(), - minimum: None, - maximum: None, - heartbeat: 20, - serie_type: CreateDataSourceType::Gauge, - }], - round_robin_archives: self.preset.get_round_robin_archives(), - start_timestamp, - step_seconds: 10, + let sensor = sensor.clone(); + let sensor_for_create = sensor.clone(); + let preset = self.preset.clone(); + + self.with_reconnect("create", move |client| { + let sensor = sensor_for_create.clone(); + let preset = preset.clone(); + Box::pin(async move { + client + .create(CreateArguments { + path: sensor.uuid.to_string(), + data_sources: vec![CreateDataSource { + name: "sensapp".to_string(), + minimum: None, + maximum: None, + heartbeat: 20, + serie_type: CreateDataSourceType::Gauge, + }], + round_robin_archives: preset.get_round_robin_archives(), + start_timestamp, + step_seconds: 10, + }) + .await }) - .await?; + }) + .await?; + + let mut created_sensors = self.created_sensors.write().await; created_sensors.insert(sensor.uuid); } Ok(()) @@ -211,11 +481,11 @@ impl StorageInstance for RrdCachedStorage { if timestamp < min_timestamp { min_timestamp = timestamp; } - batch_updates.push(BatchUpdate::new( - &name, - Some(timestamp), - vec![value.value], - )?); + batch_updates.push(PreparedBatchUpdate { + path: name.clone(), + timestamp: Some(timestamp), + data: vec![value.value], + }); } } TypedSamples::Numeric(samples) => { @@ -225,11 +495,11 @@ impl StorageInstance for RrdCachedStorage { min_timestamp = timestamp; } use rust_decimal::prelude::ToPrimitive; - batch_updates.push(BatchUpdate::new( - &name, - Some(timestamp), - vec![value.value.to_f64().unwrap_or(f64::NAN)], - )?); + batch_updates.push(PreparedBatchUpdate { + path: name.clone(), + timestamp: Some(timestamp), + data: vec![value.value.to_f64().unwrap_or(f64::NAN)], + }); } } TypedSamples::Integer(samples) => { @@ -238,11 +508,11 @@ impl StorageInstance for RrdCachedStorage { if timestamp < min_timestamp { min_timestamp = timestamp; } - batch_updates.push(BatchUpdate::new( - &name, - Some(timestamp), - vec![value.value as f64], - )?); + batch_updates.push(PreparedBatchUpdate { + path: name.clone(), + timestamp: Some(timestamp), + data: vec![value.value as f64], + }); } } TypedSamples::Boolean(samples) => { @@ -251,15 +521,18 @@ impl StorageInstance for RrdCachedStorage { if timestamp < min_timestamp { min_timestamp = timestamp; } - batch_updates.push(BatchUpdate::new( - &name, - Some(timestamp), - vec![if value.value { 1.0 } else { 0.0 }], - )?); + batch_updates.push(PreparedBatchUpdate { + path: name.clone(), + timestamp: Some(timestamp), + data: vec![if value.value { 1.0 } else { 0.0 }], + }); } } _ => { - print!("Unsupported type"); + tracing::warn!( + "RRDCached: unsupported sensor type {:?}, skipping", + single_sensor_batch.sensor.sensor_type + ); } } } @@ -287,30 +560,26 @@ impl StorageInstance for RrdCachedStorage { .await?; } - { - let mut client = self.client.write().await; - match client.batch(batch_updates).await { - Ok(_) => {} - Err(e) => { - error!("RRDCached: Failed to batch update: {:?}", e); - match e { - RRDCachedClientError::BatchUpdateErrorResponse(string, errors) => { - error!("RRDCached: Batch update error response: {:?}", string); - for error in errors { - error!("RRDCached: Batch update error: {:?}", error); - } - } - _ => {} - } - } - } + if batch_updates.is_empty() { + return Ok(()); } - // Flush the RRD cached client - { - let mut client = self.client.write().await; - client.flush_all().await?; - } + self.with_reconnect("batch update", move |client| { + let batch_updates = batch_updates.clone(); + Box::pin(async move { + let batch_updates = batch_updates + .into_iter() + .map(PreparedBatchUpdate::into_batch_update) + .collect::, _>>()?; + client.batch(batch_updates).await + }) + }) + .await?; + + self.with_reconnect("flush_all", |client| { + Box::pin(async move { client.flush_all().await }) + }) + .await?; Ok(()) } @@ -319,41 +588,301 @@ impl StorageInstance for RrdCachedStorage { Ok(()) } - async fn list_series(&self) -> Result> { - unimplemented!(); + async fn list_series( + &self, + metric_filter: Option<&str>, + limit: Option, + bookmark: Option<&str>, + ) -> Result { + // Validate and cap the limit + let effective_limit = limit + .unwrap_or(crate::storage::DEFAULT_LIST_SERIES_LIMIT) + .min(crate::storage::MAX_LIST_SERIES_LIMIT); + + match self + .with_reconnect("list", |client| { + Box::pin(async move { client.list(true, None).await }) + }) + .await + { + Ok(rrd_files) => { + let mut sensors = Vec::new(); + let mut last_uuid: Option = None; + + for rrd_file in rrd_files { + // Trim whitespace from filename (RRDcached LIST returns names with trailing newlines) + let rrd_file = rrd_file.trim(); + + // Filter by metric name if provided + if let Some(filter) = metric_filter + && !rrd_file.contains(filter) + { + continue; + } + + // Extract UUID from filename (assuming format: ".rrd") + let filename = rrd_file.rsplit('/').next().unwrap_or(rrd_file); + let uuid_str = filename.strip_suffix(".rrd").unwrap_or(filename); + + // Apply bookmark-based pagination: skip entries <= bookmark + if let Some(bm) = bookmark + && uuid_str <= bm + { + continue; + } + + if let Ok(uuid) = uuid_str.parse::() { + let sensor = crate::datamodel::Sensor { + uuid, + name: uuid.to_string(), + sensor_type: crate::datamodel::SensorType::Float, + unit: None, + labels: SmallVec::new(), + }; + last_uuid = Some(uuid_str.to_string()); + sensors.push(sensor); + } else { + let sensor = crate::datamodel::Sensor { + uuid: Uuid::new_v4(), + name: filename.to_string(), + sensor_type: crate::datamodel::SensorType::Float, + unit: None, + labels: SmallVec::new(), + }; + last_uuid = Some(uuid_str.to_string()); + sensors.push(sensor); + } + + if sensors.len() >= effective_limit { + break; + } + } + + // Return bookmark for next page if we hit the limit + let next_bookmark = if sensors.len() == effective_limit { + last_uuid + } else { + None + }; + + Ok(crate::storage::ListSeriesResult { + series: sensors, + bookmark: next_bookmark, + }) + } + Err(e) => { + error!("Failed to list RRD files: {:?}", e); + + // Fallback to tracking created sensors in this session + let created_sensors = self.created_sensors.read().await; + let mut sensors = Vec::new(); + + for uuid in created_sensors.iter() { + sensors.push(crate::datamodel::Sensor { + uuid: *uuid, + name: uuid.to_string(), + sensor_type: crate::datamodel::SensorType::Float, + unit: None, + labels: SmallVec::new(), + }); + } + + Ok(crate::storage::ListSeriesResult { + series: sensors, + bookmark: None, + }) + } + } + } + + async fn list_metrics(&self) -> Result> { + // RRDcached doesn't support metric-level operations like PostgreSQL + // Return empty list for now, as RRDcached focuses on individual series + Ok(vec![]) } async fn query_sensor_data( &self, - _sensor_uuid: &str, - _start_time: Option, - _end_time: Option, - _limit: Option, + sensor_uuid: &str, + start_time: Option, + end_time: Option, + _limit: Option, // RRD doesn't support limiting results directly ) -> Result> { - unimplemented!("RRDCached sensor data querying not yet implemented"); + use crate::datamodel::{ + Sample, SensorData, SensorType, sensapp_datetime::SensAppDateTimeExt, + }; + use smallvec::SmallVec; + + // Check if sensor exists in our created_sensors set + let created_sensors = self.created_sensors.read().await; + let sensor_uuid_obj = sensor_uuid + .parse::() + .map_err(|e| anyhow::anyhow!("Invalid sensor UUID: {}", e))?; + + drop(created_sensors); + + // Convert time parameters to Unix timestamps + let start_timestamp = start_time.map(|t| t.to_unix_seconds().floor() as i64); + let end_timestamp = end_time.map(|t| t.to_unix_seconds().floor() as i64); + + // Fetch data from RRDcached - first flush to ensure data is written + if let Err(e) = self + .with_reconnect("flush before query", |client| { + Box::pin(async move { client.flush_all().await }) + }) + .await + { + tracing::warn!("Failed to flush before query: {:?}", e); + } + let rrd_path = sensor_uuid.to_string(); + + let fetch_response = match self + .with_reconnect("fetch", |client| { + let rrd_path = rrd_path.clone(); + Box::pin(async move { + client + .fetch( + &rrd_path, + ConsolidationFunction::Average, + start_timestamp, + end_timestamp, + None, + ) + .await + }) + }) + .await + { + Ok(response) => { + tracing::debug!( + "Fetch successful for sensor {}: {} data points", + sensor_uuid, + response.data.len() + ); + response + } + Err(e) => { + // If fetch fails, it might be because no data exists yet + tracing::debug!("Failed to fetch data for sensor {}: {:?}", sensor_uuid, e); + return Ok(None); + } + }; + + // Convert RRD data to SensApp samples + let mut samples = SmallVec::new(); + + tracing::debug!("Processing {} RRD data points", fetch_response.data.len()); + + // RRD returns data as Vec<(timestamp, Vec)> + // We use the first data source (index 0) since we create RRDs with one DS + for (timestamp, values) in fetch_response.data { + tracing::debug!( + "RRD data point: timestamp={}, values={:?}", + timestamp, + values + ); + if let Some(&value) = values.first() { + // Skip NaN values (RRD uses NaN for missing data) + if !value.is_nan() { + let datetime = + crate::datamodel::SensAppDateTime::from_unix_seconds_i64(timestamp as i64); + samples.push(Sample { datetime, value }); + tracing::trace!("Added sample: time={:?}, value={}", datetime, value); + } else { + tracing::trace!("Skipped NaN value at timestamp {}", timestamp); + } + } + } + + tracing::debug!("Converted {} valid samples from RRD data", samples.len()); + + if samples.is_empty() { + tracing::debug!("No valid samples found, returning None"); + return Ok(None); + } + + // Create sensor metadata (we have to reconstruct it since RRD doesn't store it) + let sensor = crate::datamodel::Sensor { + uuid: sensor_uuid_obj, + name: sensor_uuid.to_string(), // Use UUID as name since we don't have the original + sensor_type: SensorType::Float, // Assume Float since RRD stores f64 values + unit: None, // We don't have unit information + labels: SmallVec::new(), // We don't have labels information + }; + + let typed_samples = TypedSamples::Float(samples); + let sensor_data = SensorData::new(sensor, typed_samples); + + Ok(Some(sensor_data)) } async fn query_sensors_by_labels( &self, - _matchers: &[super::LabelMatcher], - _start_time: Option, - _end_time: Option, - _limit: Option, + matchers: &[super::LabelMatcher], + start_time: Option, + end_time: Option, + limit: Option, _numeric_only: bool, ) -> Result> { - // TODO: Implement label-based query for RRDCached - anyhow::bail!("query_sensors_by_labels not yet implemented for RRDCached") + // RRDcached doesn't store label metadata, so we can only match on __name__ + // (which corresponds to sensor name / UUID). + // All RRD data is numeric, so numeric_only is always satisfied. + if matchers.is_empty() { + return Ok(Vec::new()); + } + + // Get all available series + let result = self.list_series(None, None, None).await?; + + // Filter sensors by matchers (only __name__ is meaningful for RRDcached) + let filtered_sensors: Vec<_> = result + .series + .into_iter() + .filter(|sensor| { + matchers.iter().all(|matcher| { + if matcher.is_name_matcher() { + match matcher.matcher_type { + super::MatcherType::Equal => sensor.name == matcher.value, + super::MatcherType::NotEqual => sensor.name != matcher.value, + super::MatcherType::RegexMatch => regex::Regex::new(&matcher.value) + .map(|re| re.is_match(&sensor.name)) + .unwrap_or(false), + super::MatcherType::RegexNotMatch => regex::Regex::new(&matcher.value) + .map(|re| !re.is_match(&sensor.name)) + .unwrap_or(true), + } + } else { + // RRDcached has no labels, so non-name matchers + // match nothing for Equal/RegexMatch, everything for NotEqual/RegexNotMatch + matcher.matcher_type.is_negated() + } + }) + }) + .collect(); + + // Fetch data for each matching sensor + let mut results = Vec::with_capacity(filtered_sensors.len()); + for sensor in filtered_sensors { + let uuid_str = sensor.uuid.to_string(); + if let Some(sensor_data) = self + .query_sensor_data(&uuid_str, start_time, end_time, limit) + .await? + { + results.push(sensor_data); + } + } + + Ok(results) } /// Health check for RRDCached storage /// Verifies the connection to RRDCached by checking if client can respond async fn health_check(&self) -> Result<()> { - // For RRDCached, we can try to flush which should return quickly if connected - let mut client = self.client.write().await; - client - .flush_all() - .await - .context("RRDCached health check failed")?; + self.with_reconnect("health check", |client| { + Box::pin(async move { client.flush_all().await }) + }) + .await + .context("RRDCached health check failed")?; Ok(()) } @@ -368,3 +897,153 @@ impl StorageInstance for RrdCachedStorage { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + collections::VecDeque, + io, + sync::{ + Mutex, + atomic::{AtomicUsize, Ordering}, + }, + }; + + #[derive(Debug)] + struct MockClient { + list_results: Mutex, RRDCachedClientError>>>, + flush_results: Mutex>>, + } + + impl MockClient { + fn with_list_results(results: Vec, RRDCachedClientError>>) -> Self { + Self { + list_results: Mutex::new(results.into()), + flush_results: Mutex::new(VecDeque::new()), + } + } + + fn with_flush_results(results: Vec>) -> Self { + Self { + list_results: Mutex::new(VecDeque::new()), + flush_results: Mutex::new(results.into()), + } + } + } + + #[async_trait] + impl RRDCachedClientTrait for MockClient { + async fn create(&mut self, _args: CreateArguments) -> Result<(), RRDCachedClientError> { + Ok(()) + } + + async fn batch( + &mut self, + _batch_updates: Vec, + ) -> Result<(), RRDCachedClientError> { + Ok(()) + } + + async fn flush_all(&mut self) -> Result<(), RRDCachedClientError> { + self.flush_results + .lock() + .unwrap() + .pop_front() + .unwrap_or(Ok(())) + } + + async fn list( + &mut self, + _recursive: bool, + _path: Option<&str>, + ) -> Result, RRDCachedClientError> { + self.list_results + .lock() + .unwrap() + .pop_front() + .unwrap_or(Ok(Vec::new())) + } + + async fn fetch( + &mut self, + _path: &str, + _consolidation_function: ConsolidationFunction, + _start: Option, + _end: Option, + _columns: Option>, + ) -> Result { + Ok(rrdcached_client::fetch::FetchResponse { + flush_version: 0, + start: 0, + end: 0, + step: 1, + ds_count: 0, + ds_names: Vec::new(), + data: Vec::new(), + }) + } + } + + #[derive(Debug)] + struct MockConnector { + connect_calls: AtomicUsize, + clients: Mutex>>, + } + + impl MockConnector { + fn new(clients: Vec>) -> Self { + Self { + connect_calls: AtomicUsize::new(0), + clients: Mutex::new(clients.into()), + } + } + + fn connect_calls(&self) -> usize { + self.connect_calls.load(Ordering::SeqCst) + } + } + + #[async_trait] + impl RRDCachedConnectorTrait for MockConnector { + async fn connect(&self) -> Result, RRDCachedClientError> { + self.connect_calls.fetch_add(1, Ordering::SeqCst); + self.clients + .lock() + .unwrap() + .pop_front() + .ok_or_else(|| RRDCachedClientError::Parsing("missing mock client".to_string())) + } + } + + #[tokio::test] + async fn list_series_reconnects_after_io_error() { + let initial_client = Box::new(MockClient::with_list_results(vec![Err( + RRDCachedClientError::Io(io::Error::new(io::ErrorKind::BrokenPipe, "boom")), + )])); + let connector = Arc::new(MockConnector::new(vec![Box::new( + MockClient::with_list_results(vec![Ok(vec![format!("{}.rrd\n", Uuid::nil())])]), + )])); + let storage = RrdCachedStorage::new_for_test(initial_client, connector.clone()); + + let result = storage.list_series(None, Some(10), None).await.unwrap(); + + assert_eq!(result.series.len(), 1); + assert_eq!(result.series[0].uuid, Uuid::nil()); + assert_eq!(connector.connect_calls(), 1); + } + + #[tokio::test] + async fn health_check_does_not_retry_non_reconnectable_errors() { + let initial_client = Box::new(MockClient::with_flush_results(vec![Err( + RRDCachedClientError::UnexpectedResponse(-1, "protocol failure".to_string()), + )])); + let connector = Arc::new(MockConnector::new(Vec::new())); + let storage = RrdCachedStorage::new_for_test(initial_client, connector.clone()); + + let error = storage.health_check().await.unwrap_err().to_string(); + + assert!(error.contains("RRDCached health check failed")); + assert_eq!(connector.connect_calls(), 0); + } +} diff --git a/src/storage/sqlite/mod.rs b/src/storage/sqlite/mod.rs index 9299b7ad..b58fcb80 100644 --- a/src/storage/sqlite/mod.rs +++ b/src/storage/sqlite/mod.rs @@ -3,6 +3,7 @@ mod matchers; pub mod sqlite_publishers; pub mod sqlite_utilities; mod storage; +mod storage_query_helpers; // rexport SqliteStorage pub use storage::SqliteStorage; diff --git a/src/storage/sqlite/sqlite_utilities.rs b/src/storage/sqlite/sqlite_utilities.rs index afefb00c..e319972d 100644 --- a/src/storage/sqlite/sqlite_utilities.rs +++ b/src/storage/sqlite/sqlite_utilities.rs @@ -3,7 +3,6 @@ use crate::datamodel::unit::Unit; use anyhow::Result; use cached::proc_macro::cached; use sqlx::{Sqlite, Transaction, prelude::*}; -use std::time::Duration; use uuid::Uuid; #[cached( diff --git a/src/storage/sqlite/storage.rs b/src/storage/sqlite/storage.rs index 74fb9e9b..2f5705c1 100644 --- a/src/storage/sqlite/storage.rs +++ b/src/storage/sqlite/storage.rs @@ -1,5 +1,9 @@ use super::sqlite_publishers::*; use super::sqlite_utilities::get_sensor_id_or_create_sensor; +use super::storage_query_helpers::{ + sqlite_bucketed_cte, sqlite_first_last_query, sqlite_float_expression, sqlite_group_by_clause, + sqlite_integer_expression, sqlite_numeric_expression, +}; use crate::datamodel::batch::{Batch, SingleSensorBatch}; use crate::datamodel::sensapp_datetime::SensAppDateTimeExt; use crate::datamodel::unit::Unit; @@ -8,7 +12,9 @@ use crate::datamodel::{ sensapp_vec::SensAppLabels, }; use crate::storage::{ - DEFAULT_QUERY_LIMIT, StorageError, StorageInstance, common::datetime_to_micros, + Aggregation, DEFAULT_LIST_SERIES_LIMIT, DEFAULT_QUERY_LIMIT, MAX_LIST_SERIES_LIMIT, + SensorAvailabilitySummary, SensorDataQueryOptions, StorageError, StorageInstance, + common::datetime_to_micros, }; use anyhow::{Context, Result}; use async_trait::async_trait; @@ -88,7 +94,26 @@ impl StorageInstance for SqliteStorage { async fn list_series( &self, metric_filter: Option<&str>, - ) -> Result> { + limit: Option, + bookmark: Option<&str>, + ) -> Result { + let bookmark_id = if let Some(bookmark_str) = bookmark { + Some(bookmark_str.parse::().map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Invalid bookmark format: {}", e), + None, + None, + )) + })?) + } else { + None + }; + + let effective_limit = limit + .unwrap_or(DEFAULT_LIST_SERIES_LIMIT) + .min(MAX_LIST_SERIES_LIMIT); + let fetch_limit = effective_limit.saturating_add(1) as i64; + #[derive(sqlx::FromRow)] struct SensorRow { sensor_id: Option, @@ -99,22 +124,28 @@ impl StorageInstance for SqliteStorage { unit_description: Option, } - // Query sensors with their metadata using the catalog view, optionally filtered by metric name + // Query sensors with cursor pagination based on sensor_id. let sensor_rows: Vec = sqlx::query_as( r#" SELECT sensor_id, uuid, name, type, unit_name, unit_description FROM sensor_catalog_view WHERE (?1 IS NULL OR name = ?1) - ORDER BY uuid ASC + AND (?2 IS NULL OR sensor_id > ?2) + ORDER BY sensor_id ASC + LIMIT ?3 "#, ) .bind(metric_filter) + .bind(bookmark_id) + .bind(fetch_limit) .fetch_all(&self.pool) .await?; + let has_more = sensor_rows.len() > effective_limit; let mut sensors = Vec::new(); + let mut last_sensor_id = None; - for sensor_row in sensor_rows { + for sensor_row in sensor_rows.into_iter().take(effective_limit) { // Parse sensor metadata with improved error handling let sensor_uuid = Uuid::parse_str(&sensor_row.uuid).map_err(|e| { anyhow::Error::from(StorageError::invalid_data_format( @@ -150,6 +181,7 @@ impl StorageInstance for SqliteStorage { Some(sensor_name), )) })?; + last_sensor_id = Some(sensor_id); #[derive(sqlx::FromRow)] struct LabelRow { @@ -192,7 +224,14 @@ impl StorageInstance for SqliteStorage { sensors.push(sensor); } - Ok(sensors) + Ok(crate::storage::ListSeriesResult { + series: sensors, + bookmark: if has_more { + last_sensor_id.map(|sensor_id| sensor_id.to_string()) + } else { + None + }, + }) } async fn list_metrics(&self) -> Result> { @@ -426,6 +465,240 @@ impl StorageInstance for SqliteStorage { Ok(Some(SensorData::new(sensor, samples))) } + async fn query_sensor_data_advanced( + &self, + sensor_uuid: &str, + options: &SensorDataQueryOptions, + ) -> Result> { + options.validate()?; + + if let (Some(step_ms), Some(aggregation)) = (options.step_ms, options.aggregation) { + if let Some((sensor_id, mut sensor)) = self.get_sensor_metadata(sensor_uuid).await? { + let start_time_us = options.start_time.as_ref().map(datetime_to_micros); + let end_time_us = options.end_time.as_ref().map(datetime_to_micros); + let step_us = step_ms * 1000; + let origin_us = start_time_us.unwrap_or(0); + + let samples = match sensor.sensor_type { + SensorType::Integer => { + self.query_integer_samples_aggregated( + sensor_id, + start_time_us, + end_time_us, + step_us, + origin_us, + aggregation, + options.limit, + ) + .await? + } + SensorType::Numeric => { + self.query_numeric_samples_aggregated( + sensor_id, + start_time_us, + end_time_us, + step_us, + origin_us, + aggregation, + options.limit, + ) + .await? + } + SensorType::Float => { + self.query_float_samples_aggregated( + sensor_id, + start_time_us, + end_time_us, + step_us, + origin_us, + aggregation, + options.limit, + ) + .await? + } + _ => { + let raw = self + .query_sensor_data( + sensor_uuid, + options.start_time, + options.end_time, + options.limit, + ) + .await?; + return raw + .map(|sensor_data| { + crate::storage::common::apply_query_options(sensor_data, options) + }) + .transpose(); + } + }; + + sensor.sensor_type = match &samples { + TypedSamples::Integer(_) => SensorType::Integer, + TypedSamples::Numeric(_) => SensorType::Numeric, + TypedSamples::Float(_) => SensorType::Float, + _ => sensor.sensor_type, + }; + if aggregation.output_is_count() { + sensor.unit = None; + } + + let post_query_options = SensorDataQueryOptions { + start_time: options.start_time, + end_time: options.end_time, + limit: options.limit, + step_ms: None, + aggregation: None, + simplify: options.simplify, + }; + + return crate::storage::common::apply_query_options( + SensorData::new(sensor, samples), + &post_query_options, + ) + .map(Some); + } + + return Ok(None); + } + + let raw = self + .query_sensor_data( + sensor_uuid, + options.start_time, + options.end_time, + options.limit, + ) + .await?; + + raw.map(|sensor_data| crate::storage::common::apply_query_options(sensor_data, options)) + .transpose() + } + + async fn query_sensor_data_latest( + &self, + sensor_uuid: &str, + start_time: Option, + end_time: Option, + ) -> Result> { + let Some((sensor_id, sensor)) = self.get_sensor_metadata(sensor_uuid).await? else { + return Ok(None); + }; + + let start_time_us = start_time.as_ref().map(datetime_to_micros); + let end_time_us = end_time.as_ref().map(datetime_to_micros); + let table_name = Self::sensor_table_name(sensor.sensor_type); + + let Some(latest_timestamp_us) = self + .query_latest_timestamp_us(table_name, sensor_id, start_time_us, end_time_us) + .await? + else { + return Ok(None); + }; + + let samples = match sensor.sensor_type { + SensorType::Integer => { + self.query_integer_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Numeric => { + self.query_numeric_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Float => { + self.query_float_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::String => { + self.query_string_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Boolean => { + self.query_boolean_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Location => { + self.query_location_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Json => { + self.query_json_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Blob => { + self.query_blob_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + }; + + Ok(Some(SensorData::new(sensor, samples))) + } + + async fn query_sensor_data_availability( + &self, + sensor_uuid: &str, + start_time: SensAppDateTime, + end_time: SensAppDateTime, + step_ms: Option, + ) -> Result> { + let Some((sensor_id, sensor)) = self.get_sensor_metadata(sensor_uuid).await? else { + return Ok(None); + }; + + let summary = self + .query_availability_summary_native( + Self::sensor_table_name(sensor.sensor_type), + sensor_id, + sensor, + datetime_to_micros(&start_time), + datetime_to_micros(&end_time), + step_ms, + ) + .await?; + + Ok(Some(summary)) + } + async fn query_sensors_by_labels( &self, matchers: &[crate::storage::LabelMatcher], @@ -590,6 +863,226 @@ impl StorageInstance for SqliteStorage { } impl SqliteStorage { + async fn get_sensor_metadata(&self, sensor_uuid: &str) -> Result> { + let parsed_uuid = Uuid::from_str(sensor_uuid).context("Failed to parse sensor UUID")?; + let uuid_string = parsed_uuid.to_string(); + + #[derive(sqlx::FromRow)] + struct SensorMetadataRow { + sensor_id: Option, + uuid: String, + name: String, + r#type: String, + unit_name: Option, + unit_description: Option, + } + + let sensor_row: Option = sqlx::query_as( + r#" + SELECT s.sensor_id, s.uuid, s.name, s.type, u.name as unit_name, u.description as unit_description + FROM sensors s + LEFT JOIN units u ON s.unit = u.id + WHERE s.uuid = ? + "#, + ) + .bind(&uuid_string) + .fetch_optional(&self.pool) + .await?; + + let Some(sensor_row) = sensor_row else { + return Ok(None); + }; + + let sensor_uuid = Uuid::parse_str(&sensor_row.uuid).map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Failed to parse sensor UUID '{}': {}", sensor_row.uuid, e), + None, + Some(&sensor_row.name), + )) + })?; + + let sensor_type = SensorType::from_str(&sensor_row.r#type).map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Failed to parse sensor type '{}': {}", sensor_row.r#type, e), + Some(sensor_uuid), + Some(&sensor_row.name), + )) + })?; + + let unit = match (sensor_row.unit_name, sensor_row.unit_description) { + (Some(name), description) if !name.is_empty() => Some(Unit::new(name, description)), + _ => None, + }; + + let sensor_id = sensor_row.sensor_id.ok_or_else(|| { + anyhow::Error::from(StorageError::missing_field( + "sensor_id", + Some(sensor_uuid), + Some(&sensor_row.name), + )) + })?; + + #[derive(sqlx::FromRow)] + struct LabelRow { + label_name: String, + label_value: String, + } + + let labels_rows: Vec = sqlx::query_as( + r#" + SELECT lnd.name as label_name, ldd.description as label_value + FROM labels l + JOIN labels_name_dictionary lnd ON l.name = lnd.id + JOIN labels_description_dictionary ldd ON l.description = ldd.id + WHERE l.sensor_id = ? + "#, + ) + .bind(sensor_id) + .fetch_all(&self.pool) + .await + .with_context(|| { + format!( + "Failed to query labels for sensor UUID={} name='{}'", + sensor_uuid, sensor_row.name + ) + })?; + + let mut labels: SensAppLabels = smallvec![]; + for label_row in labels_rows { + labels.push((label_row.label_name, label_row.label_value)); + } + + Ok(Some(( + sensor_id, + Sensor::new( + sensor_uuid, + sensor_row.name, + sensor_type, + unit, + Some(labels), + ), + ))) + } + + fn sensor_table_name(sensor_type: SensorType) -> &'static str { + match sensor_type { + SensorType::Integer => "integer_values", + SensorType::Numeric => "numeric_values", + SensorType::Float => "float_values", + SensorType::String => "string_values", + SensorType::Boolean => "boolean_values", + SensorType::Location => "location_values", + SensorType::Json => "json_values", + SensorType::Blob => "blob_values", + } + } + + async fn query_latest_timestamp_us( + &self, + table_name: &str, + sensor_id: i64, + start_time: Option, + end_time: Option, + ) -> Result> { + let sql = format!( + r#" + SELECT MAX(timestamp_us) AS timestamp_us + FROM {table_name} + WHERE sensor_id = ? + AND (? IS NULL OR timestamp_us >= ?) + AND (? IS NULL OR timestamp_us <= ?) + "# + ); + + let timestamp_us: Option = sqlx::query_scalar(&sql) + .bind(sensor_id) + .bind(start_time) + .bind(start_time) + .bind(end_time) + .bind(end_time) + .fetch_one(&self.pool) + .await?; + + Ok(timestamp_us) + } + + async fn query_availability_summary_native( + &self, + table_name: &str, + sensor_id: i64, + sensor: Sensor, + start_time: i64, + end_time: i64, + step_ms: Option, + ) -> Result { + #[derive(sqlx::FromRow)] + struct Row { + sample_count: i64, + first_sample_at: Option, + last_sample_at: Option, + covered_buckets: Option, + } + + let row: Row = if let Some(step_ms) = step_ms { + let step_us = step_ms.checked_mul(1000).context("step is too large")?; + let sql = format!( + r#" + SELECT + COUNT(*) AS sample_count, + MIN(timestamp_us) AS first_sample_at, + MAX(timestamp_us) AS last_sample_at, + COUNT(DISTINCT ((timestamp_us - ?) / ?)) AS covered_buckets + FROM {table_name} + WHERE sensor_id = ? + AND timestamp_us >= ? + AND timestamp_us <= ? + "# + ); + + sqlx::query_as(&sql) + .bind(start_time) + .bind(step_us) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .fetch_one(&self.pool) + .await? + } else { + let sql = format!( + r#" + SELECT + COUNT(*) AS sample_count, + MIN(timestamp_us) AS first_sample_at, + MAX(timestamp_us) AS last_sample_at, + NULL AS covered_buckets + FROM {table_name} + WHERE sensor_id = ? + AND timestamp_us >= ? + AND timestamp_us <= ? + "# + ); + + sqlx::query_as(&sql) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .fetch_one(&self.pool) + .await? + }; + + Ok(SensorAvailabilitySummary { + sensor, + sample_count: row.sample_count.max(0) as usize, + first_sample_at: row + .first_sample_at + .map(SensAppDateTime::from_unix_microseconds_i64), + last_sample_at: row + .last_sample_at + .map(SensAppDateTime::from_unix_microseconds_i64), + covered_buckets: row.covered_buckets.map(|value| value.max(0) as usize), + }) + } + async fn publish_single_sensor_batch( &self, transaction: &mut Transaction<'_, Sqlite>, @@ -703,6 +1196,362 @@ impl SqliteStorage { Ok(TypedSamples::Integer(samples)) } + #[allow(clippy::too_many_arguments)] + async fn query_integer_samples_aggregated( + &self, + sensor_id: i64, + start_time: Option, + end_time: Option, + step_us: i64, + origin_us: i64, + aggregation: Aggregation, + limit: Option, + ) -> Result { + let limit = limit.unwrap_or(DEFAULT_QUERY_LIMIT) as i64; + + match aggregation { + Aggregation::Avg => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: f64, + } + + let rows: Vec = sqlx::query_as(&format!( + "{} SELECT bucket_us AS timestamp_us, AVG(value) AS value {}", + sqlite_bucketed_cte("integer_values"), + sqlite_group_by_clause() + )) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_us) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Float(samples)) + } + Aggregation::Count => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: i64, + } + + let rows: Vec = sqlx::query_as(&format!( + "{} SELECT bucket_us AS timestamp_us, COUNT(*) AS value {}", + sqlite_bucketed_cte("integer_values"), + sqlite_group_by_clause() + )) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_us) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + Aggregation::First | Aggregation::Last => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: i64, + } + + let rows: Vec = sqlx::query_as(&sqlite_first_last_query( + "integer_values", + aggregation, + "value", + )) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_us) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + _ => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: i64, + } + + let rows: Vec = sqlx::query_as(&format!( + "{} SELECT bucket_us AS timestamp_us, {} AS value {}", + sqlite_bucketed_cte("integer_values"), + sqlite_integer_expression(aggregation), + sqlite_group_by_clause() + )) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_us) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + } + } + + #[allow(clippy::too_many_arguments)] + async fn query_float_samples_aggregated( + &self, + sensor_id: i64, + start_time: Option, + end_time: Option, + step_us: i64, + origin_us: i64, + aggregation: Aggregation, + limit: Option, + ) -> Result { + let limit = limit.unwrap_or(DEFAULT_QUERY_LIMIT) as i64; + + match aggregation { + Aggregation::Count => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: i64, + } + + let rows: Vec = sqlx::query_as(&format!( + "{} SELECT bucket_us AS timestamp_us, COUNT(*) AS value {}", + sqlite_bucketed_cte("float_values"), + sqlite_group_by_clause() + )) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_us) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + Aggregation::First | Aggregation::Last => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: f64, + } + + let rows: Vec = sqlx::query_as(&sqlite_first_last_query( + "float_values", + aggregation, + "value", + )) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_us) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Float(samples)) + } + _ => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: f64, + } + + let rows: Vec = sqlx::query_as(&format!( + "{} SELECT bucket_us AS timestamp_us, {} AS value {}", + sqlite_bucketed_cte("float_values"), + sqlite_float_expression(aggregation), + sqlite_group_by_clause() + )) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_us) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Float(samples)) + } + } + } + + #[allow(clippy::too_many_arguments)] + async fn query_numeric_samples_aggregated( + &self, + sensor_id: i64, + start_time: Option, + end_time: Option, + step_us: i64, + origin_us: i64, + aggregation: Aggregation, + limit: Option, + ) -> Result { + let limit = limit.unwrap_or(DEFAULT_QUERY_LIMIT) as i64; + + match aggregation { + Aggregation::Count => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: i64, + } + + let rows: Vec = sqlx::query_as(&format!( + "{} SELECT bucket_us AS timestamp_us, COUNT(*) AS value {}", + sqlite_bucketed_cte("numeric_values"), + sqlite_group_by_clause() + )) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_us) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + Aggregation::First | Aggregation::Last => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: String, + } + + let rows: Vec = sqlx::query_as(&sqlite_first_last_query( + "numeric_values", + aggregation, + "CAST(value AS TEXT)", + )) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_us) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: Decimal::from_str(&row.value) + .context("Failed to parse aggregated SQLite numeric value")?, + }); + } + Ok(TypedSamples::Numeric(samples)) + } + _ => { + #[derive(sqlx::FromRow)] + struct Row { + timestamp_us: i64, + value: String, + } + + let rows: Vec = sqlx::query_as(&format!( + "{} SELECT bucket_us AS timestamp_us, CAST({} AS TEXT) AS value {}", + sqlite_bucketed_cte("numeric_values"), + sqlite_numeric_expression(aggregation), + sqlite_group_by_clause() + )) + .bind(sensor_id) + .bind(start_time) + .bind(end_time) + .bind(step_us) + .bind(origin_us) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: SensAppDateTime::from_unix_microseconds_i64(row.timestamp_us), + value: Decimal::from_str(&row.value) + .context("Failed to parse aggregated SQLite numeric value")?, + }); + } + Ok(TypedSamples::Numeric(samples)) + } + } + } + async fn query_numeric_samples( &self, sensor_id: i64, diff --git a/src/storage/sqlite/storage_query_helpers.rs b/src/storage/sqlite/storage_query_helpers.rs new file mode 100644 index 00000000..dbe853c6 --- /dev/null +++ b/src/storage/sqlite/storage_query_helpers.rs @@ -0,0 +1,96 @@ +use crate::storage::Aggregation; + +pub(super) fn sqlite_bucketed_cte(table_name: &str) -> String { + format!( + r#" + WITH bucketed AS ( + SELECT + timestamp_us, + value, + (?5 + ((timestamp_us - ?5) / ?4) * ?4) AS bucket_us + FROM {table_name} + WHERE sensor_id = ?1 + AND (?2 IS NULL OR timestamp_us >= ?2) + AND (?3 IS NULL OR timestamp_us <= ?3) + ) + "# + ) +} + +pub(super) fn sqlite_group_by_clause() -> &'static str { + "FROM bucketed GROUP BY 1 ORDER BY 1 ASC LIMIT ?6" +} + +pub(super) fn sqlite_first_last_query( + table_name: &str, + aggregation: Aggregation, + value_expression: &str, +) -> String { + let direction = match aggregation { + Aggregation::First => "ASC", + Aggregation::Last => "DESC", + _ => unreachable!("only first/last use sqlite_first_last_query"), + }; + + format!( + r#" + WITH bucketed AS ( + SELECT + timestamp_us, + {value_expression} AS value, + (?5 + ((timestamp_us - ?5) / ?4) * ?4) AS bucket_us + FROM {table_name} + WHERE sensor_id = ?1 + AND (?2 IS NULL OR timestamp_us >= ?2) + AND (?3 IS NULL OR timestamp_us <= ?3) + ), + ranked AS ( + SELECT + bucket_us, + value, + ROW_NUMBER() OVER (PARTITION BY bucket_us ORDER BY timestamp_us {direction}) AS row_num + FROM bucketed + ) + SELECT bucket_us AS timestamp_us, value + FROM ranked + WHERE row_num = 1 + ORDER BY bucket_us ASC + LIMIT ?6 + "# + ) +} + +pub(super) fn sqlite_integer_expression(aggregation: Aggregation) -> &'static str { + match aggregation { + Aggregation::Min => "MIN(value)", + Aggregation::Max => "MAX(value)", + Aggregation::Sum => "SUM(value)", + Aggregation::First | Aggregation::Last | Aggregation::Avg | Aggregation::Count => { + unreachable!("handled separately") + } + } +} + +pub(super) fn sqlite_float_expression(aggregation: Aggregation) -> &'static str { + match aggregation { + Aggregation::Avg => "AVG(value)", + Aggregation::Min => "MIN(value)", + Aggregation::Max => "MAX(value)", + Aggregation::Sum => "SUM(value)", + Aggregation::First | Aggregation::Last | Aggregation::Count => { + unreachable!("handled separately") + } + } +} + +pub(super) fn sqlite_numeric_expression(aggregation: Aggregation) -> &'static str { + match aggregation { + Aggregation::Avg => "AVG(value)", + Aggregation::Min => "MIN(value)", + Aggregation::Max => "MAX(value)", + Aggregation::Sum => "SUM(value)", + Aggregation::First | Aggregation::Last | Aggregation::Count => { + unreachable!("handled separately") + } + } +} diff --git a/src/storage/timescaledb/mod.rs b/src/storage/timescaledb/mod.rs index feb4ac6f..3ea0662d 100644 --- a/src/storage/timescaledb/mod.rs +++ b/src/storage/timescaledb/mod.rs @@ -3,16 +3,24 @@ pub mod timescaledb_utilities; use self::timescaledb_publishers::*; use self::timescaledb_utilities::get_sensor_id_or_create_sensor; -use super::{DEFAULT_QUERY_LIMIT, StorageError, StorageInstance}; +use super::{ + Aggregation, DEFAULT_LIST_SERIES_LIMIT, DEFAULT_QUERY_LIMIT, MAX_LIST_SERIES_LIMIT, + SensorAvailabilitySummary, SensorDataQueryOptions, StorageError, StorageInstance, + common::datetime_to_micros, +}; +use crate::datamodel::sensapp_datetime::SensAppDateTimeExt; use crate::datamodel::{ - SensAppDateTime, Sensor, SensorData, SensorType, TypedSamples, batch::Batch, + Sample, SensAppDateTime, Sensor, SensorData, SensorType, TypedSamples, batch::Batch, }; use crate::datamodel::{sensapp_vec::SensAppLabels, unit::Unit}; use anyhow::{Context, Result}; use async_trait::async_trait; use smallvec::smallvec; -use sqlx::{PgPool, postgres::PgConnectOptions}; -use std::{str::FromStr, sync::Arc}; +use sqlx::{ + PgPool, + postgres::{PgConnectOptions, PgPoolOptions}, +}; +use std::{collections::HashMap, str::FromStr, sync::Arc}; use uuid::Uuid; #[derive(Debug)] @@ -20,6 +28,15 @@ pub struct TimeScaleDBStorage { pool: PgPool, } +fn micros_to_offset_datetime(timestamp_us: i64) -> sqlx::types::time::OffsetDateTime { + sqlx::types::time::OffsetDateTime::from_unix_timestamp_nanos((timestamp_us as i128) * 1000) + .unwrap_or(sqlx::types::time::OffsetDateTime::UNIX_EPOCH) +} + +fn offset_datetime_to_sensapp(datetime: sqlx::types::time::OffsetDateTime) -> SensAppDateTime { + SensAppDateTime::from_unix_microseconds_i64((datetime.unix_timestamp_nanos() / 1000) as i64) +} + impl TimeScaleDBStorage { pub async fn connect(connection_string: &str) -> Result { // Convert timescaledb:// to postgres:// for sqlx compatibility @@ -32,12 +49,495 @@ impl TimeScaleDBStorage { let connect_options = PgConnectOptions::from_str(&postgres_connection_string) .context("Failed to create timescaledb connection options")?; - let pool = PgPool::connect_with(connect_options) + let max_connections = std::env::var("SENSAPP_PG_POOL_MAX_CONNECTIONS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(10); + + let pool = PgPoolOptions::new() + .max_connections(max_connections) + .connect_with(connect_options) .await .context("Failed to create timescaledb pool")?; Ok(Self { pool }) } + + async fn find_sensors_by_matchers( + &self, + name_matchers: &[&super::LabelMatcher], + label_matchers: &[&super::LabelMatcher], + numeric_only: bool, + ) -> Result> { + let mut sql = String::from( + r#"SELECT DISTINCT s.sensor_id, s.uuid, s.name, s.type, + u.name as unit_name, u.description as unit_description + FROM sensors s + LEFT JOIN units u ON s.unit = u.id"#, + ); + let mut where_clauses: Vec = Vec::new(); + let mut params: Vec = Vec::new(); + let mut param_idx = 1; + + if numeric_only { + where_clauses.push("s.type IN ('Integer', 'Numeric', 'Float')".to_string()); + } + + for matcher in name_matchers { + let clause = match matcher.matcher_type { + super::MatcherType::Equal => { + params.push(matcher.value.clone()); + let clause = format!("s.name = ${}", param_idx); + param_idx += 1; + clause + } + super::MatcherType::NotEqual => { + params.push(matcher.value.clone()); + let clause = format!("s.name != ${}", param_idx); + param_idx += 1; + clause + } + super::MatcherType::RegexMatch => { + params.push(matcher.value.clone()); + let clause = format!("s.name ~ ${}", param_idx); + param_idx += 1; + clause + } + super::MatcherType::RegexNotMatch => { + params.push(matcher.value.clone()); + let clause = format!("s.name !~ ${}", param_idx); + param_idx += 1; + clause + } + }; + where_clauses.push(clause); + } + + for matcher in label_matchers { + let subquery = match matcher.matcher_type { + super::MatcherType::Equal => { + params.push(matcher.name.clone()); + params.push(matcher.value.clone()); + let subquery = format!( + r#"s.sensor_id IN ( + SELECT l.sensor_id FROM labels l + JOIN labels_name_dictionary lnd ON l.name = lnd.id + JOIN labels_description_dictionary ldd ON l.description = ldd.id + WHERE lnd.name = ${} AND ldd.description = ${} + )"#, + param_idx, + param_idx + 1 + ); + param_idx += 2; + subquery + } + super::MatcherType::NotEqual => { + params.push(matcher.name.clone()); + params.push(matcher.value.clone()); + let subquery = format!( + r#"s.sensor_id NOT IN ( + SELECT l.sensor_id FROM labels l + JOIN labels_name_dictionary lnd ON l.name = lnd.id + JOIN labels_description_dictionary ldd ON l.description = ldd.id + WHERE lnd.name = ${} AND ldd.description = ${} + )"#, + param_idx, + param_idx + 1 + ); + param_idx += 2; + subquery + } + super::MatcherType::RegexMatch => { + params.push(matcher.name.clone()); + params.push(matcher.value.clone()); + let subquery = format!( + r#"s.sensor_id IN ( + SELECT l.sensor_id FROM labels l + JOIN labels_name_dictionary lnd ON l.name = lnd.id + JOIN labels_description_dictionary ldd ON l.description = ldd.id + WHERE lnd.name = ${} AND ldd.description ~ ${} + )"#, + param_idx, + param_idx + 1 + ); + param_idx += 2; + subquery + } + super::MatcherType::RegexNotMatch => { + params.push(matcher.name.clone()); + params.push(matcher.value.clone()); + let subquery = format!( + r#"s.sensor_id NOT IN ( + SELECT l.sensor_id FROM labels l + JOIN labels_name_dictionary lnd ON l.name = lnd.id + JOIN labels_description_dictionary ldd ON l.description = ldd.id + WHERE lnd.name = ${} AND ldd.description ~ ${} + )"#, + param_idx, + param_idx + 1 + ); + param_idx += 2; + subquery + } + }; + where_clauses.push(subquery); + } + + if !where_clauses.is_empty() { + sql.push_str(" WHERE "); + sql.push_str(&where_clauses.join(" AND ")); + } + sql.push_str(" ORDER BY s.sensor_id"); + + #[derive(sqlx::FromRow)] + struct SensorRow { + sensor_id: i64, + uuid: Uuid, + name: String, + r#type: String, + unit_name: Option, + unit_description: Option, + } + + let mut query = sqlx::query_as::<_, SensorRow>(&sql); + for param in ¶ms { + query = query.bind(param); + } + + let sensor_rows = query.fetch_all(&self.pool).await?; + + if sensor_rows.is_empty() { + return Ok(Vec::new()); + } + + let sensor_ids: Vec = sensor_rows.iter().map(|row| row.sensor_id).collect(); + + #[derive(sqlx::FromRow)] + struct LabelRow { + sensor_id: i64, + label_name: String, + label_value: String, + } + + let labels_rows: Vec = sqlx::query_as( + r#" + SELECT l.sensor_id, lnd.name as label_name, ldd.description as label_value + FROM labels l + JOIN labels_name_dictionary lnd ON l.name = lnd.id + JOIN labels_description_dictionary ldd ON l.description = ldd.id + WHERE l.sensor_id = ANY($1) + ORDER BY l.sensor_id + "#, + ) + .bind(&sensor_ids) + .fetch_all(&self.pool) + .await?; + + let mut labels_map: HashMap = HashMap::new(); + for label_row in labels_rows { + labels_map + .entry(label_row.sensor_id) + .or_insert_with(|| smallvec![]) + .push((label_row.label_name, label_row.label_value)); + } + + let mut results = Vec::with_capacity(sensor_rows.len()); + for row in sensor_rows { + let sensor_type = SensorType::from_str(&row.r#type).map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Failed to parse sensor type '{}': {}", row.r#type, e), + Some(row.uuid), + Some(&row.name), + )) + })?; + + let unit = match (row.unit_name, row.unit_description) { + (Some(name), description) => Some(Unit::new(name, description)), + _ => None, + }; + + let labels = labels_map.remove(&row.sensor_id).unwrap_or_default(); + let sensor = Sensor::new(row.uuid, row.name, sensor_type, unit, Some(labels)); + results.push((row.sensor_id, sensor)); + } + + Ok(results) + } + + async fn get_sensor_metadata(&self, sensor_uuid: &str) -> Result> { + let parsed_uuid = Uuid::from_str(sensor_uuid).context("Failed to parse sensor UUID")?; + + #[derive(sqlx::FromRow)] + struct SensorMetadataRow { + sensor_id: Option, + uuid: Option, + name: Option, + r#type: Option, + unit_name: Option, + unit_description: Option, + } + + let sensor_row: Option = sqlx::query_as( + r#" + SELECT sensor_id, uuid, name, type, unit_name, unit_description + FROM sensor_catalog_view + WHERE uuid = $1 + "#, + ) + .bind(parsed_uuid) + .fetch_optional(&self.pool) + .await?; + + let Some(sensor_row) = sensor_row else { + return Ok(None); + }; + + let sensor_uuid = sensor_row.uuid.ok_or_else(|| { + anyhow::Error::from(StorageError::missing_field( + "UUID", + None, + sensor_row.name.as_deref(), + )) + })?; + + let sensor_name = sensor_row.name.ok_or_else(|| { + anyhow::Error::from(StorageError::missing_field("name", Some(sensor_uuid), None)) + })?; + + let sensor_type_str = sensor_row.r#type.ok_or_else(|| { + anyhow::Error::from(StorageError::missing_field( + "type", + Some(sensor_uuid), + Some(&sensor_name), + )) + })?; + + let sensor_type = SensorType::from_str(&sensor_type_str).map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Failed to parse sensor type '{}': {}", sensor_type_str, e), + Some(sensor_uuid), + Some(&sensor_name), + )) + })?; + + let unit = match (sensor_row.unit_name, sensor_row.unit_description) { + (Some(name), description) => Some(Unit::new(name, description)), + _ => None, + }; + + let sensor_id = sensor_row.sensor_id.ok_or_else(|| { + anyhow::Error::from(StorageError::missing_field( + "sensor_id", + Some(sensor_uuid), + Some(&sensor_name), + )) + })?; + + #[derive(sqlx::FromRow)] + struct LabelRow { + label_name: String, + label_value: String, + } + + let labels_rows: Vec = sqlx::query_as( + r#" + SELECT lnd.name as label_name, ldd.description as label_value + FROM labels l + JOIN labels_name_dictionary lnd ON l.name = lnd.id + JOIN labels_description_dictionary ldd ON l.description = ldd.id + WHERE l.sensor_id = $1 + "#, + ) + .bind(sensor_id) + .fetch_all(&self.pool) + .await?; + + let mut labels: SensAppLabels = smallvec![]; + for label_row in labels_rows { + labels.push((label_row.label_name, label_row.label_value)); + } + + Ok(Some(( + sensor_id, + Sensor::new(sensor_uuid, sensor_name, sensor_type, unit, Some(labels)), + ))) + } + + fn sensor_table_name(sensor_type: SensorType) -> &'static str { + match sensor_type { + SensorType::Integer => "integer_values", + SensorType::Numeric => "numeric_values", + SensorType::Float => "float_values", + SensorType::String => "string_values", + SensorType::Boolean => "boolean_values", + SensorType::Location => "location_values", + SensorType::Json => "json_values", + SensorType::Blob => "blob_values", + } + } + + async fn query_latest_timestamp_us( + &self, + table_name: &str, + sensor_id: i64, + start_time: Option, + end_time: Option, + ) -> Result> { + let sql = format!( + r#" + SELECT (EXTRACT(EPOCH FROM MAX(time)) * 1000000)::bigint AS timestamp_us + FROM {table_name} + WHERE sensor_id = $1 + AND ($2::TIMESTAMPTZ IS NULL OR time >= $2) + AND ($3::TIMESTAMPTZ IS NULL OR time <= $3) + "# + ); + + let timestamp_us: Option = sqlx::query_scalar(&sql) + .bind(sensor_id) + .bind(start_time.map(micros_to_offset_datetime)) + .bind(end_time.map(micros_to_offset_datetime)) + .fetch_one(&self.pool) + .await?; + + Ok(timestamp_us) + } + + async fn query_availability_summary_native( + &self, + table_name: &str, + sensor_id: i64, + sensor: Sensor, + start_time: SensAppDateTime, + end_time: SensAppDateTime, + step_ms: Option, + ) -> Result { + #[derive(sqlx::FromRow)] + struct Row { + sample_count: i64, + first_sample_at: Option, + last_sample_at: Option, + covered_buckets: Option, + } + + let start_ts = micros_to_offset_datetime(datetime_to_micros(&start_time)); + let end_ts = micros_to_offset_datetime(datetime_to_micros(&end_time)); + + let row: Row = if let Some(step_ms) = step_ms { + let sql = format!( + r#" + SELECT + COUNT(*)::bigint AS sample_count, + MIN(time) AS first_sample_at, + MAX(time) AS last_sample_at, + COUNT(DISTINCT time_bucket($1::bigint * INTERVAL '1 millisecond', time, $2::timestamptz))::bigint AS covered_buckets + FROM {table_name} + WHERE sensor_id = $3 + AND time >= $4 + AND time <= $5 + "# + ); + + sqlx::query_as(&sql) + .bind(step_ms) + .bind(start_ts) + .bind(sensor_id) + .bind(start_ts) + .bind(end_ts) + .fetch_one(&self.pool) + .await? + } else { + let sql = format!( + r#" + SELECT + COUNT(*)::bigint AS sample_count, + MIN(time) AS first_sample_at, + MAX(time) AS last_sample_at, + NULL::bigint AS covered_buckets + FROM {table_name} + WHERE sensor_id = $1 + AND time >= $2 + AND time <= $3 + "# + ); + + sqlx::query_as(&sql) + .bind(sensor_id) + .bind(start_ts) + .bind(end_ts) + .fetch_one(&self.pool) + .await? + }; + + Ok(SensorAvailabilitySummary { + sensor, + sample_count: row.sample_count.max(0) as usize, + first_sample_at: row.first_sample_at.map(offset_datetime_to_sensapp), + last_sample_at: row.last_sample_at.map(offset_datetime_to_sensapp), + covered_buckets: row.covered_buckets.map(|value| value.max(0) as usize), + }) + } +} + +fn timescaledb_bucketed_cte(table_name: &str) -> String { + format!( + r#" + WITH bucketed AS ( + SELECT + time_bucket( + $4::bigint * INTERVAL '1 millisecond', + time, + to_timestamp($5::double precision / 1000000.0) + ) AS bucket_time, + time, + value + FROM {table_name} + WHERE sensor_id = $1 + AND ($2::TIMESTAMPTZ IS NULL OR time >= $2) + AND ($3::TIMESTAMPTZ IS NULL OR time <= $3) + ) + "# + ) +} + +fn timescaledb_group_by_clause() -> &'static str { + "FROM bucketed GROUP BY 1 ORDER BY 1 ASC LIMIT $6" +} + +fn timescaledb_integer_expression(aggregation: Aggregation) -> &'static str { + match aggregation { + Aggregation::Min => "MIN(value)", + Aggregation::Max => "MAX(value)", + Aggregation::Sum => "SUM(value)", + Aggregation::First => "first(value, time)", + Aggregation::Last => "last(value, time)", + Aggregation::Avg | Aggregation::Count => unreachable!("handled separately"), + } +} + +fn timescaledb_float_expression(aggregation: Aggregation) -> &'static str { + match aggregation { + Aggregation::Avg => "AVG(value)", + Aggregation::Min => "MIN(value)", + Aggregation::Max => "MAX(value)", + Aggregation::Sum => "SUM(value)", + Aggregation::First => "first(value, time)", + Aggregation::Last => "last(value, time)", + Aggregation::Count => unreachable!("handled separately"), + } +} + +fn timescaledb_numeric_expression(aggregation: Aggregation) -> &'static str { + match aggregation { + Aggregation::Avg => "AVG(value)", + Aggregation::Min => "MIN(value)", + Aggregation::Max => "MAX(value)", + Aggregation::Sum => "SUM(value)", + Aggregation::First => "first(value, time)", + Aggregation::Last => "last(value, time)", + Aggregation::Count => unreachable!("handled separately"), + } } #[async_trait] @@ -68,7 +568,9 @@ impl StorageInstance for TimeScaleDBStorage { async fn list_series( &self, metric_filter: Option<&str>, - ) -> Result> { + limit: Option, + bookmark: Option<&str>, + ) -> Result { #[derive(sqlx::FromRow)] struct SensorRow { sensor_id: Option, @@ -79,22 +581,45 @@ impl StorageInstance for TimeScaleDBStorage { unit_description: Option, } + let bookmark_id = if let Some(bookmark_str) = bookmark { + Some(bookmark_str.parse::().map_err(|e| { + anyhow::Error::from(StorageError::invalid_data_format( + &format!("Invalid bookmark format: {}", e), + None, + None, + )) + })?) + } else { + None + }; + + let effective_limit = limit + .unwrap_or(DEFAULT_LIST_SERIES_LIMIT) + .min(MAX_LIST_SERIES_LIMIT); + let fetch_limit = effective_limit.saturating_add(1); + // Query sensors with their metadata using the catalog view, optionally filtered by metric name let sensor_rows: Vec = sqlx::query_as( r#" SELECT sensor_id, uuid, name, type, unit_name, unit_description FROM sensor_catalog_view WHERE ($1::TEXT IS NULL OR name = $1) - ORDER BY uuid ASC + AND ($2::BIGINT IS NULL OR sensor_id > $2) + ORDER BY sensor_id ASC + LIMIT $3 "#, ) .bind(metric_filter) + .bind(bookmark_id) + .bind(fetch_limit as i64) .fetch_all(&self.pool) .await?; - let mut sensors = Vec::new(); + let has_more = sensor_rows.len() > effective_limit; + let mut last_sensor_id = None; + let mut sensors = Vec::with_capacity(sensor_rows.len().min(effective_limit)); - for sensor_row in sensor_rows { + for sensor_row in sensor_rows.into_iter().take(effective_limit) { // Parse sensor metadata with improved error handling let sensor_uuid = sensor_row .uuid @@ -149,6 +674,7 @@ impl StorageInstance for TimeScaleDBStorage { Some(&sensor_name), )) })?; + last_sensor_id = Some(sensor_id); #[derive(sqlx::FromRow)] struct LabelRow { @@ -191,7 +717,14 @@ impl StorageInstance for TimeScaleDBStorage { sensors.push(sensor); } - Ok(sensors) + Ok(crate::storage::ListSeriesResult { + series: sensors, + bookmark: if has_more { + last_sensor_id.map(|sensor_id| sensor_id.to_string()) + } else { + None + }, + }) } async fn list_metrics(&self) -> Result> { @@ -382,17 +915,8 @@ impl StorageInstance for TimeScaleDBStorage { let sensor = Sensor::new(sensor_uuid, sensor_name, sensor_type, unit, Some(labels)); - // Convert SensAppDateTime to microseconds using a custom function for TimescaleDB - let start_time_us = start_time.as_ref().map(|dt| { - let unix_seconds = dt.to_unix_seconds(); - let subsec_nanos = dt.to_et_duration().total_nanoseconds() % 1_000_000_000; - (unix_seconds as i64) * 1_000_000 + (subsec_nanos / 1000) as i64 - }); - let end_time_us = end_time.as_ref().map(|dt| { - let unix_seconds = dt.to_unix_seconds(); - let subsec_nanos = dt.to_et_duration().total_nanoseconds() % 1_000_000_000; - (unix_seconds as i64) * 1_000_000 + (subsec_nanos / 1000) as i64 - }); + let start_time_us = start_time.as_ref().map(datetime_to_micros); + let end_time_us = end_time.as_ref().map(datetime_to_micros); // Query samples based on sensor type let samples = match sensor.sensor_type { @@ -433,16 +957,307 @@ impl StorageInstance for TimeScaleDBStorage { Ok(Some(SensorData::new(sensor, samples))) } + async fn query_sensor_data_advanced( + &self, + sensor_uuid: &str, + options: &SensorDataQueryOptions, + ) -> Result> { + options.validate()?; + + if let (Some(step_ms), Some(aggregation)) = (options.step_ms, options.aggregation) { + if let Some((sensor_id, mut sensor)) = self.get_sensor_metadata(sensor_uuid).await? { + let start_time_us = options.start_time.as_ref().map(datetime_to_micros); + let end_time_us = options.end_time.as_ref().map(datetime_to_micros); + let origin_us = start_time_us.unwrap_or(0); + + let samples = match sensor.sensor_type { + SensorType::Integer => { + self.query_integer_samples_aggregated( + sensor_id, + start_time_us, + end_time_us, + step_ms, + origin_us, + aggregation, + options.limit, + ) + .await? + } + SensorType::Numeric => { + self.query_numeric_samples_aggregated( + sensor_id, + start_time_us, + end_time_us, + step_ms, + origin_us, + aggregation, + options.limit, + ) + .await? + } + SensorType::Float => { + self.query_float_samples_aggregated( + sensor_id, + start_time_us, + end_time_us, + step_ms, + origin_us, + aggregation, + options.limit, + ) + .await? + } + _ => { + let raw = self + .query_sensor_data( + sensor_uuid, + options.start_time, + options.end_time, + options.limit, + ) + .await?; + return raw + .map(|sensor_data| { + crate::storage::common::apply_query_options(sensor_data, options) + }) + .transpose(); + } + }; + + sensor.sensor_type = match &samples { + TypedSamples::Integer(_) => SensorType::Integer, + TypedSamples::Numeric(_) => SensorType::Numeric, + TypedSamples::Float(_) => SensorType::Float, + _ => sensor.sensor_type, + }; + if aggregation.output_is_count() { + sensor.unit = None; + } + + let post_query_options = SensorDataQueryOptions { + start_time: options.start_time, + end_time: options.end_time, + limit: options.limit, + step_ms: None, + aggregation: None, + simplify: options.simplify, + }; + + return crate::storage::common::apply_query_options( + SensorData::new(sensor, samples), + &post_query_options, + ) + .map(Some); + } + + return Ok(None); + } + + let raw = self + .query_sensor_data( + sensor_uuid, + options.start_time, + options.end_time, + options.limit, + ) + .await?; + + raw.map(|sensor_data| crate::storage::common::apply_query_options(sensor_data, options)) + .transpose() + } + + async fn query_sensor_data_latest( + &self, + sensor_uuid: &str, + start_time: Option, + end_time: Option, + ) -> Result> { + let Some((sensor_id, sensor)) = self.get_sensor_metadata(sensor_uuid).await? else { + return Ok(None); + }; + + let start_time_us = start_time.as_ref().map(datetime_to_micros); + let end_time_us = end_time.as_ref().map(datetime_to_micros); + let table_name = Self::sensor_table_name(sensor.sensor_type); + + let Some(latest_timestamp_us) = self + .query_latest_timestamp_us(table_name, sensor_id, start_time_us, end_time_us) + .await? + else { + return Ok(None); + }; + + let samples = match sensor.sensor_type { + SensorType::Integer => { + self.query_integer_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Numeric => { + self.query_numeric_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Float => { + self.query_float_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::String => { + self.query_string_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Boolean => { + self.query_boolean_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Location => { + self.query_location_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Json => { + self.query_json_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + SensorType::Blob => { + self.query_blob_samples( + sensor_id, + Some(latest_timestamp_us), + Some(latest_timestamp_us), + Some(1), + ) + .await? + } + }; + + Ok(Some(SensorData::new(sensor, samples))) + } + + async fn query_sensor_data_availability( + &self, + sensor_uuid: &str, + start_time: SensAppDateTime, + end_time: SensAppDateTime, + step_ms: Option, + ) -> Result> { + let Some((sensor_id, sensor)) = self.get_sensor_metadata(sensor_uuid).await? else { + return Ok(None); + }; + + let summary = self + .query_availability_summary_native( + Self::sensor_table_name(sensor.sensor_type), + sensor_id, + sensor, + start_time, + end_time, + step_ms, + ) + .await?; + + Ok(Some(summary)) + } + async fn query_sensors_by_labels( &self, - _matchers: &[super::LabelMatcher], - _start_time: Option, - _end_time: Option, - _limit: Option, - _numeric_only: bool, + matchers: &[super::LabelMatcher], + start_time: Option, + end_time: Option, + limit: Option, + numeric_only: bool, ) -> Result> { - // TODO: Implement label-based query for TimescaleDB - anyhow::bail!("query_sensors_by_labels not yet implemented for TimescaleDB") + if matchers.is_empty() { + return Ok(Vec::new()); + } + + let (name_matchers, label_matchers): (Vec<_>, Vec<_>) = matchers + .iter() + .partition(|matcher| matcher.is_name_matcher()); + + let sensors = self + .find_sensors_by_matchers(&name_matchers, &label_matchers, numeric_only) + .await?; + + if sensors.is_empty() { + return Ok(Vec::new()); + } + + let start_time_us = start_time.as_ref().map(datetime_to_micros); + let end_time_us = end_time.as_ref().map(datetime_to_micros); + + let mut results = Vec::with_capacity(sensors.len()); + for (sensor_id, sensor) in sensors { + let samples = match sensor.sensor_type { + SensorType::Integer => { + self.query_integer_samples(sensor_id, start_time_us, end_time_us, limit) + .await? + } + SensorType::Numeric => { + self.query_numeric_samples(sensor_id, start_time_us, end_time_us, limit) + .await? + } + SensorType::Float => { + self.query_float_samples(sensor_id, start_time_us, end_time_us, limit) + .await? + } + SensorType::String => { + self.query_string_samples(sensor_id, start_time_us, end_time_us, limit) + .await? + } + SensorType::Boolean => { + self.query_boolean_samples(sensor_id, start_time_us, end_time_us, limit) + .await? + } + SensorType::Location => { + self.query_location_samples(sensor_id, start_time_us, end_time_us, limit) + .await? + } + SensorType::Json => { + self.query_json_samples(sensor_id, start_time_us, end_time_us, limit) + .await? + } + SensorType::Blob => { + self.query_blob_samples(sensor_id, start_time_us, end_time_us, limit) + .await? + } + }; + + results.push(SensorData::new(sensor, samples)); + } + + Ok(results) } /// Health check for TimescaleDB storage @@ -587,6 +1402,276 @@ impl TimeScaleDBStorage { Ok(()) } + #[allow(clippy::too_many_arguments)] + async fn query_integer_samples_aggregated( + &self, + sensor_id: i64, + start_time: Option, + end_time: Option, + step_ms: i64, + origin_us: i64, + aggregation: Aggregation, + limit: Option, + ) -> Result { + let start_time_ts = start_time.map(micros_to_offset_datetime); + let end_time_ts = end_time.map(micros_to_offset_datetime); + let limit = limit.unwrap_or(DEFAULT_QUERY_LIMIT) as i64; + + match aggregation { + Aggregation::Avg => { + #[derive(sqlx::FromRow)] + struct Row { + bucket_time: sqlx::types::time::OffsetDateTime, + value: f64, + } + + let rows: Vec = sqlx::query_as(&format!( + "{} SELECT bucket_time, AVG(value)::double precision AS value {}", + timescaledb_bucketed_cte("integer_values"), + timescaledb_group_by_clause() + )) + .bind(sensor_id) + .bind(start_time_ts) + .bind(end_time_ts) + .bind(step_ms) + .bind(origin_us as f64) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: offset_datetime_to_sensapp(row.bucket_time), + value: row.value, + }); + } + Ok(TypedSamples::Float(samples)) + } + Aggregation::Count => { + #[derive(sqlx::FromRow)] + struct Row { + bucket_time: sqlx::types::time::OffsetDateTime, + value: i64, + } + + let rows: Vec = sqlx::query_as(&format!( + "{} SELECT bucket_time, COUNT(*)::bigint AS value {}", + timescaledb_bucketed_cte("integer_values"), + timescaledb_group_by_clause() + )) + .bind(sensor_id) + .bind(start_time_ts) + .bind(end_time_ts) + .bind(step_ms) + .bind(origin_us as f64) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: offset_datetime_to_sensapp(row.bucket_time), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + _ => { + #[derive(sqlx::FromRow)] + struct Row { + bucket_time: sqlx::types::time::OffsetDateTime, + value: i64, + } + + let rows: Vec = sqlx::query_as(&format!( + "{} SELECT bucket_time, {} AS value {}", + timescaledb_bucketed_cte("integer_values"), + timescaledb_integer_expression(aggregation), + timescaledb_group_by_clause() + )) + .bind(sensor_id) + .bind(start_time_ts) + .bind(end_time_ts) + .bind(step_ms) + .bind(origin_us as f64) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: offset_datetime_to_sensapp(row.bucket_time), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + } + } + + #[allow(clippy::too_many_arguments)] + async fn query_float_samples_aggregated( + &self, + sensor_id: i64, + start_time: Option, + end_time: Option, + step_ms: i64, + origin_us: i64, + aggregation: Aggregation, + limit: Option, + ) -> Result { + let start_time_ts = start_time.map(micros_to_offset_datetime); + let end_time_ts = end_time.map(micros_to_offset_datetime); + let limit = limit.unwrap_or(DEFAULT_QUERY_LIMIT) as i64; + + match aggregation { + Aggregation::Count => { + #[derive(sqlx::FromRow)] + struct Row { + bucket_time: sqlx::types::time::OffsetDateTime, + value: i64, + } + + let rows: Vec = sqlx::query_as(&format!( + "{} SELECT bucket_time, COUNT(*)::bigint AS value {}", + timescaledb_bucketed_cte("float_values"), + timescaledb_group_by_clause() + )) + .bind(sensor_id) + .bind(start_time_ts) + .bind(end_time_ts) + .bind(step_ms) + .bind(origin_us as f64) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: offset_datetime_to_sensapp(row.bucket_time), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + _ => { + #[derive(sqlx::FromRow)] + struct Row { + bucket_time: sqlx::types::time::OffsetDateTime, + value: f64, + } + + let rows: Vec = sqlx::query_as(&format!( + "{} SELECT bucket_time, {} AS value {}", + timescaledb_bucketed_cte("float_values"), + timescaledb_float_expression(aggregation), + timescaledb_group_by_clause() + )) + .bind(sensor_id) + .bind(start_time_ts) + .bind(end_time_ts) + .bind(step_ms) + .bind(origin_us as f64) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: offset_datetime_to_sensapp(row.bucket_time), + value: row.value, + }); + } + Ok(TypedSamples::Float(samples)) + } + } + } + + #[allow(clippy::too_many_arguments)] + async fn query_numeric_samples_aggregated( + &self, + sensor_id: i64, + start_time: Option, + end_time: Option, + step_ms: i64, + origin_us: i64, + aggregation: Aggregation, + limit: Option, + ) -> Result { + let start_time_ts = start_time.map(micros_to_offset_datetime); + let end_time_ts = end_time.map(micros_to_offset_datetime); + let limit = limit.unwrap_or(DEFAULT_QUERY_LIMIT) as i64; + + match aggregation { + Aggregation::Count => { + #[derive(sqlx::FromRow)] + struct Row { + bucket_time: sqlx::types::time::OffsetDateTime, + value: i64, + } + + let rows: Vec = sqlx::query_as(&format!( + "{} SELECT bucket_time, COUNT(*)::bigint AS value {}", + timescaledb_bucketed_cte("numeric_values"), + timescaledb_group_by_clause() + )) + .bind(sensor_id) + .bind(start_time_ts) + .bind(end_time_ts) + .bind(step_ms) + .bind(origin_us as f64) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: offset_datetime_to_sensapp(row.bucket_time), + value: row.value, + }); + } + Ok(TypedSamples::Integer(samples)) + } + _ => { + #[derive(sqlx::FromRow)] + struct Row { + bucket_time: sqlx::types::time::OffsetDateTime, + value: rust_decimal::Decimal, + } + + let rows: Vec = sqlx::query_as(&format!( + "{} SELECT bucket_time, {} AS value {}", + timescaledb_bucketed_cte("numeric_values"), + timescaledb_numeric_expression(aggregation), + timescaledb_group_by_clause() + )) + .bind(sensor_id) + .bind(start_time_ts) + .bind(end_time_ts) + .bind(step_ms) + .bind(origin_us as f64) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut samples = smallvec![]; + for row in rows { + samples.push(Sample { + datetime: offset_datetime_to_sensapp(row.bucket_time), + value: row.value, + }); + } + Ok(TypedSamples::Numeric(samples)) + } + } + } + async fn query_integer_samples( &self, sensor_id: i64, @@ -594,7 +1679,7 @@ impl TimeScaleDBStorage { end_time: Option, limit: Option, ) -> Result { - use crate::datamodel::{Sample, SensAppDateTime}; + use crate::datamodel::Sample; use smallvec::smallvec; #[derive(sqlx::FromRow)] @@ -604,14 +1689,8 @@ impl TimeScaleDBStorage { } // Convert microsecond timestamps to OffsetDateTime for TimescaleDB queries - let start_time_ts = start_time.map(|t| { - sqlx::types::time::OffsetDateTime::from_unix_timestamp_nanos((t * 1000) as i128) - .unwrap_or(sqlx::types::time::OffsetDateTime::UNIX_EPOCH) - }); - let end_time_ts = end_time.map(|t| { - sqlx::types::time::OffsetDateTime::from_unix_timestamp_nanos((t * 1000) as i128) - .unwrap_or(sqlx::types::time::OffsetDateTime::UNIX_EPOCH) - }); + let start_time_ts = start_time.map(micros_to_offset_datetime); + let end_time_ts = end_time.map(micros_to_offset_datetime); let rows: Vec = sqlx::query_as( r#" @@ -632,10 +1711,7 @@ impl TimeScaleDBStorage { let mut samples = smallvec![]; for row in rows { - // Convert OffsetDateTime back to SensAppDateTime - let unix_timestamp = - row.time.unix_timestamp() as f64 + (row.time.nanosecond() as f64 / 1_000_000_000.0); - let datetime = SensAppDateTime::from_unix_seconds(unix_timestamp); + let datetime = offset_datetime_to_sensapp(row.time); let value = row.value; samples.push(Sample { datetime, value }); } @@ -706,7 +1782,7 @@ impl TimeScaleDBStorage { end_time: Option, limit: Option, ) -> Result { - use crate::datamodel::{Sample, SensAppDateTime}; + use crate::datamodel::Sample; use smallvec::smallvec; #[derive(sqlx::FromRow)] @@ -716,14 +1792,8 @@ impl TimeScaleDBStorage { } // Convert microsecond timestamps to OffsetDateTime for TimescaleDB queries - let start_time_ts = start_time.map(|t| { - sqlx::types::time::OffsetDateTime::from_unix_timestamp_nanos((t * 1000) as i128) - .unwrap_or(sqlx::types::time::OffsetDateTime::UNIX_EPOCH) - }); - let end_time_ts = end_time.map(|t| { - sqlx::types::time::OffsetDateTime::from_unix_timestamp_nanos((t * 1000) as i128) - .unwrap_or(sqlx::types::time::OffsetDateTime::UNIX_EPOCH) - }); + let start_time_ts = start_time.map(micros_to_offset_datetime); + let end_time_ts = end_time.map(micros_to_offset_datetime); let rows: Vec = sqlx::query_as( r#" @@ -744,10 +1814,7 @@ impl TimeScaleDBStorage { let mut samples = smallvec![]; for row in rows { - // Convert OffsetDateTime back to SensAppDateTime - let unix_timestamp = - row.time.unix_timestamp() as f64 + (row.time.nanosecond() as f64 / 1_000_000_000.0); - let datetime = SensAppDateTime::from_unix_seconds(unix_timestamp); + let datetime = offset_datetime_to_sensapp(row.time); let value = row.value; samples.push(Sample { datetime, value }); } diff --git a/src/storage/timescaledb/timescaledb_publishers.rs b/src/storage/timescaledb/timescaledb_publishers.rs index 8fed0c06..ad95b66c 100644 --- a/src/storage/timescaledb/timescaledb_publishers.rs +++ b/src/storage/timescaledb/timescaledb_publishers.rs @@ -31,7 +31,6 @@ pub async fn publish_numeric_values( ) -> Result<()> { for value in values { let time = sensapp_datetime_to_offset_datetime(&value.datetime)?; - let string_value = value.value.to_string(); let query = sqlx::query( r#" INSERT INTO numeric_values (sensor_id, time, value) @@ -40,7 +39,7 @@ pub async fn publish_numeric_values( ) .bind(sensor_id) .bind(time) - .bind(string_value); + .bind(value.value); transaction.execute(query).await?; } Ok(()) diff --git a/src/storage/timescaledb/timescaledb_utilities.rs b/src/storage/timescaledb/timescaledb_utilities.rs index d95eb19c..dfb4a1a6 100644 --- a/src/storage/timescaledb/timescaledb_utilities.rs +++ b/src/storage/timescaledb/timescaledb_utilities.rs @@ -3,7 +3,6 @@ use crate::datamodel::unit::Unit; use anyhow::Result; use cached::proc_macro::cached; use sqlx::{Executor, Postgres, Row, Transaction}; -use std::time::Duration; use uuid::Uuid; /** diff --git a/src/test_utils.rs b/src/test_utils.rs index 2ed3eac5..785850ab 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -3,14 +3,38 @@ //! This module provides centralized test configuration and utilities, //! particularly for database connection management. +use anyhow::Result; +#[cfg(any(feature = "postgres", feature = "timescaledb"))] +use anyhow::anyhow; +#[cfg(any(feature = "postgres", feature = "timescaledb"))] +use sqlx::postgres::PgPoolOptions; +use std::path::{Path, PathBuf}; +use url::Url; + /// Default PostgreSQL connection string for tests const DEFAULT_POSTGRES_CONNECTION_STRING: &str = "postgres://postgres:postgres@localhost:5432/sensapp-test"; -/// Get the test database connection string from environment or use default PostgreSQL +/// Default TimeScaleDB connection string for tests +const DEFAULT_TIMESCALEDB_CONNECTION_STRING: &str = + "timescaledb://postgres:postgres@localhost:5433/sensapp-test"; + +/// Default SQLite connection string for tests +const DEFAULT_SQLITE_CONNECTION_STRING: &str = "sqlite://test.db"; + +/// Default DuckDB connection string for tests +const DEFAULT_DUCKDB_CONNECTION_STRING: &str = "duckdb://test.duckdb"; + +/// Default ClickHouse connection string for tests +const DEFAULT_CLICKHOUSE_CONNECTION_STRING: &str = + "clickhouse://default:password@localhost:8123/sensapp_test"; + +/// Get the test database connection string from environment or use the default +/// connection string for an enabled storage backend. /// -/// Checks the `TEST_DATABASE_URL` environment variable first, falling back to -/// the default PostgreSQL connection string if not set. +/// Checks the `TEST_DATABASE_URL` environment variable first. If it is not set, +/// the fallback is selected from the enabled cargo features so sqlite-only test +/// runs do not accidentally try to boot a postgres backend. /// /// # Example /// @@ -21,6 +45,258 @@ const DEFAULT_POSTGRES_CONNECTION_STRING: &str = /// // Use connection_string for testing /// ``` pub fn get_test_database_url() -> String { - std::env::var("TEST_DATABASE_URL") - .unwrap_or_else(|_| DEFAULT_POSTGRES_CONNECTION_STRING.to_string()) + let connection_string = std::env::var("TEST_DATABASE_URL") + .unwrap_or_else(|_| default_test_database_url().to_string()); + + isolate_test_database_url(&connection_string) +} + +pub async fn ensure_test_database_exists(connection_string: &str) -> Result<()> { + #[cfg(any(feature = "postgres", feature = "timescaledb"))] + if connection_string.starts_with("postgres://") + || connection_string.starts_with("postgresql://") + || connection_string.starts_with("timescaledb://") + { + return ensure_postgres_test_database(connection_string).await; + } + + #[cfg(not(any(feature = "postgres", feature = "timescaledb")))] + let _ = connection_string; + + Ok(()) +} + +fn default_test_database_url() -> &'static str { + if cfg!(feature = "postgres") { + DEFAULT_POSTGRES_CONNECTION_STRING + } else if cfg!(feature = "timescaledb") { + DEFAULT_TIMESCALEDB_CONNECTION_STRING + } else if cfg!(feature = "sqlite") { + DEFAULT_SQLITE_CONNECTION_STRING + } else if cfg!(feature = "duckdb") { + DEFAULT_DUCKDB_CONNECTION_STRING + } else if cfg!(feature = "clickhouse") { + DEFAULT_CLICKHOUSE_CONNECTION_STRING + } else { + DEFAULT_POSTGRES_CONNECTION_STRING + } +} + +pub fn isolate_test_database_url(connection_string: &str) -> String { + if connection_string.starts_with("postgres://") + || connection_string.starts_with("postgresql://") + || connection_string.starts_with("timescaledb://") + { + return isolate_postgres_test_database(connection_string); + } + + if let Some(path) = connection_string.strip_prefix("sqlite://") { + return format!("sqlite://{}", isolate_test_database_path(path)); + } + + if let Some(path) = connection_string.strip_prefix("duckdb://") { + return format!("duckdb://{}", isolate_test_database_path(path)); + } + + connection_string.to_string() +} + +fn isolate_postgres_test_database(connection_string: &str) -> String { + let Ok(mut url) = Url::parse(connection_string) else { + return connection_string.to_string(); + }; + + let Some(database_name) = url + .path_segments() + .and_then(|mut segments| segments.next_back()) + .filter(|segment| !segment.is_empty()) + else { + return connection_string.to_string(); + }; + + let isolated_name = format!("{}-{}", database_name, std::process::id()); + url.set_path(&format!("/{}", isolated_name)); + url.to_string() +} + +#[cfg(any(feature = "postgres", feature = "timescaledb"))] +async fn ensure_postgres_test_database(connection_string: &str) -> Result<()> { + let sqlx_connection_string = normalize_postgres_like_url(connection_string); + + let url = Url::parse(connection_string).map_err(|e| { + anyhow!( + "Failed to parse PostgreSQL test URL '{}': {}", + connection_string, + e + ) + })?; + + let database_name = url + .path_segments() + .and_then(|mut segments| segments.next_back()) + .filter(|segment| !segment.is_empty()) + .ok_or_else(|| { + anyhow!( + "PostgreSQL test URL '{}' is missing a database name", + connection_string + ) + })? + .to_string(); + + let mut admin_url = Url::parse(&sqlx_connection_string).map_err(|e| { + anyhow!( + "Failed to normalize PostgreSQL-compatible test URL '{}': {}", + connection_string, + e + ) + })?; + admin_url.set_path("/postgres"); + + let pool = PgPoolOptions::new() + .max_connections(1) + .connect(admin_url.as_str()) + .await + .map_err(|e| { + anyhow!( + "Failed to connect to PostgreSQL admin database '{}': {}", + admin_url, + e + ) + })?; + + let create_database_sql = format!("CREATE DATABASE \"{}\"", database_name.replace('"', "\"\"")); + + match sqlx::query(&create_database_sql).execute(&pool).await { + Ok(_) => Ok(()), + Err(sqlx::Error::Database(db_err)) if db_err.code().as_deref() == Some("42P04") => Ok(()), + Err(e) => Err(anyhow!( + "Failed to create PostgreSQL test database '{}': {}", + database_name, + e + )), + } +} + +#[cfg(any(feature = "postgres", feature = "timescaledb"))] +fn normalize_postgres_like_url(connection_string: &str) -> String { + if connection_string.starts_with("timescaledb://") { + connection_string.replacen("timescaledb://", "postgres://", 1) + } else { + connection_string.to_string() + } +} + +fn isolate_test_database_path(path: &str) -> String { + if path.is_empty() || path == ":memory:" || path.starts_with("file:") { + return path.to_string(); + } + + let process_id = std::process::id(); + let path = Path::new(path); + + let Some(file_name) = path.file_name() else { + return path.to_string_lossy().into_owned(); + }; + + let file_name = file_name.to_string_lossy(); + let stem = path + .file_stem() + .map(|stem| stem.to_string_lossy().into_owned()) + .unwrap_or_else(|| file_name.into_owned()); + let extension = path + .extension() + .map(|ext| ext.to_string_lossy().into_owned()); + + let isolated_name = match extension { + Some(extension) => format!("{}-{}.{}", stem, process_id, extension), + None => format!("{}-{}", stem, process_id), + }; + + let mut isolated_path = path.parent().map(PathBuf::from).unwrap_or_default(); + isolated_path.push(isolated_name); + isolated_path.to_string_lossy().into_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn respects_test_database_url_override() { + temp_env::with_var("TEST_DATABASE_URL", Some("sqlite://override.db"), || { + let process_id = std::process::id(); + assert_eq!( + get_test_database_url(), + format!("sqlite://override-{}.db", process_id) + ); + }); + } + + #[test] + fn feature_aware_default_matches_enabled_backends() { + let expected = if cfg!(feature = "postgres") { + DEFAULT_POSTGRES_CONNECTION_STRING + } else if cfg!(feature = "timescaledb") { + DEFAULT_TIMESCALEDB_CONNECTION_STRING + } else if cfg!(feature = "sqlite") { + DEFAULT_SQLITE_CONNECTION_STRING + } else if cfg!(feature = "duckdb") { + DEFAULT_DUCKDB_CONNECTION_STRING + } else if cfg!(feature = "clickhouse") { + DEFAULT_CLICKHOUSE_CONNECTION_STRING + } else { + DEFAULT_POSTGRES_CONNECTION_STRING + }; + + assert_eq!(default_test_database_url(), expected); + } + + #[cfg(any(feature = "postgres", feature = "timescaledb"))] + #[test] + fn isolates_postgres_database_name_per_process() { + temp_env::with_var( + "TEST_DATABASE_URL", + Some("postgres://postgres:postgres@localhost:5432/sensapp-test"), + || { + let process_id = std::process::id(); + assert_eq!( + get_test_database_url(), + format!( + "postgres://postgres:postgres@localhost:5432/sensapp-test-{}", + process_id + ) + ); + }, + ); + } + + #[cfg(any(feature = "postgres", feature = "timescaledb"))] + #[test] + fn isolates_timescaledb_database_name_per_process() { + temp_env::with_var( + "TEST_DATABASE_URL", + Some("timescaledb://postgres:postgres@localhost:5433/sensapp-test"), + || { + let process_id = std::process::id(); + assert_eq!( + get_test_database_url(), + format!( + "timescaledb://postgres:postgres@localhost:5433/sensapp-test-{}", + process_id + ) + ); + }, + ); + } + + #[cfg(any(feature = "postgres", feature = "timescaledb"))] + #[test] + fn normalizes_timescaledb_url_for_sqlx() { + assert_eq!( + normalize_postgres_like_url( + "timescaledb://postgres:postgres@localhost:5433/sensapp-test" + ), + "postgres://postgres:postgres@localhost:5433/sensapp-test" + ); + } } diff --git a/tests/advanced_backend_queries.rs b/tests/advanced_backend_queries.rs new file mode 100644 index 00000000..e5cf6640 --- /dev/null +++ b/tests/advanced_backend_queries.rs @@ -0,0 +1,273 @@ +#![cfg_attr( + not(any( + feature = "sqlite", + feature = "duckdb", + feature = "timescaledb", + feature = "clickhouse" + )), + allow(dead_code, unused_imports) +)] + +mod common; + +use anyhow::Result; +use common::{DatabaseType, TestDb}; +use sensapp::config::load_configuration_for_tests; +use sensapp::datamodel::batch_builder::BatchBuilder; +use sensapp::datamodel::{Sample, Sensor, SensorType, TypedSamples}; +use sensapp::storage::{Aggregation, SensorDataQueryOptions}; +use serial_test::serial; +use std::sync::Arc; +use uuid::Uuid; + +static INIT: std::sync::Once = std::sync::Once::new(); + +fn ensure_config() { + INIT.call_once(|| { + load_configuration_for_tests().expect("Failed to load configuration for tests"); + }); +} + +async fn assert_bucketed_average_for_backend(db_type: DatabaseType) -> Result<()> { + ensure_config(); + + let test_db = match TestDb::new_with_type(db_type.clone()).await { + Ok(test_db) => test_db, + Err(error) => { + eprintln!("skipping {db_type:?} backend test: {error:#}"); + return Ok(()); + } + }; + let storage = test_db.storage(); + let sensor = Arc::new(Sensor::new( + Uuid::new_v4(), + format!("advanced_avg_{:?}", test_db.db_type), + SensorType::Float, + None, + None, + )); + + let samples = TypedSamples::Float(smallvec::smallvec![ + Sample { + datetime: hifitime::Epoch::from_unix_seconds(1_704_067_200.0), + value: 20.5, + }, + Sample { + datetime: hifitime::Epoch::from_unix_seconds(1_704_067_260.0), + value: 21.0, + }, + Sample { + datetime: hifitime::Epoch::from_unix_seconds(1_704_067_320.0), + value: 21.5, + }, + Sample { + datetime: hifitime::Epoch::from_unix_seconds(1_704_067_380.0), + value: 22.0, + }, + Sample { + datetime: hifitime::Epoch::from_unix_seconds(1_704_067_440.0), + value: 20.8, + }, + ]); + + let mut batch_builder = BatchBuilder::new()?; + batch_builder.add(sensor.clone(), samples).await?; + batch_builder.send_what_is_left(storage.clone()).await?; + + let sensor_data = storage + .query_sensor_data_advanced( + &sensor.uuid.to_string(), + &SensorDataQueryOptions { + start_time: None, + end_time: None, + limit: None, + step_ms: Some(120_000), + aggregation: Some(Aggregation::Avg), + simplify: None, + }, + ) + .await? + .expect("sensor data should exist"); + + match sensor_data.samples { + sensapp::datamodel::TypedSamples::Float(samples) => { + assert_eq!(samples.len(), 3, "expected 3 aggregated rows"); + assert_eq!( + samples[0].datetime.to_unix_milliseconds().floor() as i64, + 1_704_067_200_000 + ); + assert_eq!(samples[0].value, 20.75); + assert_eq!( + samples[1].datetime.to_unix_milliseconds().floor() as i64, + 1_704_067_320_000 + ); + assert_eq!(samples[1].value, 21.75); + assert_eq!( + samples[2].datetime.to_unix_milliseconds().floor() as i64, + 1_704_067_440_000 + ); + assert_eq!(samples[2].value, 20.8); + } + other => panic!("expected float samples, got {other:?}"), + } + + Ok(()) +} + +async fn assert_latest_and_availability_for_backend(db_type: DatabaseType) -> Result<()> { + ensure_config(); + + let test_db = match TestDb::new_with_type(db_type.clone()).await { + Ok(test_db) => test_db, + Err(error) => { + eprintln!("skipping {db_type:?} backend test: {error:#}"); + return Ok(()); + } + }; + let storage = test_db.storage(); + let sensor = Arc::new(Sensor::new( + Uuid::new_v4(), + format!("advanced_latest_{:?}", test_db.db_type), + SensorType::Float, + None, + None, + )); + + let samples = TypedSamples::Float(smallvec::smallvec![ + Sample { + datetime: hifitime::Epoch::from_unix_seconds(1_704_067_200.0), + value: 20.5, + }, + Sample { + datetime: hifitime::Epoch::from_unix_seconds(1_704_067_260.0), + value: 21.0, + }, + Sample { + datetime: hifitime::Epoch::from_unix_seconds(1_704_067_320.0), + value: 21.5, + }, + Sample { + datetime: hifitime::Epoch::from_unix_seconds(1_704_067_380.0), + value: 22.0, + }, + Sample { + datetime: hifitime::Epoch::from_unix_seconds(1_704_067_440.0), + value: 20.8, + }, + ]); + + let mut batch_builder = BatchBuilder::new()?; + batch_builder.add(sensor.clone(), samples).await?; + batch_builder.send_what_is_left(storage.clone()).await?; + + let latest = storage + .query_sensor_data_latest(&sensor.uuid.to_string(), None, None) + .await? + .expect("latest sample should exist"); + + match latest.samples { + sensapp::datamodel::TypedSamples::Float(samples) => { + assert_eq!(samples.len(), 1, "expected a single latest sample"); + assert_eq!( + samples[0].datetime.to_unix_milliseconds().floor() as i64, + 1_704_067_440_000 + ); + assert_eq!(samples[0].value, 20.8); + } + other => panic!("expected float samples, got {other:?}"), + } + + let availability = storage + .query_sensor_data_availability( + &sensor.uuid.to_string(), + hifitime::Epoch::from_unix_seconds(1_704_067_200.0), + hifitime::Epoch::from_unix_seconds(1_704_067_440.0), + Some(120_000), + ) + .await? + .expect("availability summary should exist"); + + assert_eq!(availability.sample_count, 5); + assert_eq!(availability.covered_buckets, Some(3)); + assert_eq!( + availability + .first_sample_at + .expect("first sample timestamp should exist") + .to_unix_milliseconds() + .floor() as i64, + 1_704_067_200_000 + ); + assert_eq!( + availability + .last_sample_at + .expect("last sample timestamp should exist") + .to_unix_milliseconds() + .floor() as i64, + 1_704_067_440_000 + ); + + Ok(()) +} + +#[cfg(feature = "postgres")] +#[tokio::test] +#[serial] +async fn test_postgresql_native_latest_and_availability_queries() -> Result<()> { + assert_latest_and_availability_for_backend(DatabaseType::PostgreSQL).await +} + +#[cfg(feature = "sqlite")] +#[tokio::test] +#[serial] +async fn test_sqlite_native_bucketed_average_query() -> Result<()> { + assert_bucketed_average_for_backend(DatabaseType::SQLite).await +} + +#[cfg(feature = "sqlite")] +#[tokio::test] +#[serial] +async fn test_sqlite_native_latest_and_availability_queries() -> Result<()> { + assert_latest_and_availability_for_backend(DatabaseType::SQLite).await +} + +#[cfg(feature = "duckdb")] +#[tokio::test] +#[serial] +async fn test_duckdb_native_bucketed_average_query() -> Result<()> { + assert_bucketed_average_for_backend(DatabaseType::DuckDB).await +} + +#[cfg(feature = "duckdb")] +#[tokio::test] +#[serial] +async fn test_duckdb_native_latest_and_availability_queries() -> Result<()> { + assert_latest_and_availability_for_backend(DatabaseType::DuckDB).await +} + +#[cfg(feature = "timescaledb")] +#[tokio::test] +#[serial] +async fn test_timescaledb_native_bucketed_average_query() -> Result<()> { + assert_bucketed_average_for_backend(DatabaseType::TimescaleDB).await +} + +#[cfg(feature = "timescaledb")] +#[tokio::test] +#[serial] +async fn test_timescaledb_native_latest_and_availability_queries() -> Result<()> { + assert_latest_and_availability_for_backend(DatabaseType::TimescaleDB).await +} + +#[cfg(feature = "clickhouse")] +#[tokio::test] +#[serial] +async fn test_clickhouse_native_bucketed_average_query() -> Result<()> { + assert_bucketed_average_for_backend(DatabaseType::ClickHouse).await +} + +#[cfg(feature = "clickhouse")] +#[tokio::test] +#[serial] +async fn test_clickhouse_native_latest_and_availability_queries() -> Result<()> { + assert_latest_and_availability_for_backend(DatabaseType::ClickHouse).await +} diff --git a/tests/arrow_integration.rs b/tests/arrow_integration.rs index d99abd11..fd90871b 100644 --- a/tests/arrow_integration.rs +++ b/tests/arrow_integration.rs @@ -1,8 +1,10 @@ +#![cfg_attr(feature = "rrdcached", allow(unused_imports, dead_code))] + mod common; use anyhow::Result; use arrow::record_batch::RecordBatch; -use arrow_ipc::writer::FileWriter; +use arrow_ipc::reader::StreamReader; use axum::http::StatusCode; use common::db::DbHelpers; use common::http::TestApp; @@ -12,6 +14,7 @@ use sensapp::datamodel::{Sample, SensAppDateTime, Sensor, SensorData, SensorType use sensapp::exporters::ArrowConverter; use serial_test::serial; use smallvec::smallvec; +use std::io::Cursor; use uuid::Uuid; // Ensure configuration is loaded once for all tests in this module @@ -22,49 +25,16 @@ fn ensure_config() { }); } -/// Convert multiple SensorData to Arrow format with multiple RecordBatches -/// -/// This is primarily intended for testing scenarios where you need to combine -/// data from multiple sensors into a single Arrow file. -fn sensor_data_list_to_arrow_file(sensor_data_list: &[SensorData]) -> Result> { - if sensor_data_list.is_empty() { - return Err(anyhow::anyhow!( - "Cannot create Arrow file from empty sensor data list" - )); - } - - let batches: Result> = sensor_data_list - .iter() - .map(ArrowConverter::to_record_batch) - .collect(); - let batches = batches?; - - record_batches_to_arrow_file(&batches) -} - -/// Convert multiple RecordBatches to Arrow file format -/// -/// This is a utility function primarily used by `sensor_data_list_to_arrow_file` -/// for combining multiple record batches into a single Arrow file. -fn record_batches_to_arrow_file(batches: &[RecordBatch]) -> Result> { - if batches.is_empty() { - return Err(anyhow::anyhow!( - "Cannot create Arrow file from empty batch list" - )); - } - - let mut buffer = Vec::new(); - { - let mut writer = FileWriter::try_new(&mut buffer, &batches[0].schema())?; - for batch in batches { - writer.write(batch)?; - } - writer.finish()?; - } - Ok(buffer) +fn read_arrow_stream_batches(bytes: &[u8]) -> Result> { + let reader = StreamReader::try_new(Cursor::new(bytes), None)?; + reader + .into_iter() + .map(|batch| batch.map_err(Into::into)) + .collect() } /// Test Arrow data export functionality +#[cfg(not(feature = "rrdcached"))] mod export_tests { use super::*; @@ -92,12 +62,21 @@ mod export_tests { // Verify response response .assert_status(StatusCode::OK) - .assert_content_type("application/vnd.apache.arrow.file"); + .assert_content_type("application/vnd.apache.arrow.stream"); - // Verify Arrow file format + // Verify Arrow stream payload and schema metadata let body_bytes = response.body_bytes(); - assert!(!body_bytes.is_empty()); - assert_eq!(&body_bytes[0..6], b"ARROW1"); // Arrow magic number + let batches = read_arrow_stream_batches(body_bytes)?; + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_columns(), 2); + assert_eq!( + batches[0] + .schema() + .metadata() + .get("sensapp.sensor.name") + .map(String::as_str), + Some(sensor_name.as_str()) + ); Ok(()) } @@ -124,8 +103,8 @@ mod export_tests { response.assert_status(StatusCode::OK); let body_bytes = response.body_bytes(); - assert!(!body_bytes.is_empty()); - assert_eq!(&body_bytes[0..6], b"ARROW1"); + let batches = read_arrow_stream_batches(body_bytes)?; + assert_eq!(batches.len(), 1); // Export humidity sensor as Arrow let humidity_sensor = DbHelpers::get_sensor_by_name(&storage, &humidity_name) @@ -137,8 +116,8 @@ mod export_tests { response.assert_status(StatusCode::OK); let body_bytes = response.body_bytes(); - assert!(!body_bytes.is_empty()); - assert_eq!(&body_bytes[0..6], b"ARROW1"); + let batches = read_arrow_stream_batches(body_bytes)?; + assert_eq!(batches.len(), 1); Ok(()) } @@ -171,14 +150,15 @@ mod export_tests { response.assert_status(StatusCode::OK); let body_bytes = response.body_bytes(); - assert!(!body_bytes.is_empty()); - assert_eq!(&body_bytes[0..6], b"ARROW1"); + let batches = read_arrow_stream_batches(body_bytes)?; + assert_eq!(batches.len(), 1); Ok(()) } } /// Test Arrow data import functionality +#[cfg(not(feature = "rrdcached"))] mod import_tests { use super::*; @@ -219,13 +199,13 @@ mod import_tests { ]); let sensor_data = SensorData::new(sensor, samples); - let arrow_bytes = ArrowConverter::to_arrow_file(&sensor_data)?; + let arrow_bytes = ArrowConverter::to_arrow_stream(&sensor_data)?; // Upload Arrow data let response = app .post_binary( "/sensors/publish", - "application/vnd.apache.arrow.file", + "application/vnd.apache.arrow.stream", &arrow_bytes, ) .await?; @@ -268,13 +248,13 @@ mod import_tests { ]); let sensor_data = SensorData::new(sensor, samples); - let arrow_bytes = ArrowConverter::to_arrow_file(&sensor_data)?; + let arrow_bytes = ArrowConverter::to_arrow_stream(&sensor_data)?; // Upload Arrow data let response = app .post_binary( "/sensors/publish", - "application/vnd.apache.arrow.file", + "application/vnd.apache.arrow.stream", &arrow_bytes, ) .await?; @@ -317,13 +297,13 @@ mod import_tests { ]); let sensor_data = SensorData::new(sensor, samples); - let arrow_bytes = ArrowConverter::to_arrow_file(&sensor_data)?; + let arrow_bytes = ArrowConverter::to_arrow_stream(&sensor_data)?; // Upload Arrow data let response = app .post_binary( "/sensors/publish", - "application/vnd.apache.arrow.file", + "application/vnd.apache.arrow.stream", &arrow_bytes, ) .await?; @@ -349,7 +329,7 @@ mod import_tests { let response = app .post_binary( "/sensors/publish", - "application/vnd.apache.arrow.file", + "application/vnd.apache.arrow.stream", invalid_data, ) .await?; @@ -364,6 +344,7 @@ mod import_tests { } /// Test Arrow round-trip functionality (export then import) +#[cfg(not(feature = "rrdcached"))] mod roundtrip_tests { use super::*; @@ -398,7 +379,7 @@ mod roundtrip_tests { let import_response = app .post_binary( "/sensors/publish", - "application/vnd.apache.arrow.file", + "application/vnd.apache.arrow.stream", arrow_bytes, ) .await?; @@ -450,7 +431,7 @@ mod roundtrip_tests { let temp_import_response = app .post_binary( "/sensors/publish", - "application/vnd.apache.arrow.file", + "application/vnd.apache.arrow.stream", temp_arrow_bytes, ) .await?; @@ -458,7 +439,7 @@ mod roundtrip_tests { let humidity_import_response = app .post_binary( "/sensors/publish", - "application/vnd.apache.arrow.file", + "application/vnd.apache.arrow.stream", humidity_arrow_bytes, ) .await?; @@ -503,13 +484,19 @@ mod converter_tests { let batch = ArrowConverter::to_record_batch(&sensor_data).unwrap(); assert_eq!(batch.num_rows(), 2); - assert_eq!(batch.num_columns(), 4); // timestamp, value, sensor_id, sensor_name + assert_eq!(batch.num_columns(), 2); // Verify schema let schema = batch.schema(); assert_eq!(schema.field(0).name(), "timestamp"); assert_eq!(schema.field(1).name(), "value"); - assert_eq!(schema.field(2).name(), "sensor_id"); + assert_eq!( + schema + .metadata() + .get("sensapp.sensor.name") + .map(String::as_str), + Some("test_sensor") + ); } #[test] @@ -565,12 +552,12 @@ mod converter_tests { let batch = result.unwrap(); assert_eq!(batch.num_rows(), 1); - assert!(batch.num_columns() >= 3); // At least timestamp, value, sensor_id + assert_eq!(batch.num_columns(), 2); } } #[test] - fn test_arrow_file_format_validation() { + fn test_arrow_stream_format_validation() { let sensor = Sensor { uuid: Uuid::new_v4(), name: "test_sensor".to_string(), @@ -585,11 +572,11 @@ mod converter_tests { }]); let sensor_data = SensorData::new(sensor, samples); - let arrow_bytes = ArrowConverter::to_arrow_file(&sensor_data).unwrap(); + let arrow_bytes = ArrowConverter::to_arrow_stream(&sensor_data).unwrap(); - // Verify Arrow file format - assert!(!arrow_bytes.is_empty()); - assert_eq!(&arrow_bytes[0..6], b"ARROW1"); // Arrow magic number + let batches = read_arrow_stream_batches(&arrow_bytes).unwrap(); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 1); } #[test] @@ -624,9 +611,23 @@ mod converter_tests { let sensor_data2 = SensorData::new(sensor2, samples2); let sensor_list = vec![sensor_data1, sensor_data2]; - let arrow_bytes = sensor_data_list_to_arrow_file(&sensor_list).unwrap(); - - assert!(!arrow_bytes.is_empty()); - assert_eq!(&arrow_bytes[0..6], b"ARROW1"); + let arrow_bytes = ArrowConverter::to_arrow_stream_multi(&sensor_list).unwrap(); + + let batches = read_arrow_stream_batches(&arrow_bytes).unwrap(); + assert_eq!(batches.len(), 1); + assert!( + batches[0] + .schema() + .fields() + .iter() + .any(|field| field.name() == "integer_value") + ); + assert!( + batches[0] + .schema() + .fields() + .iter() + .any(|field| field.name() == "float_value") + ); } } diff --git a/tests/clickhouse_http_lifecycle.rs b/tests/clickhouse_http_lifecycle.rs new file mode 100644 index 00000000..1e9a21de --- /dev/null +++ b/tests/clickhouse_http_lifecycle.rs @@ -0,0 +1,146 @@ +mod common; + +#[cfg(feature = "clickhouse")] +mod clickhouse_http_lifecycle_tests { + use super::common::db::DbHelpers; + use super::common::http::TestApp; + use super::common::{DatabaseType, TestDb}; + use anyhow::Result; + use axum::http::StatusCode; + use hifitime::Epoch; + use sensapp::config::load_configuration_for_tests; + use sensapp::storage::StorageInstance; + use serde_json::Value; + use serial_test::serial; + use std::sync::Arc; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + use uuid::Uuid; + + static INIT: std::sync::Once = std::sync::Once::new(); + + fn ensure_config() { + INIT.call_once(|| { + load_configuration_for_tests().expect("Failed to load configuration for tests"); + }); + } + + fn unique_metric_name(prefix: &str) -> String { + format!("{}_{}", prefix, Uuid::new_v4().simple()) + } + + fn iso8601_from_unix_seconds(seconds: i64) -> String { + Epoch::from_unix_seconds(seconds as f64).to_rfc3339() + } + + async fn clickhouse_app() -> Result<(Arc, TestApp)> { + let test_db = TestDb::new_with_type(DatabaseType::ClickHouse).await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + Ok((storage, app)) + } + + #[tokio::test] + #[serial] + async fn test_clickhouse_http_csv_lifecycle() -> Result<()> { + ensure_config(); + let (storage, app) = clickhouse_app().await?; + + let readiness = app.get("/health/ready").await?; + readiness.assert_status(StatusCode::OK); + let readiness_body: Value = readiness.json()?; + assert_eq!(readiness_body["status"], "ready"); + + let sensor_name = unique_metric_name("temperature_http"); + let now = Epoch::now()?.to_unix_seconds().floor() as i64; + let csv_data = format!( + "datetime,sensor_name,value,unit\n{0},{3},20.5,C\n{1},{3},21.0,C\n{2},{3},21.5,C", + iso8601_from_unix_seconds(now - 120), + iso8601_from_unix_seconds(now - 60), + iso8601_from_unix_seconds(now), + sensor_name, + ); + + app.post_csv("/sensors/publish", &csv_data) + .await? + .assert_status(StatusCode::OK) + .assert_body_contains("ingested successfully"); + + let series = app.get("/series").await?; + series.assert_status(StatusCode::OK); + series.assert_body_contains(&sensor_name); + + let metrics = app.get("/metrics").await?; + metrics.assert_status(StatusCode::OK); + metrics.assert_body_contains(&sensor_name); + + let sensor = DbHelpers::get_sensor_by_name(&storage, &sensor_name) + .await? + .expect("sensor should exist after CSV publish"); + + let query_response = app + .get(&format!("/api/v1/query?query={}&format=jsonl", sensor_name)) + .await?; + query_response.assert_status(StatusCode::OK); + query_response.assert_body_contains(&sensor_name); + + let export_response = app + .get(&format!("/series/{}?format=csv", sensor.uuid)) + .await?; + export_response.assert_status(StatusCode::OK); + export_response.assert_content_type("text/csv"); + export_response.assert_body_contains("timestamp"); + + let last_response = app.get(&format!("/series/{}/last", sensor.uuid)).await?; + last_response.assert_status(StatusCode::OK); + let last_payload: Value = last_response.json()?; + assert_eq!(last_payload["sensor_name"], sensor_name); + assert_eq!(last_payload["value"], 21.5); + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_clickhouse_http_influx_lifecycle() -> Result<()> { + ensure_config(); + let (_storage, app) = clickhouse_app().await?; + + let metric_name = unique_metric_name("power_http"); + let series_name = format!("{} value", metric_name); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock after unix epoch"); + let first_timestamp_ns = now + .checked_sub(Duration::from_secs(60)) + .expect("enough time since epoch") + .as_nanos(); + let second_timestamp_ns = now.as_nanos(); + let influx_body = format!( + "{0},site=lab value=42.5 {1}\n{0},site=lab value=43.0 {2}", + metric_name, first_timestamp_ns, second_timestamp_ns + ); + + app.post_influxdb( + "/api/v2/write?bucket=clickhouse_http_test&org=sensapp", + &influx_body, + ) + .await? + .assert_status(StatusCode::NO_CONTENT); + + let query_response = app + .get(&format!( + "/api/v1/query?query=%7B__name__%3D%22{}%22%7D&format=csv", + urlencoding::encode(&series_name) + )) + .await?; + query_response.assert_status(StatusCode::OK); + query_response.assert_content_type("text/csv"); + query_response.assert_body_contains(&series_name); + + let series_response = app.get("/series").await?; + series_response.assert_status(StatusCode::OK); + series_response.assert_body_contains(&series_name); + + Ok(()) + } +} diff --git a/tests/clickhouse_integration.rs b/tests/clickhouse_integration.rs index 48b8928a..ac5a2417 100644 --- a/tests/clickhouse_integration.rs +++ b/tests/clickhouse_integration.rs @@ -5,7 +5,13 @@ mod clickhouse_tests { use crate::common::{DatabaseType, TestDb}; use anyhow::Result; use sensapp::config::load_configuration_for_tests; + use sensapp::datamodel::batch_builder::BatchBuilder; + use sensapp::datamodel::sensapp_vec::SensAppLabels; + use sensapp::datamodel::{Sample, Sensor, SensorType, TypedSamples}; + use sensapp::storage::query::LabelMatcher; use serial_test::serial; + use std::sync::Arc; + use uuid::Uuid; // Ensure configuration is loaded once for all tests in this module static INIT: std::sync::Once = std::sync::Once::new(); @@ -15,6 +21,46 @@ mod clickhouse_tests { }); } + fn create_sensor_with_labels( + name: &str, + sensor_type: SensorType, + labels: Vec<(String, String)>, + ) -> Sensor { + let labels: SensAppLabels = labels.into_iter().collect(); + Sensor::new( + Uuid::new_v4(), + name.to_string(), + sensor_type, + None, + Some(labels), + ) + } + + fn create_float_samples(count: usize) -> TypedSamples { + let samples: smallvec::SmallVec<[Sample; 4]> = (0..count) + .map(|index| Sample { + datetime: hifitime::Epoch::from_unix_seconds((1704067200 + index * 60) as f64), + value: 20.0 + index as f64, + }) + .collect(); + TypedSamples::Float(samples) + } + + async fn publish_test_sensors( + storage: &Arc, + sensors_with_samples: Vec<(Sensor, TypedSamples)>, + ) -> Result<()> { + let mut batch_builder = BatchBuilder::new()?; + + for (sensor, samples) in sensors_with_samples { + let sensor = Arc::new(sensor); + batch_builder.add(sensor.clone(), samples).await?; + } + + batch_builder.send_what_is_left(storage.clone()).await?; + Ok(()) + } + /// Test basic ClickHouse connection and database setup #[tokio::test] #[serial] @@ -28,10 +74,44 @@ mod clickhouse_tests { storage.create_or_migrate().await?; // Then: The operations should succeed (database is accessible) - let sensors = storage.list_series(None).await?; + let result = storage.list_series(None, None, None).await?; // Database should be empty or contain existing sensors - println!("Found {} sensors in ClickHouse database", sensors.len()); + println!( + "Found {} sensors in ClickHouse database", + result.series.len() + ); + + Ok(()) + } + + /// Test repeated migrations are safe and idempotent + #[tokio::test] + #[serial] + async fn test_clickhouse_create_or_migrate_is_idempotent() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::ClickHouse).await?; + let storage = test_db.storage(); + + storage.create_or_migrate().await?; + storage.create_or_migrate().await?; + + let result = storage.list_series(None, None, None).await?; + assert!(result.series.is_empty()); + + Ok(()) + } + + /// Test ClickHouse storage health check against a real service + #[tokio::test] + #[serial] + async fn test_clickhouse_health_check() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::ClickHouse).await?; + let storage = test_db.storage(); + + storage.create_or_migrate().await?; + storage.health_check().await?; Ok(()) } @@ -86,4 +166,185 @@ mod clickhouse_tests { Ok(()) } + + #[tokio::test] + #[serial] + async fn test_clickhouse_query_sensors_by_labels_exact_match() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::ClickHouse).await?; + let storage = test_db.storage(); + + let sensor1 = create_sensor_with_labels( + "cpu_usage", + SensorType::Float, + vec![("environment".to_string(), "production".to_string())], + ); + let sensor2 = create_sensor_with_labels( + "cpu_usage", + SensorType::Float, + vec![("environment".to_string(), "staging".to_string())], + ); + + publish_test_sensors( + &storage, + vec![ + (sensor1, create_float_samples(3)), + (sensor2, create_float_samples(2)), + ], + ) + .await?; + + let matchers = vec![ + LabelMatcher::eq("__name__", "cpu_usage"), + LabelMatcher::eq("environment", "production"), + ]; + let results = storage + .query_sensors_by_labels(&matchers, None, None, None, false) + .await?; + + assert_eq!(results.len(), 1); + assert_eq!(results[0].sensor.name, "cpu_usage"); + assert_eq!(results[0].sensor.labels[0].0, "environment"); + assert_eq!(results[0].sensor.labels[0].1, "production"); + + match &results[0].samples { + TypedSamples::Float(samples) => assert_eq!(samples.len(), 3), + other => panic!("Expected float samples, got {other:?}"), + } + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_clickhouse_query_sensors_by_labels_numeric_only() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::ClickHouse).await?; + let storage = test_db.storage(); + + let numeric_sensor = create_sensor_with_labels( + "room_value", + SensorType::Float, + vec![("site".to_string(), "lab".to_string())], + ); + let string_sensor = create_sensor_with_labels( + "room_state", + SensorType::String, + vec![("site".to_string(), "lab".to_string())], + ); + + publish_test_sensors( + &storage, + vec![ + (numeric_sensor, create_float_samples(2)), + ( + string_sensor, + TypedSamples::String(smallvec::smallvec![ + Sample { + datetime: hifitime::Epoch::from_unix_seconds(1704067200.0), + value: "ok".to_string(), + }, + Sample { + datetime: hifitime::Epoch::from_unix_seconds(1704067260.0), + value: "warn".to_string(), + } + ]), + ), + ], + ) + .await?; + + let matchers = vec![LabelMatcher::eq("site", "lab")]; + let results = storage + .query_sensors_by_labels(&matchers, None, None, None, true) + .await?; + + assert_eq!(results.len(), 1); + assert_eq!(results[0].sensor.name, "room_value"); + assert_eq!(results[0].sensor.sensor_type, SensorType::Float); + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_clickhouse_list_series_pagination() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::ClickHouse).await?; + let storage = test_db.storage(); + + let sensors = vec![ + create_sensor_with_labels("sensor_a", SensorType::Float, vec![]), + create_sensor_with_labels("sensor_b", SensorType::Float, vec![]), + create_sensor_with_labels("sensor_c", SensorType::Float, vec![]), + ]; + + publish_test_sensors( + &storage, + sensors + .into_iter() + .map(|sensor| (sensor, create_float_samples(1))) + .collect(), + ) + .await?; + + let first_page = storage.list_series(None, Some(2), None).await?; + assert_eq!(first_page.series.len(), 2); + assert!(first_page.bookmark.is_some()); + + let first_page_ids: Vec<_> = first_page.series.iter().map(|sensor| sensor.uuid).collect(); + let second_page = storage + .list_series(None, Some(2), first_page.bookmark.as_deref()) + .await?; + + assert_eq!(second_page.series.len(), 1); + assert!(second_page.bookmark.is_none()); + for sensor in &second_page.series { + assert!(!first_page_ids.contains(&sensor.uuid)); + } + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_clickhouse_list_series_exact_last_page_has_no_bookmark() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::ClickHouse).await?; + let storage = test_db.storage(); + + publish_test_sensors( + &storage, + vec![ + ( + create_sensor_with_labels("sensor_x", SensorType::Float, vec![]), + create_float_samples(1), + ), + ( + create_sensor_with_labels("sensor_y", SensorType::Float, vec![]), + create_float_samples(1), + ), + ], + ) + .await?; + + let page = storage.list_series(None, Some(2), None).await?; + assert_eq!(page.series.len(), 2); + assert!(page.bookmark.is_none()); + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_clickhouse_list_series_invalid_bookmark() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::ClickHouse).await?; + let storage = test_db.storage(); + + let error = storage.list_series(None, Some(2), Some("invalid")).await; + assert!(error.is_err(), "invalid bookmark should be rejected"); + + Ok(()) + } } diff --git a/tests/common/db.rs b/tests/common/db.rs index 0d47c20b..6e6844df 100644 --- a/tests/common/db.rs +++ b/tests/common/db.rs @@ -12,10 +12,10 @@ impl DbHelpers { /// Count total number of sensor samples in the database #[allow(dead_code)] // Test helper method pub async fn count_total_samples(storage: &Arc) -> Result { - let sensors = storage.list_series(None).await?; + let result = storage.list_series(None, None, None).await?; let mut total = 0; - for sensor in sensors { + for sensor in result.series { if let Some(sensor_data) = storage .query_sensor_data(&sensor.uuid.to_string(), None, None, None) .await? @@ -42,8 +42,8 @@ impl DbHelpers { storage: &Arc, name: &str, ) -> Result> { - let sensors = storage.list_series(None).await?; - Ok(sensors.into_iter().find(|s| s.name == name)) + let result = storage.list_series(None, None, None).await?; + Ok(result.series.into_iter().find(|s| s.name == name)) } /// Verify sensor data exists and has expected sample count @@ -89,8 +89,8 @@ impl DbHelpers { /// Get all sensor names in the database #[allow(dead_code)] // Test helper method pub async fn get_sensor_names(storage: &Arc) -> Result> { - let sensors = storage.list_series(None).await?; - Ok(sensors.into_iter().map(|s| s.name).collect()) + let result = storage.list_series(None, None, None).await?; + Ok(result.series.into_iter().map(|s| s.name).collect()) } } diff --git a/tests/common/fixtures.rs b/tests/common/fixtures.rs index 88a22ff0..f5f63934 100644 --- a/tests/common/fixtures.rs +++ b/tests/common/fixtures.rs @@ -48,6 +48,25 @@ pub fn multi_sensor_csv() -> String { ) } +/// Sample CSV data for three sensors +#[allow(dead_code)] // Test fixture +pub fn three_sensor_csv() -> String { + let test_id = generate_test_id(); + format!( + r#"datetime,sensor_name,value,unit +2024-01-01T00:00:00Z,temperature_{},20.5,°C +2024-01-01T00:00:00Z,humidity_{},65.0,% +2024-01-01T00:00:00Z,pressure_{},1013.0,hPa +2024-01-01T00:01:00Z,temperature_{},21.0,°C +2024-01-01T00:01:00Z,humidity_{},64.5,% +2024-01-01T00:01:00Z,pressure_{},1012.8,hPa +2024-01-01T00:02:00Z,temperature_{},21.5,°C +2024-01-01T00:02:00Z,humidity_{},64.0,% +2024-01-01T00:02:00Z,pressure_{},1012.6,hPa"#, + test_id, test_id, test_id, test_id, test_id, test_id, test_id, test_id, test_id + ) +} + /// Sample CSV data for multiple sensors with known sensor names /// Returns (csv_data, temperature_sensor_name, humidity_sensor_name) #[allow(dead_code)] // Test fixture diff --git a/tests/common/http.rs b/tests/common/http.rs index 496ad01b..cebb62c6 100644 --- a/tests/common/http.rs +++ b/tests/common/http.rs @@ -4,13 +4,18 @@ use axum::Router; use axum::body::Body; use axum::http::{HeaderMap, Request, StatusCode}; use axum::routing::{get, post}; -use sensapp::ingestors::http::crud::{get_series_data, list_metrics, list_series}; -use sensapp::ingestors::http::health::{liveness, readiness}; -use sensapp::ingestors::http::server::publish_senml_data; -use sensapp::ingestors::http::state::HttpServerState; +use sensapp::http::crud::{ + get_series_availability, get_series_data, get_series_last_sample, list_metrics, list_series, +}; +use sensapp::http::health::{liveness, readiness}; +use sensapp::http::influxdb::publish_influxdb; +use sensapp::http::metrics::{HttpMetrics, prometheus_metrics, track_http_metrics}; +use sensapp::http::server::publish_senml_data; +use sensapp::http::simple_promql::simple_promql_query; +use sensapp::http::state::HttpServerState; use sensapp::storage::StorageInstance; use std::sync::Arc; -use tower::ServiceExt; // for `oneshot` and `ready` +use tower::ServiceExt; /// HTTP test client for making requests to our app #[allow(dead_code)] // Test helper struct @@ -25,18 +30,32 @@ impl TestApp { let state = HttpServerState { name: Arc::new("SensApp Test".to_string()), storage, + metrics: Arc::new(HttpMetrics::new()), influxdb_with_numeric: false, + auth: None, }; // Create a minimal router for testing (without middleware that might interfere) // We'll define simple test handlers that delegate to the import functions let app = Router::new() .route("/sensors/publish", post(test_publish_handler)) + .route("/api/v2/write", post(publish_influxdb)) .route("/metrics", get(list_metrics)) + .route("/prometheus/metrics", get(prometheus_metrics)) .route("/series", get(list_series)) + .route("/api/v1/query", get(simple_promql_query)) .route("/series/{series_uuid}", get(get_series_data)) + .route("/series/{series_uuid}/last", get(get_series_last_sample)) + .route( + "/series/{series_uuid}/availability", + get(get_series_availability), + ) .route("/health/live", get(liveness)) .route("/health/ready", get(readiness)) + .layer(axum::middleware::from_fn_with_state( + state.metrics.clone(), + track_http_metrics, + )) .with_state(state); Self { app } @@ -123,6 +142,13 @@ impl TestApp { let response = self.app.clone().oneshot(request).await?; Ok(TestResponse::new(response).await) } + + /// Send a raw request (for custom headers, compression, etc.) + #[allow(dead_code)] // Test helper method + pub async fn raw_request(&self, request: Request) -> Result { + let response = self.app.clone().oneshot(request).await?; + Ok(TestResponse::new(response).await) + } } /// Test response wrapper for easier assertions @@ -269,7 +295,9 @@ async fn test_publish_handler( })?; Ok("ok".to_string()) - } else if content_type.contains("application/vnd.apache.arrow.file") { + } else if content_type.contains("application/vnd.apache.arrow.stream") + || content_type.contains("application/vnd.apache.arrow.file") + { // Handle Arrow data use sensapp::importers::arrow::publish_arrow_async; @@ -282,9 +310,11 @@ async fn test_publish_handler( .map_err(|e| { // Arrow parsing errors should be bad requests, not internal server errors if e.to_string().contains("Failed to create Arrow file reader") - || e.to_string() - .contains("Arrow file contains no data batches") || e.to_string().contains("Failed to read Arrow batch") + || e.to_string() + .contains("Failed to read Arrow batch from stream") + || e.to_string() + .contains("Arrow IPC payload contains no data batches") { ( StatusCode::BAD_REQUEST, diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 4695ac57..9447fd2d 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,6 +1,8 @@ use anyhow::{Result, anyhow}; use sensapp::storage::{StorageInstance, storage_factory::create_storage_from_connection_string}; -use sensapp::test_utils::get_test_database_url; +use sensapp::test_utils::{ + ensure_test_database_exists, get_test_database_url, isolate_test_database_url, +}; use std::sync::Arc; pub mod db; @@ -13,6 +15,9 @@ pub enum DatabaseType { PostgreSQL, SQLite, ClickHouse, + TimescaleDB, + DuckDB, + RRDcached, } impl DatabaseType { @@ -23,8 +28,18 @@ impl DatabaseType { DatabaseType::SQLite => std::env::var("TEST_DATABASE_URL") .unwrap_or_else(|_| "sqlite://test.db".to_string()), DatabaseType::ClickHouse => std::env::var("TEST_DATABASE_URL").unwrap_or_else(|_| { - "clickhouse://default:password@localhost:9000/sensapp_test".to_string() + "clickhouse://default:password@localhost:8123/sensapp_test".to_string() }), + DatabaseType::TimescaleDB => { + let url = std::env::var("TEST_DATABASE_URL").unwrap_or_else(|_| { + "timescaledb://postgres:postgres@localhost:5433/sensapp-test".to_string() + }); + isolate_test_database_url(&url) + } + DatabaseType::DuckDB => std::env::var("TEST_DATABASE_URL") + .unwrap_or_else(|_| "duckdb://test.duckdb".to_string()), + DatabaseType::RRDcached => std::env::var("TEST_DATABASE_URL") + .unwrap_or_else(|_| "rrdcached://127.0.0.1:42217?preset=hoarder".to_string()), } } @@ -34,6 +49,12 @@ impl DatabaseType { DatabaseType::SQLite } else if connection_string.starts_with("clickhouse://") { DatabaseType::ClickHouse + } else if connection_string.starts_with("timescaledb://") { + DatabaseType::TimescaleDB + } else if connection_string.starts_with("duckdb://") { + DatabaseType::DuckDB + } else if connection_string.starts_with("rrdcached://") { + DatabaseType::RRDcached } else { // Default to PostgreSQL for postgres://, postgresql://, or any other prefix DatabaseType::PostgreSQL @@ -42,14 +63,31 @@ impl DatabaseType { /// Get the database type from environment variables or use default pub fn from_env() -> Self { - let connection_string = get_test_database_url(); - Self::from_connection_string(&connection_string) + if let Ok(connection_string) = std::env::var("TEST_DATABASE_URL") { + return Self::from_connection_string(&connection_string); + } + + if cfg!(feature = "timescaledb") { + DatabaseType::TimescaleDB + } else if cfg!(feature = "clickhouse") { + DatabaseType::ClickHouse + } else if cfg!(feature = "duckdb") { + DatabaseType::DuckDB + } else if cfg!(feature = "sqlite") { + DatabaseType::SQLite + } else if cfg!(feature = "rrdcached") { + DatabaseType::RRDcached + } else { + DatabaseType::PostgreSQL + } } } /// Test database manager that creates isolated test databases pub struct TestDb { + #[allow(dead_code)] // Kept for diagnostics and per-test context pub db_name: String, + #[allow(dead_code)] // Kept for diagnostics and per-test context pub db_type: DatabaseType, #[allow(dead_code)] // Used by some tests pub storage: Arc, @@ -73,10 +111,20 @@ impl TestDb { DatabaseType::PostgreSQL => "sensapp".to_string(), DatabaseType::SQLite => "test.db".to_string(), DatabaseType::ClickHouse => "sensapp_test".to_string(), + DatabaseType::TimescaleDB => "sensapp-test".to_string(), + DatabaseType::DuckDB => "test.duckdb".to_string(), + DatabaseType::RRDcached => "rrdcached".to_string(), }; let connection_string = db_type.default_connection_string(); + if matches!( + db_type, + DatabaseType::PostgreSQL | DatabaseType::TimescaleDB + ) { + ensure_test_database_exists(&connection_string).await?; + } + // Connect to the database let storage = create_storage_from_connection_string(&connection_string) .await @@ -130,16 +178,6 @@ impl TestDb { } } -impl Drop for TestDb { - fn drop(&mut self) { - // Async cleanup would be better, but this ensures cleanup happens - println!( - "Test database {} ({:?}) cleaned up", - self.db_name, self.db_type - ); - } -} - /// Helper trait for easier testing pub trait TestHelpers { #[allow(dead_code)] // Test helper method @@ -151,13 +189,13 @@ pub trait TestHelpers { impl TestHelpers for Arc { async fn expect_sensor_count(&self, expected: usize) -> Result<()> { - let sensors = self.list_series(None).await?; - if sensors.len() != expected { + let result = self.list_series(None, None, None).await?; + if result.series.len() != expected { return Err(anyhow!( "Expected {} sensors, found {}. Sensors: {:#?}", expected, - sensors.len(), - sensors + result.series.len(), + result.series )); } Ok(()) diff --git a/tests/crud_dcat_api.rs b/tests/crud_dcat_api.rs index 1f201fe3..ee3a834e 100644 --- a/tests/crud_dcat_api.rs +++ b/tests/crud_dcat_api.rs @@ -1,3 +1,5 @@ +#![cfg_attr(feature = "rrdcached", allow(unused_imports, dead_code))] + mod common; use anyhow::Result; @@ -5,8 +7,12 @@ use axum::http::StatusCode; use common::http::TestApp; use common::{TestDb, fixtures}; use sensapp::config::load_configuration_for_tests; +use sensapp::datamodel::batch_builder::BatchBuilder; +use sensapp::datamodel::{Sample, Sensor, SensorType, TypedSamples}; use serde_json::Value; use serial_test::serial; +use std::sync::Arc; +use uuid::Uuid; // Ensure configuration is loaded once for all tests in this module static INIT: std::sync::Once = std::sync::Once::new(); @@ -16,7 +22,45 @@ fn ensure_config() { }); } +fn create_float_sensor(name: &str, start_value: f64) -> (Sensor, TypedSamples) { + let sensor = Sensor::new( + Uuid::new_v4(), + name.to_string(), + SensorType::Float, + None, + None, + ); + let samples = TypedSamples::Float(smallvec::smallvec![ + Sample { + datetime: hifitime::Epoch::from_unix_seconds(1704067200.0), + value: start_value, + }, + Sample { + datetime: hifitime::Epoch::from_unix_seconds(1704067260.0), + value: start_value + 0.5, + } + ]); + (sensor, samples) +} + +async fn seed_three_series(storage: &Arc) -> Result<()> { + let mut batch_builder = BatchBuilder::new()?; + + for (sensor, samples) in [ + create_float_sensor("temperature_test", 20.0), + create_float_sensor("humidity_test", 60.0), + create_float_sensor("pressure_test", 1013.0), + ] { + let sensor = Arc::new(sensor); + batch_builder.add(sensor.clone(), samples).await?; + } + + batch_builder.send_what_is_left(storage.clone()).await?; + Ok(()) +} + /// Test CRUD/DCAT API functionality with new series terminology +#[cfg(not(feature = "rrdcached"))] mod crud_dcat_tests { use super::*; @@ -30,8 +74,7 @@ mod crud_dcat_tests { let app = TestApp::new(storage.clone()).await; // Ingest data from multiple sensors with same name but different labels - let csv_data = fixtures::multi_sensor_csv(); - app.post_csv("/sensors/publish", &csv_data).await?; + seed_three_series(&storage).await?; // When: We query the metrics endpoint let response = app.get("/metrics").await?; @@ -230,8 +273,7 @@ mod crud_dcat_tests { let app = TestApp::new(storage.clone()).await; // Ingest data with labels (using multi-sensor CSV which should have varied data) - let csv_data = fixtures::multi_sensor_csv(); - app.post_csv("/sensors/publish", &csv_data).await?; + seed_three_series(&storage).await?; // When: We query the series endpoint let response = app.get("/series").await?; @@ -512,4 +554,271 @@ mod crud_dcat_tests { Ok(()) } + + #[tokio::test] + #[serial] + async fn test_series_pagination_basic() -> Result<()> { + ensure_config(); + // Given: A database with multiple sensors + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + // Seed multiple sensors directly so pagination tests do not depend on importer behavior + seed_three_series(&storage).await?; + + let seeded_page = storage.list_series(None, Some(2), None).await?; + assert_eq!( + seeded_page.series.len(), + 2, + "Seeded first page should have 2 series" + ); + assert!( + seeded_page.bookmark.is_some(), + "Seeded storage pagination should produce a bookmark" + ); + + // When: We query with a limit of 2 + let response = app.get("/series?limit=2").await?; + + // Then: Should get 2 series and a bookmark + response.assert_status(StatusCode::OK); + let catalog: Value = response.json()?; + + let datasets = catalog["dcat:dataset"].as_array().unwrap(); + assert_eq!(datasets.len(), 2, "Should return exactly 2 series"); + + // Should have hydra:view with next link + assert!(catalog["hydra:view"].is_object(), "Should have hydra:view"); + assert_eq!( + catalog["hydra:view"]["@type"], + "hydra:PartialCollectionView" + ); + assert!( + catalog["hydra:view"]["hydra:next"].is_string(), + "Should have next link" + ); + assert_eq!(catalog["hydra:view"]["hydra:itemsPerPage"], 2); + + // Check Link header + let link_header = response.headers().get("Link"); + assert!(link_header.is_some(), "Should have Link header"); + let link_str = link_header.unwrap().to_str().unwrap(); + assert!( + link_str.contains("rel=\"next\""), + "Link header should have rel=next" + ); + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_series_pagination_with_bookmark() -> Result<()> { + ensure_config(); + // Given: A database with multiple sensors + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + // Seed multiple sensors directly so pagination tests do not depend on importer behavior + seed_three_series(&storage).await?; + + let seeded_page = storage.list_series(None, Some(2), None).await?; + assert_eq!( + seeded_page.series.len(), + 2, + "Seeded first page should have 2 series" + ); + assert!( + seeded_page.bookmark.is_some(), + "Seeded storage pagination should produce a bookmark" + ); + + // When: We query the first page + let first_response = app.get("/series?limit=2").await?; + first_response.assert_status(StatusCode::OK); + let first_catalog: Value = first_response.json()?; + + let first_datasets = first_catalog["dcat:dataset"].as_array().unwrap(); + let first_ids: Vec = first_datasets + .iter() + .map(|d| d["dct:identifier"].as_str().unwrap().to_string()) + .collect(); + + // Extract bookmark from next link + let next_url = first_catalog["hydra:view"]["hydra:next"].as_str().unwrap(); + let bookmark = next_url + .split("bookmark=") + .nth(1) + .unwrap() + .split('&') + .next() + .unwrap(); + + // When: We query the second page with the bookmark + let second_response = app + .get(&format!("/series?limit=2&bookmark={}", bookmark)) + .await?; + second_response.assert_status(StatusCode::OK); + let second_catalog: Value = second_response.json()?; + + let second_datasets = second_catalog["dcat:dataset"].as_array().unwrap(); + let second_ids: Vec = second_datasets + .iter() + .map(|d| d["dct:identifier"].as_str().unwrap().to_string()) + .collect(); + + // Then: Second page should have different sensors + for first_id in &first_ids { + assert!( + !second_ids.contains(first_id), + "Second page should not contain sensors from first page" + ); + } + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_series_pagination_last_page() -> Result<()> { + ensure_config(); + // Given: A database with exactly 3 sensors + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + // Ingest temperature sensor data (creates 1 sensor) + let csv_data = fixtures::temperature_sensor_csv(); + app.post_csv("/sensors/publish", &csv_data).await?; + + // When: We query with limit=5 (more than available) + let response = app.get("/series?limit=5").await?; + + // Then: Should get all sensors and NO bookmark + response.assert_status(StatusCode::OK); + let catalog: Value = response.json()?; + + // Should NOT have hydra:view (no next page) + assert!( + catalog["hydra:view"].is_null(), + "Should not have hydra:view on last page" + ); + + // Should NOT have Link header + let link_header = response.headers().get("Link"); + assert!( + link_header.is_none(), + "Should not have Link header on last page" + ); + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_series_pagination_with_metric_filter() -> Result<()> { + ensure_config(); + // Given: A database with sensors for different metrics + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + // Ingest temperature and multi-sensor data + let csv_temp = fixtures::temperature_sensor_csv(); + app.post_csv("/sensors/publish", &csv_temp).await?; + + // Get first sensor name from temperature data + let all_response = app.get("/series").await?; + all_response.assert_status(StatusCode::OK); + let all_catalog: Value = all_response.json()?; + let first_metric = all_catalog["dcat:dataset"][0]["dct:title"] + .as_str() + .unwrap(); + + // When: We query with metric filter and pagination + let response = app + .get(&format!("/series?metric={}&limit=1", first_metric)) + .await?; + + // Then: Should only get sensors for that metric + response.assert_status(StatusCode::OK); + let catalog: Value = response.json()?; + + let datasets = catalog["dcat:dataset"].as_array().unwrap(); + for dataset in datasets { + let title = dataset["dct:title"].as_str().unwrap(); + assert_eq!( + title, first_metric, + "All results should be for metric {}, got: {}", + first_metric, title + ); + } + + // If there's a next link, it should include the metric filter + if let Some(next_url) = catalog["hydra:view"]["hydra:next"].as_str() { + assert!( + next_url.contains(&format!("metric={}", urlencoding::encode(first_metric))), + "Next link should preserve metric filter" + ); + } + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_series_pagination_max_limit() -> Result<()> { + ensure_config(); + // Given: A database with sensors + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + let csv_data = fixtures::multi_sensor_csv(); + app.post_csv("/sensors/publish", &csv_data).await?; + + // When: We query with limit exceeding MAX_LIST_SERIES_LIMIT + let response = app.get("/series?limit=99999").await?; + + // Then: Should cap at MAX_LIST_SERIES_LIMIT + response.assert_status(StatusCode::OK); + let catalog: Value = response.json()?; + + // The actual limit used should be capped + if let Some(items_per_page) = catalog["hydra:view"]["hydra:itemsPerPage"].as_u64() { + assert!( + items_per_page <= 16384, + "Should cap limit at MAX_LIST_SERIES_LIMIT (16384), got {}", + items_per_page + ); + } + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_series_pagination_invalid_bookmark() -> Result<()> { + ensure_config(); + // Given: A database with sensors + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + let csv_data = fixtures::multi_sensor_csv(); + app.post_csv("/sensors/publish", &csv_data).await?; + + // When: We query with an invalid bookmark + let response = app.get("/series?bookmark=invalid").await; + + // Then: Should return an error + assert!( + response.is_err() || response.as_ref().unwrap().status() != StatusCode::OK, + "Should reject invalid bookmark" + ); + + Ok(()) + } } diff --git a/tests/influxdb_integration.rs b/tests/influxdb_integration.rs new file mode 100644 index 00000000..9bd73aa2 --- /dev/null +++ b/tests/influxdb_integration.rs @@ -0,0 +1,335 @@ +#![cfg(not(feature = "rrdcached"))] + +mod common; + +use anyhow::Result; +use axum::http::StatusCode; +use common::TestDb; +use common::db::DbHelpers; +use common::http::TestApp; +use sensapp::config::load_configuration_for_tests; +use serial_test::serial; + +static INIT: std::sync::Once = std::sync::Once::new(); +fn ensure_config() { + INIT.call_once(|| { + load_configuration_for_tests().expect("Failed to load configuration for tests"); + }); +} + +/// Test basic InfluxDB line protocol ingestion +#[tokio::test] +#[serial] +async fn test_influxdb_basic_write() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + let influx_data = "cpu,host=serverA,region=us-west usage_idle=99.1 1704067200000000000\n\ + cpu,host=serverA,region=us-west usage_idle=98.5 1704067260000000000\n\ + cpu,host=serverA,region=us-west usage_idle=97.8 1704067320000000000"; + + let response = app + .post_influxdb("/api/v2/write?bucket=test&org=sensapp", influx_data) + .await?; + + response.assert_status(StatusCode::NO_CONTENT); + + // Verify data was stored — InfluxDB creates sensor names as "measurement field_key" + let sensor_name = "cpu usage_idle"; + let sensor = DbHelpers::get_sensor_by_name(&storage, sensor_name).await?; + assert!(sensor.is_some(), "Sensor '{}' should exist", sensor_name); + + let sensor_data = DbHelpers::verify_sensor_data(&storage, sensor_name, 3).await?; + + // InfluxDB float values should be stored as Float type + if let sensapp::datamodel::TypedSamples::Float(samples) = &sensor_data.samples { + assert_eq!(samples[0].value, 99.1); + assert_eq!(samples[1].value, 98.5); + assert_eq!(samples[2].value, 97.8); + } else { + panic!("Expected float samples for cpu sensor"); + } + + Ok(()) +} + +/// Test InfluxDB write with multiple measurements +#[tokio::test] +#[serial] +async fn test_influxdb_multiple_measurements() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + let influx_data = "temperature,room=kitchen value=22.5 1704067200000000000\n\ + humidity,room=kitchen value=65.0 1704067200000000000\n\ + temperature,room=kitchen value=23.0 1704067260000000000\n\ + humidity,room=kitchen value=64.0 1704067260000000000"; + + let response = app + .post_influxdb("/api/v2/write?bucket=test&org=sensapp", influx_data) + .await?; + + response.assert_status(StatusCode::NO_CONTENT); + + // Both sensors should exist + let temp_sensor = DbHelpers::get_sensor_by_name(&storage, "temperature value").await?; + assert!(temp_sensor.is_some(), "Temperature sensor should exist"); + + let humidity_sensor = DbHelpers::get_sensor_by_name(&storage, "humidity value").await?; + assert!(humidity_sensor.is_some(), "Humidity sensor should exist"); + + // Verify sample counts + DbHelpers::verify_sensor_data(&storage, "temperature value", 2).await?; + DbHelpers::verify_sensor_data(&storage, "humidity value", 2).await?; + + Ok(()) +} + +/// Test InfluxDB write with multiple fields per measurement +#[tokio::test] +#[serial] +async fn test_influxdb_multiple_fields() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + // InfluxDB supports multiple fields per line — each becomes a separate sensor + let influx_data = + "weather,city=oslo temperature=5.0,humidity=80.0,wind_speed=12.3 1704067200000000000"; + + let response = app + .post_influxdb("/api/v2/write?bucket=test&org=sensapp", influx_data) + .await?; + + response.assert_status(StatusCode::NO_CONTENT); + + // Each field creates a separate sensor named "measurement field_key" + DbHelpers::verify_sensor_data(&storage, "weather temperature", 1).await?; + DbHelpers::verify_sensor_data(&storage, "weather humidity", 1).await?; + DbHelpers::verify_sensor_data(&storage, "weather wind_speed", 1).await?; + + Ok(()) +} + +/// Test InfluxDB write with integer fields +#[tokio::test] +#[serial] +async fn test_influxdb_integer_fields() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + // InfluxDB integer values end with 'i' + let influx_data = "counter,host=web1 requests=42i 1704067200000000000\n\ + counter,host=web1 requests=100i 1704067260000000000"; + + let response = app + .post_influxdb("/api/v2/write?bucket=test&org=sensapp", influx_data) + .await?; + + response.assert_status(StatusCode::NO_CONTENT); + + let sensor_data = DbHelpers::verify_sensor_data(&storage, "counter requests", 2).await?; + + if let sensapp::datamodel::TypedSamples::Integer(samples) = &sensor_data.samples { + assert_eq!(samples[0].value, 42); + assert_eq!(samples[1].value, 100); + } else { + panic!("Expected integer samples for counter sensor"); + } + + Ok(()) +} + +/// Test InfluxDB write with boolean fields +#[tokio::test] +#[serial] +async fn test_influxdb_boolean_fields() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + let influx_data = "system,host=web1 online=true 1704067200000000000\n\ + system,host=web1 online=false 1704067260000000000"; + + let response = app + .post_influxdb("/api/v2/write?bucket=test&org=sensapp", influx_data) + .await?; + + response.assert_status(StatusCode::NO_CONTENT); + + let sensor_data = DbHelpers::verify_sensor_data(&storage, "system online", 2).await?; + + if let sensapp::datamodel::TypedSamples::Boolean(samples) = &sensor_data.samples { + assert!(samples[0].value); + assert!(!samples[1].value); + } else { + panic!("Expected boolean samples for system online sensor"); + } + + Ok(()) +} + +/// Test InfluxDB write requires org or org_id +#[tokio::test] +#[serial] +async fn test_influxdb_requires_org() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + let influx_data = "cpu,host=serverA usage_idle=99.1 1704067200000000000"; + + // Missing org and org_id should return 400 + let response = app + .post_influxdb("/api/v2/write?bucket=test", influx_data) + .await?; + + response.assert_status(StatusCode::BAD_REQUEST); + + Ok(()) +} + +/// Test InfluxDB write with different precision values +#[tokio::test] +#[serial] +async fn test_influxdb_precision_seconds() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + // Timestamp in seconds precision + let influx_data = "temp,room=lab value=25.0 1704067200"; + + let response = app + .post_influxdb( + "/api/v2/write?bucket=test&org=sensapp&precision=s", + influx_data, + ) + .await?; + + response.assert_status(StatusCode::NO_CONTENT); + + let sensor_data = DbHelpers::verify_sensor_data(&storage, "temp value", 1).await?; + + if let sensapp::datamodel::TypedSamples::Float(samples) = &sensor_data.samples { + assert_eq!(samples[0].value, 25.0); + } else { + panic!("Expected float samples for temp sensor"); + } + + Ok(()) +} + +/// Test InfluxDB write with tags stored as labels +#[tokio::test] +#[serial] +async fn test_influxdb_tags_become_labels() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + let influx_data = "cpu,host=serverA,region=eu-west usage_idle=95.0 1704067200000000000"; + + let response = app + .post_influxdb("/api/v2/write?bucket=mybucket&org=myorg", influx_data) + .await?; + + response.assert_status(StatusCode::NO_CONTENT); + + let sensor = DbHelpers::get_sensor_by_name(&storage, "cpu usage_idle") + .await? + .expect("Sensor should exist"); + + // Tags should be stored as labels + let label_names: Vec<&str> = sensor.labels.iter().map(|(n, _)| n.as_str()).collect(); + assert!( + label_names.contains(&"host"), + "Should have 'host' label, got: {:?}", + label_names + ); + assert!( + label_names.contains(&"region"), + "Should have 'region' label, got: {:?}", + label_names + ); + // InfluxDB adapter also adds bucket and org as labels + assert!( + label_names.contains(&"influxdb_bucket"), + "Should have 'influxdb_bucket' label, got: {:?}", + label_names + ); + assert!( + label_names.contains(&"influxdb_org"), + "Should have 'influxdb_org' label, got: {:?}", + label_names + ); + + Ok(()) +} + +/// Test InfluxDB write with empty body +#[tokio::test] +#[serial] +async fn test_influxdb_empty_body() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + let response = app + .post_influxdb("/api/v2/write?bucket=test&org=sensapp", "") + .await?; + + // Empty body should succeed (no data to ingest) + response.assert_status(StatusCode::NO_CONTENT); + + Ok(()) +} + +/// Test InfluxDB write with gzip compression +#[tokio::test] +#[serial] +async fn test_influxdb_gzip_compression() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + let influx_data = "pressure,sensor=bme280 value=1013.25 1704067200000000000"; + + // Compress the data with gzip + use flate2::Compression; + use flate2::write::GzEncoder; + use std::io::Write; + + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(influx_data.as_bytes())?; + let compressed = encoder.finish()?; + + // Send with content-encoding: gzip + let request = axum::http::Request::builder() + .method("POST") + .uri("/api/v2/write?bucket=test&org=sensapp") + .header("content-type", "text/plain") + .header("content-encoding", "gzip") + .body(axum::body::Body::from(compressed))?; + + let response = app.raw_request(request).await?; + + response.assert_status(StatusCode::NO_CONTENT); + + DbHelpers::verify_sensor_data(&storage, "pressure value", 1).await?; + + Ok(()) +} diff --git a/tests/ingestion.rs b/tests/ingestion.rs index 388640a5..9f3493a0 100644 --- a/tests/ingestion.rs +++ b/tests/ingestion.rs @@ -256,3 +256,28 @@ async fn test_large_csv_ingestion() -> Result<()> { Ok(()) } + +/// Test that hyphenated sensor names are accepted during ingestion +#[tokio::test] +#[serial] +async fn test_csv_ingestion_hyphenated_sensor_name() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + let csv_data = + "datetime,sensor_name,value,unit\n2024-01-01T00:00:00Z,demo-temperature,21.5,°C\n"; + let response = app.post_csv("/sensors/publish", csv_data).await?; + + response.assert_status(StatusCode::OK); + + let sensor = DbHelpers::get_sensor_by_name(&storage, "demo-temperature") + .await? + .expect("Hyphenated sensor should exist"); + + assert_eq!(sensor.name, "demo-temperature"); + DbHelpers::verify_sensor_data(&storage, "demo-temperature", 1).await?; + + Ok(()) +} diff --git a/tests/jwt_auth.rs b/tests/jwt_auth.rs new file mode 100644 index 00000000..f698f27a --- /dev/null +++ b/tests/jwt_auth.rs @@ -0,0 +1,598 @@ +//! Integration tests for optional JWT authentication. +//! +//! These tests verify that: +//! - Without auth configured (`auth: None`), all endpoints are open. +//! - With auth configured, read endpoints require a "read" scope token. +//! - With auth configured, write endpoints require a "write" scope token. +//! - Expired and not-yet-valid tokens are rejected. +//! - Sensor allow lists are enforced. +//! - Health/docs/prometheus-metrics remain public even with auth enabled. + +mod common; + +use axum::Router; +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use axum::routing::{get, post}; +use common::TestDb; +use jsonwebtoken::{EncodingKey, Header, encode}; +use sensapp::http::auth::AuthConfig; +use sensapp::http::crud::{get_series_data, list_metrics, list_series}; +use sensapp::http::health::{liveness, readiness}; +use sensapp::http::metrics::{HttpMetrics, prometheus_metrics}; +use sensapp::http::server::publish_senml_data; +use sensapp::http::state::HttpServerState; +use sensapp::storage::StorageInstance; +use serde::Serialize; +use serial_test::serial; +use std::sync::Arc; +use tower::ServiceExt; + +const TEST_SECRET: &str = "test-secret-0123456789abcdef012345"; + +// --------------------------------------------------------------------------- +// Test claims helper +// --------------------------------------------------------------------------- + +#[derive(Serialize)] +struct TestClaims { + sub: String, + exp: u64, + #[serde(skip_serializing_if = "Option::is_none")] + nbf: Option, + #[serde(skip_serializing_if = "Option::is_none")] + iat: Option, + #[serde(skip_serializing_if = "Option::is_none")] + scope: Option, + #[serde(skip_serializing_if = "Option::is_none")] + sensors: Option>, +} + +fn make_token(claims: &TestClaims) -> String { + encode( + &Header::default(), + claims, + &EncodingKey::from_secret(TEST_SECRET.as_bytes()), + ) + .expect("token encoding should succeed") +} + +fn read_write_token() -> String { + make_token(&TestClaims { + sub: "test-user".into(), + exp: 4_102_444_800, // year 2100 + nbf: Some(0), + iat: Some(0), + scope: Some("read write".into()), + sensors: None, + }) +} + +fn read_only_token() -> String { + make_token(&TestClaims { + sub: "reader".into(), + exp: 4_102_444_800, + nbf: Some(0), + iat: Some(0), + scope: Some("read".into()), + sensors: None, + }) +} + +fn write_only_token() -> String { + make_token(&TestClaims { + sub: "writer".into(), + exp: 4_102_444_800, + nbf: Some(0), + iat: Some(0), + scope: Some("write".into()), + sensors: None, + }) +} + +fn expired_token() -> String { + make_token(&TestClaims { + sub: "old".into(), + exp: 0, // already expired + nbf: None, + iat: None, + scope: Some("read write".into()), + sensors: None, + }) +} + +fn not_yet_valid_token() -> String { + make_token(&TestClaims { + sub: "early".into(), + exp: 4_102_444_800, + nbf: Some(4_102_444_800), // not valid until year 2100 + iat: None, + scope: Some("read write".into()), + sensors: None, + }) +} + +// --------------------------------------------------------------------------- +// Router builder – mirrors the real server's route groups +// --------------------------------------------------------------------------- + +/// Unified test publish handler +async fn test_publish_handler( + axum::extract::State(state): axum::extract::State, + headers: axum::http::HeaderMap, + body: Body, +) -> Result { + let content_type = headers + .get("content-type") + .and_then(|v| v.to_str().ok()) + .unwrap_or("text/csv"); + + if content_type.contains("application/json") { + let body_bytes = axum::body::to_bytes(body, usize::MAX) + .await + .map_err(|e| (StatusCode::BAD_REQUEST, format!("body: {e}")))?; + let json_str = String::from_utf8(body_bytes.to_vec()) + .map_err(|e| (StatusCode::BAD_REQUEST, format!("utf8: {e}")))?; + publish_senml_data(&json_str, state.storage.clone()) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("{e:?}")))?; + } + Ok("ok".to_string()) +} + +/// Build a test router with the same auth layering as the real server. +fn build_test_router(storage: Arc, auth: Option) -> Router { + use sensapp::http::auth::{require_read_auth, require_write_auth}; + + let state = HttpServerState { + name: Arc::new("SensApp Auth Test".into()), + storage, + metrics: Arc::new(HttpMetrics::new()), + influxdb_with_numeric: false, + auth: auth.clone(), + }; + + let public = Router::new() + .route("/", get(|| async { "SensApp" })) + .route("/prometheus/metrics", get(prometheus_metrics)) + .route("/health/live", get(liveness)) + .route("/health/ready", get(readiness)); + + let read_routes = Router::new() + .route("/metrics", get(list_metrics)) + .route("/series", get(list_series)) + .route("/series/{series_uuid}", get(get_series_data)) + .route_layer(axum::middleware::from_fn_with_state( + auth.clone(), + require_read_auth, + )); + + let write_routes = Router::new() + .route("/publish", post(test_publish_handler)) + .route_layer(axum::middleware::from_fn_with_state( + auth, + require_write_auth, + )); + + public + .merge(read_routes) + .merge(write_routes) + .with_state(state) +} + +// --------------------------------------------------------------------------- +// Tests: no auth configured (everything open) +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +async fn no_auth_read_endpoint_is_open() { + let db = TestDb::new().await.expect("test db"); + let app = build_test_router(db.storage(), None); + + let resp = app + .oneshot(Request::get("/metrics").body(Body::empty()).unwrap()) + .await + .unwrap(); + // Should succeed (200) or at least not be 401/403 + assert_ne!(resp.status(), StatusCode::UNAUTHORIZED); + assert_ne!(resp.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +#[serial] +async fn no_auth_write_endpoint_is_open() { + let db = TestDb::new().await.expect("test db"); + let app = build_test_router(db.storage(), None); + + let resp = app + .oneshot( + Request::post("/publish") + .header("content-type", "application/json") + .body(Body::from( + r#"[{"n":"temperature","u":"Cel","v":22.5,"t":1700000000}]"#, + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_ne!(resp.status(), StatusCode::UNAUTHORIZED); + assert_ne!(resp.status(), StatusCode::FORBIDDEN); +} + +// --------------------------------------------------------------------------- +// Tests: auth configured — valid tokens +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +async fn auth_read_endpoint_with_valid_token() { + let db = TestDb::new().await.expect("test db"); + let auth = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let app = build_test_router(db.storage(), Some(auth)); + + let resp = app + .oneshot( + Request::get("/metrics") + .header("authorization", format!("Bearer {}", read_write_token())) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_ne!(resp.status(), StatusCode::UNAUTHORIZED); + assert_ne!(resp.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +#[serial] +async fn auth_write_endpoint_with_valid_token() { + let db = TestDb::new().await.expect("test db"); + let auth = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let app = build_test_router(db.storage(), Some(auth)); + + let resp = app + .oneshot( + Request::post("/publish") + .header("authorization", format!("Bearer {}", read_write_token())) + .header("content-type", "application/json") + .body(Body::from( + r#"[{"n":"temperature","u":"Cel","v":22.5,"t":1700000000}]"#, + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_ne!(resp.status(), StatusCode::UNAUTHORIZED); + assert_ne!(resp.status(), StatusCode::FORBIDDEN); +} + +// --------------------------------------------------------------------------- +// Tests: auth configured — missing/invalid tokens +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +async fn auth_read_endpoint_rejects_no_token() { + let db = TestDb::new().await.expect("test db"); + let auth = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let app = build_test_router(db.storage(), Some(auth)); + + let resp = app + .oneshot(Request::get("/metrics").body(Body::empty()).unwrap()) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +#[serial] +async fn auth_write_endpoint_rejects_no_token() { + let db = TestDb::new().await.expect("test db"); + let auth = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let app = build_test_router(db.storage(), Some(auth)); + + let resp = app + .oneshot( + Request::post("/publish") + .header("content-type", "application/json") + .body(Body::from("[]")) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +#[serial] +async fn auth_rejects_expired_token() { + let db = TestDb::new().await.expect("test db"); + let auth = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let app = build_test_router(db.storage(), Some(auth)); + + let resp = app + .oneshot( + Request::get("/metrics") + .header("authorization", format!("Bearer {}", expired_token())) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +#[serial] +async fn auth_rejects_not_yet_valid_token() { + let db = TestDb::new().await.expect("test db"); + let auth = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let app = build_test_router(db.storage(), Some(auth)); + + let resp = app + .oneshot( + Request::get("/metrics") + .header("authorization", format!("Bearer {}", not_yet_valid_token())) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +#[serial] +async fn auth_rejects_wrong_secret() { + let db = TestDb::new().await.expect("test db"); + let auth = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let app = build_test_router(db.storage(), Some(auth)); + + // Sign with a different secret + let wrong_token = encode( + &Header::default(), + &TestClaims { + sub: "hacker".into(), + exp: 4_102_444_800, + nbf: None, + iat: None, + scope: Some("read write".into()), + sensors: None, + }, + &EncodingKey::from_secret(b"wrong-secret-that-is-long-enough!!"), + ) + .unwrap(); + + let resp = app + .oneshot( + Request::get("/metrics") + .header("authorization", format!("Bearer {wrong_token}")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); +} + +// --------------------------------------------------------------------------- +// Tests: scope enforcement +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +async fn read_only_token_can_read() { + let db = TestDb::new().await.expect("test db"); + let auth = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let app = build_test_router(db.storage(), Some(auth)); + + let resp = app + .oneshot( + Request::get("/metrics") + .header("authorization", format!("Bearer {}", read_only_token())) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_ne!(resp.status(), StatusCode::UNAUTHORIZED); + assert_ne!(resp.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +#[serial] +async fn read_only_token_cannot_write() { + let db = TestDb::new().await.expect("test db"); + let auth = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let app = build_test_router(db.storage(), Some(auth)); + + let resp = app + .oneshot( + Request::post("/publish") + .header("authorization", format!("Bearer {}", read_only_token())) + .header("content-type", "application/json") + .body(Body::from("[]")) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +#[serial] +async fn write_only_token_cannot_read() { + let db = TestDb::new().await.expect("test db"); + let auth = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let app = build_test_router(db.storage(), Some(auth)); + + let resp = app + .oneshot( + Request::get("/metrics") + .header("authorization", format!("Bearer {}", write_only_token())) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +#[serial] +async fn write_only_token_can_write() { + let db = TestDb::new().await.expect("test db"); + let auth = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let app = build_test_router(db.storage(), Some(auth)); + + let resp = app + .oneshot( + Request::post("/publish") + .header("authorization", format!("Bearer {}", write_only_token())) + .header("content-type", "application/json") + .body(Body::from( + r#"[{"n":"temperature","u":"Cel","v":22.5,"t":1700000000}]"#, + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_ne!(resp.status(), StatusCode::UNAUTHORIZED); + assert_ne!(resp.status(), StatusCode::FORBIDDEN); +} + +// --------------------------------------------------------------------------- +// Tests: public endpoints stay open even with auth enabled +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +async fn health_live_is_always_public() { + let db = TestDb::new().await.expect("test db"); + let auth = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let app = build_test_router(db.storage(), Some(auth)); + + let resp = app + .oneshot(Request::get("/health/live").body(Body::empty()).unwrap()) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::OK); +} + +#[tokio::test] +#[serial] +async fn health_ready_is_always_public() { + let db = TestDb::new().await.expect("test db"); + let auth = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let app = build_test_router(db.storage(), Some(auth)); + + let resp = app + .oneshot(Request::get("/health/ready").body(Body::empty()).unwrap()) + .await + .unwrap(); + + // health/ready returns 200 or 503 depending on db health; never 401/403 + assert_ne!(resp.status(), StatusCode::UNAUTHORIZED); + assert_ne!(resp.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +#[serial] +async fn prometheus_metrics_is_always_public() { + let db = TestDb::new().await.expect("test db"); + let auth = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let app = build_test_router(db.storage(), Some(auth)); + + let resp = app + .oneshot( + Request::get("/prometheus/metrics") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_ne!(resp.status(), StatusCode::UNAUTHORIZED); + assert_ne!(resp.status(), StatusCode::FORBIDDEN); +} + +// --------------------------------------------------------------------------- +// Tests: token creation via AuthConfig +// --------------------------------------------------------------------------- + +#[test] +fn auth_config_create_token_roundtrip() { + let config = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let token = config + .create_token("my-service", "read write", 3600, None) + .expect("token creation"); + + // Validate it by decoding + let mut headers = axum::http::HeaderMap::new(); + headers.insert( + axum::http::header::AUTHORIZATION, + format!("Bearer {token}").parse().unwrap(), + ); + + let access = sensapp::http::auth::validate_token(&headers, &config).expect("should validate"); + assert_eq!(access.subject, "my-service"); + assert!(access.can_read); + assert!(access.can_write); +} + +#[test] +fn auth_config_create_token_with_sensor_filter() { + let config = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let token = config + .create_token( + "sensor-reader", + "read", + 3600, + Some(vec!["cpu_temp".to_string(), "fan_speed".to_string()]), + ) + .expect("token creation"); + + let mut headers = axum::http::HeaderMap::new(); + headers.insert( + axum::http::header::AUTHORIZATION, + format!("Bearer {token}").parse().unwrap(), + ); + + let access = sensapp::http::auth::validate_token(&headers, &config).expect("should validate"); + assert!(access.can_read); + assert!(!access.can_write); + assert!(access.can_access_sensor("cpu_temp")); + assert!(access.can_access_sensor("fan_speed")); + assert!(!access.can_access_sensor("humidity")); +} + +#[test] +fn secret_too_short_is_rejected() { + let result = AuthConfig::from_secret("short"); + assert!(result.is_err()); +} + +#[test] +fn malformed_bearer_prefix_rejected() { + let config = AuthConfig::from_secret(TEST_SECRET).unwrap(); + let mut headers = axum::http::HeaderMap::new(); + headers.insert( + axum::http::header::AUTHORIZATION, + "Token some-garbage".parse().unwrap(), + ); + assert!(sensapp::http::auth::validate_token(&headers, &config).is_err()); +} diff --git a/tests/prometheus_metrics.rs b/tests/prometheus_metrics.rs new file mode 100644 index 00000000..21b1aecb --- /dev/null +++ b/tests/prometheus_metrics.rs @@ -0,0 +1,262 @@ +mod common; + +use anyhow::Result; +use axum::http::StatusCode; +use common::TestDb; +use common::http::TestApp; +use hifitime::Unit; +use sensapp::config::load_configuration_for_tests; +use sensapp::datamodel::batch_builder::BatchBuilder; +use sensapp::datamodel::sensapp_vec::SensAppLabels; +use sensapp::datamodel::{Sample, Sensor, SensorType, TypedSamples}; +use serial_test::serial; +use std::sync::Arc; +use uuid::Uuid; + +static INIT: std::sync::Once = std::sync::Once::new(); + +fn ensure_config() { + INIT.call_once(|| { + load_configuration_for_tests().expect("Failed to load configuration for tests"); + }); +} + +fn create_sensor(name: &str, sensor_type: SensorType, labels: Vec<(&str, &str)>) -> Sensor { + let labels: SensAppLabels = labels + .into_iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(); + + Sensor { + uuid: Uuid::new_v4(), + name: name.to_string(), + sensor_type, + unit: None, + labels, + } +} + +async fn publish_test_sensors( + storage: &Arc, + sensors_with_samples: Vec<(Sensor, TypedSamples)>, +) -> Result<()> { + let mut batch_builder = BatchBuilder::new()?; + + for (sensor, samples) in sensors_with_samples { + batch_builder.add(Arc::new(sensor), samples).await?; + } + + batch_builder.send_what_is_left(storage.clone()).await?; + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_prometheus_metrics_endpoint_exposes_http_metrics() -> Result<()> { + ensure_config(); + + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage).await; + + app.get("/health/live").await?.assert_status(StatusCode::OK); + app.get("/health/live").await?.assert_status(StatusCode::OK); + + let response = app.get("/prometheus/metrics").await?; + response.assert_status(StatusCode::OK); + response.assert_content_type("text/plain; version=0.0.4; charset=utf-8"); + response.assert_body_contains( + "sensapp_http_requests_total{method=\"GET\",path=\"/health/live\",status=\"200\"} 2", + ); + response.assert_body_contains("sensapp_http_requests_in_flight 0"); + response.assert_body_contains("sensapp_storage_ready 1"); + response.assert_body_contains("sensapp_uptime_seconds"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_prometheus_metrics_can_append_latest_compatible_samples() -> Result<()> { + ensure_config(); + + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + publish_test_sensors( + &storage, + vec![ + ( + create_sensor( + "room_temperature_celsius", + SensorType::Float, + vec![("room", "lab"), ("site", "alpha")], + ), + TypedSamples::Float(smallvec::smallvec![Sample { + datetime: sensapp::datamodel::SensAppDateTime::from_unix_seconds( + 1_704_067_201.0 + ), + value: 42.5, + }]), + ), + ( + create_sensor("demo-temperature", SensorType::Float, vec![("room", "lab")]), + TypedSamples::Float(smallvec::smallvec![Sample { + datetime: sensapp::datamodel::SensAppDateTime::from_unix_seconds( + 1_704_067_202.0 + ), + value: 99.0, + }]), + ), + ( + create_sensor("status_text", SensorType::String, vec![("room", "lab")]), + TypedSamples::String(smallvec::smallvec![Sample { + datetime: sensapp::datamodel::SensAppDateTime::from_unix_seconds( + 1_704_067_203.0 + ), + value: "ok".to_string(), + }]), + ), + ], + ) + .await?; + + let response = app + .get("/prometheus/metrics?include_latest_samples=true") + .await?; + response.assert_status(StatusCode::OK); + response.assert_body_contains("sensapp_storage_ready 1"); + response.assert_body_contains( + "room_temperature_celsius{room=\"lab\",site=\"alpha\"} 42.5 1704067201000", + ); + assert!(!response.body().contains("demo-temperature{")); + assert!(!response.body().contains("status_text{")); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_prometheus_metrics_latest_samples_honors_selector() -> Result<()> { + ensure_config(); + + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + publish_test_sensors( + &storage, + vec![ + ( + create_sensor( + "room_temperature_celsius", + SensorType::Float, + vec![("site", "alpha")], + ), + TypedSamples::Float(smallvec::smallvec![Sample { + datetime: sensapp::datamodel::SensAppDateTime::from_unix_seconds( + 1_704_067_210.0 + ), + value: 21.0, + }]), + ), + ( + create_sensor( + "room_temperature_celsius", + SensorType::Float, + vec![("site", "beta")], + ), + TypedSamples::Float(smallvec::smallvec![Sample { + datetime: sensapp::datamodel::SensAppDateTime::from_unix_seconds( + 1_704_067_220.0 + ), + value: 22.0, + }]), + ), + ], + ) + .await?; + + let response = app + .get("/prometheus/metrics?include_latest_samples=true&selector=%7Bsite%3D%22alpha%22%7D") + .await?; + + response.assert_status(StatusCode::OK); + response.assert_body_contains("room_temperature_celsius{site=\"alpha\"} 21 1704067210000"); + assert!(!response.body().contains("site=\"beta\"")); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_prometheus_metrics_exposes_write_and_read_operation_metrics() -> Result<()> { + ensure_config(); + + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + let now = sensapp::datamodel::SensAppDateTime::now().expect("current time should be available"); + + let response = app + .post_influxdb( + "/api/v2/write?bucket=sensapp&org=test", + &format!( + "cpu,host=web1 usage_system=64.2,usage_user=12.3 {}", + now.to_unix(Unit::Nanosecond).floor() as i64 + ), + ) + .await?; + response.assert_status(StatusCode::NO_CONTENT); + + publish_test_sensors( + &storage, + vec![( + create_sensor( + "room_temperature_celsius", + SensorType::Float, + vec![("room", "lab")], + ), + TypedSamples::Float(smallvec::smallvec![ + Sample { + datetime: now - hifitime::Duration::from_seconds(30.0), + value: 20.5, + }, + Sample { + datetime: now - hifitime::Duration::from_seconds(15.0), + value: 21.0, + }, + ]), + )], + ) + .await?; + + let response = app + .get("/api/v1/query?query=room_temperature_celsius") + .await?; + response.assert_status(StatusCode::OK); + + let metrics = app.get("/prometheus/metrics").await?; + metrics.assert_status(StatusCode::OK); + metrics.assert_body_contains( + "sensapp_operations_total{kind=\"write\",operation=\"influxdb_write\",status=\"success\"} 1", + ); + metrics.assert_body_contains( + "sensapp_series_processed_total{kind=\"write\",operation=\"influxdb_write\"} 2", + ); + metrics.assert_body_contains( + "sensapp_samples_processed_total{kind=\"write\",operation=\"influxdb_write\"} 2", + ); + metrics.assert_body_contains( + "sensapp_operations_total{kind=\"read\",operation=\"simple_promql_query\",status=\"success\"} 1", + ); + metrics.assert_body_contains( + "sensapp_series_processed_total{kind=\"read\",operation=\"simple_promql_query\"} 1", + ); + metrics.assert_body_contains( + "sensapp_samples_processed_total{kind=\"read\",operation=\"simple_promql_query\"} 2", + ); + + Ok(()) +} diff --git a/tests/prometheus_remote_read_integration.rs b/tests/prometheus_remote_read_integration.rs index 3e9b0be8..260c7043 100644 --- a/tests/prometheus_remote_read_integration.rs +++ b/tests/prometheus_remote_read_integration.rs @@ -18,10 +18,12 @@ use sensapp::config::load_configuration_for_tests; use sensapp::datamodel::batch_builder::BatchBuilder; use sensapp::datamodel::sensapp_vec::SensAppLabels; use sensapp::datamodel::{Sample, Sensor, SensorType, TypedSamples}; -use sensapp::ingestors::http::prometheus_read::prometheus_remote_read; -use sensapp::ingestors::http::state::HttpServerState; +use sensapp::http::metrics::HttpMetrics; +use sensapp::http::prometheus_read::prometheus_remote_read; +use sensapp::http::state::HttpServerState; use sensapp::parsing::prometheus::remote_read_models::{ - LabelMatcher as PromLabelMatcher, Query, ReadRequest, ReadResponse, label_matcher, read_request, + ChunkedReadResponse, LabelMatcher as PromLabelMatcher, Query, ReadHints, ReadRequest, + ReadResponse, label_matcher, read_request, }; use serial_test::serial; use std::io::Cursor; @@ -106,7 +108,9 @@ fn create_test_app(storage: Arc) -> Route let state = HttpServerState { name: Arc::new("SensApp Test".to_string()), storage, + metrics: Arc::new(HttpMetrics::new()), influxdb_with_numeric: false, + auth: None, }; Router::new() @@ -136,6 +140,33 @@ fn parse_remote_read_response(body: &[u8]) -> Result { Ok(response) } +fn parse_streamed_chunked_response(body: &[u8]) -> Result> { + let mut cursor = 0usize; + let mut responses = Vec::new(); + + while cursor < body.len() { + let mut length = 0usize; + let mut shift = 0usize; + + loop { + let byte = body[cursor]; + cursor += 1; + length |= ((byte & 0x7F) as usize) << shift; + if byte & 0x80 == 0 { + break; + } + shift += 7; + } + + cursor += 4; // Skip CRC32C. + let message_end = cursor + length; + responses.push(ChunkedReadResponse::decode(&body[cursor..message_end])?); + cursor = message_end; + } + + Ok(responses) +} + /// Send a remote read request and return the response async fn send_remote_read_request(app: &Router, body: Vec) -> Result<(StatusCode, Vec)> { let request = Request::builder() @@ -825,6 +856,121 @@ async fn test_remote_read_streamed_xor_chunks() -> Result<()> { Ok(()) } +/// Test streamed XOR chunks response for a series larger than one XOR chunk. +#[tokio::test] +#[serial] +async fn test_remote_read_streamed_xor_chunks_large_series() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + + let sensor = create_sensor_with_labels("large_chunked_metric", SensorType::Float, vec![]); + let sample_count = 70_000usize; + let times_ms: Vec = (0..sample_count) + .map(|index| BASE_TS_MS + index as i64 * 1000) + .collect(); + let values: Vec = (0..sample_count).map(|index| index as f64).collect(); + + publish_test_sensors( + &storage, + vec![(sensor, create_float_samples_at_times(×_ms, &values))], + ) + .await?; + + let app = create_test_app(storage); + let query = Query { + start_timestamp_ms: BASE_TS_MS - 1000, + end_timestamp_ms: BASE_TS_MS + sample_count as i64 * 1000, + matchers: vec![PromLabelMatcher { + r#type: label_matcher::Type::Eq as i32, + name: "__name__".to_string(), + value: "large_chunked_metric".to_string(), + }], + hints: None, + }; + + let body = build_remote_read_request( + vec![query], + vec![read_request::ResponseType::StreamedXorChunks as i32], + )?; + + let (status, body_bytes) = send_remote_read_request(&app, body).await?; + assert_eq!(status, StatusCode::OK); + assert!( + !body_bytes.is_empty(), + "Large streamed response should have body" + ); + + let responses = parse_streamed_chunked_response(&body_bytes)?; + assert_eq!(responses.len(), 1); + assert_eq!(responses[0].chunked_series.len(), 1); + assert_eq!(responses[0].chunked_series[0].chunks.len(), 2); + + Ok(()) +} + +/// Test remote read uses common Prometheus hints to return aggregated samples. +#[tokio::test] +#[serial] +async fn test_remote_read_samples_with_avg_over_time_hints() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + + let sensor = create_sensor_with_labels("hinted_metric", SensorType::Float, vec![]); + let times_ms = vec![ + BASE_TS_MS, + BASE_TS_MS + 1000, + BASE_TS_MS + 2000, + BASE_TS_MS + 3000, + BASE_TS_MS + 4000, + ]; + let values = vec![1.0, 2.0, 3.0, 4.0, 5.0]; + + publish_test_sensors( + &storage, + vec![(sensor, create_float_samples_at_times(×_ms, &values))], + ) + .await?; + + let app = create_test_app(storage); + let query = Query { + start_timestamp_ms: BASE_TS_MS, + end_timestamp_ms: BASE_TS_MS + 5000, + matchers: vec![PromLabelMatcher { + r#type: label_matcher::Type::Eq as i32, + name: "__name__".to_string(), + value: "hinted_metric".to_string(), + }], + hints: Some(ReadHints { + step_ms: 2000, + func: "avg_over_time".to_string(), + start_ms: BASE_TS_MS, + end_ms: BASE_TS_MS + 5000, + grouping: vec![], + by: false, + range_ms: 2000, + }), + }; + + let body = build_remote_read_request(vec![query], vec![])?; + let (status, response_body) = send_remote_read_request(&app, body).await?; + + assert_eq!(status, StatusCode::OK); + + let response = parse_remote_read_response(&response_body)?; + let timeseries = &response.results[0].timeseries[0]; + assert_eq!(timeseries.samples.len(), 3); + assert_eq!(timeseries.samples[0].timestamp, BASE_TS_MS); + assert_eq!(timeseries.samples[1].timestamp, BASE_TS_MS + 2000); + assert_eq!(timeseries.samples[2].timestamp, BASE_TS_MS + 4000); + assert!((timeseries.samples[0].value - 1.5).abs() < f64::EPSILON); + assert!((timeseries.samples[1].value - 3.5).abs() < f64::EPSILON); + assert!((timeseries.samples[2].value - 5.0).abs() < f64::EPSILON); + + Ok(()) +} + /// Test that default response type is SAMPLES when no type specified #[tokio::test] #[serial] diff --git a/tests/prometheus_write_integration.rs b/tests/prometheus_write_integration.rs new file mode 100644 index 00000000..8746f4e1 --- /dev/null +++ b/tests/prometheus_write_integration.rs @@ -0,0 +1,346 @@ +mod common; + +use anyhow::Result; +use axum::Router; +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use axum::routing::post; +use common::TestDb; +use common::db::DbHelpers; +use prost::Message; +use sensapp::config::load_configuration_for_tests; +use sensapp::http::metrics::HttpMetrics; +use sensapp::http::prometheus_write::publish_prometheus; +use sensapp::http::state::HttpServerState; +use sensapp::parsing::prometheus::remote_write_models::{ + Label, Sample as PromSample, TimeSeries, WriteRequest, +}; +use serial_test::serial; +use std::sync::Arc; +use tower::ServiceExt; + +static INIT: std::sync::Once = std::sync::Once::new(); +fn ensure_config() { + INIT.call_once(|| { + load_configuration_for_tests().expect("Failed to load configuration for tests"); + }); +} + +/// Create a test app with the Prometheus remote write endpoint +fn create_test_app(storage: Arc) -> Router { + let state = HttpServerState { + name: Arc::new("SensApp Test".to_string()), + storage, + metrics: Arc::new(HttpMetrics::new()), + influxdb_with_numeric: false, + auth: None, + }; + + Router::new() + .route("/api/v1/prometheus_remote_write", post(publish_prometheus)) + .with_state(state) +} + +/// Build a Prometheus remote write request body (snappy-compressed protobuf) +fn build_write_request(timeseries: Vec) -> Result> { + let request = WriteRequest { timeseries }; + let encoded = request.encode_to_vec(); + let compressed = snap::raw::Encoder::new().compress_vec(&encoded)?; + Ok(compressed) +} + +/// Send a remote write request +async fn send_write_request(app: &Router, body: Vec) -> Result { + let request = Request::builder() + .method("POST") + .uri("/api/v1/prometheus_remote_write") + .header("content-type", "application/x-protobuf") + .header("content-encoding", "snappy") + .header("x-prometheus-remote-write-version", "0.1.0") + .body(Body::from(body))?; + + let response = app.clone().oneshot(request).await?; + Ok(response.status()) +} + +/// Test basic Prometheus remote write with a single time series +#[tokio::test] +#[serial] +async fn test_prometheus_write_basic() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = create_test_app(storage.clone()); + + let ts = TimeSeries { + labels: vec![ + Label { + name: "__name__".to_string(), + value: "cpu_usage".to_string(), + }, + Label { + name: "host".to_string(), + value: "server1".to_string(), + }, + ], + samples: vec![ + PromSample { + value: 75.5, + timestamp: 1704067200000, // Jan 1, 2024 in ms + }, + PromSample { + value: 80.2, + timestamp: 1704067260000, + }, + PromSample { + value: 65.0, + timestamp: 1704067320000, + }, + ], + }; + + let body = build_write_request(vec![ts])?; + let status = send_write_request(&app, body).await?; + + assert_eq!(status, StatusCode::NO_CONTENT); + + // Verify data was stored + let sensor = DbHelpers::get_sensor_by_name(&storage, "cpu_usage").await?; + assert!(sensor.is_some(), "cpu_usage sensor should exist"); + + let sensor_data = DbHelpers::verify_sensor_data(&storage, "cpu_usage", 3).await?; + + if let sensapp::datamodel::TypedSamples::Float(samples) = &sensor_data.samples { + assert_eq!(samples[0].value, 75.5); + assert_eq!(samples[1].value, 80.2); + assert_eq!(samples[2].value, 65.0); + } else { + panic!("Expected float samples for Prometheus data"); + } + + Ok(()) +} + +/// Test Prometheus remote write with multiple time series +#[tokio::test] +#[serial] +async fn test_prometheus_write_multiple_series() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = create_test_app(storage.clone()); + + let ts1 = TimeSeries { + labels: vec![Label { + name: "__name__".to_string(), + value: "up".to_string(), + }], + samples: vec![PromSample { + value: 1.0, + timestamp: 1704067200000, + }], + }; + + let ts2 = TimeSeries { + labels: vec![Label { + name: "__name__".to_string(), + value: "process_cpu_seconds_total".to_string(), + }], + samples: vec![PromSample { + value: 42.5, + timestamp: 1704067200000, + }], + }; + + let body = build_write_request(vec![ts1, ts2])?; + let status = send_write_request(&app, body).await?; + + assert_eq!(status, StatusCode::NO_CONTENT); + + DbHelpers::verify_sensor_data(&storage, "up", 1).await?; + DbHelpers::verify_sensor_data(&storage, "process_cpu_seconds_total", 1).await?; + + Ok(()) +} + +/// Test Prometheus remote write with empty timeseries (metadata request) +#[tokio::test] +#[serial] +async fn test_prometheus_write_empty_timeseries() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = create_test_app(storage.clone()); + + // Prometheus regularly sends empty timeseries for metadata + let body = build_write_request(vec![])?; + let status = send_write_request(&app, body).await?; + + assert_eq!(status, StatusCode::NO_CONTENT); + + Ok(()) +} + +/// Test Prometheus remote write rejects missing __name__ label +#[tokio::test] +#[serial] +async fn test_prometheus_write_missing_name_label() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = create_test_app(storage.clone()); + + let ts = TimeSeries { + labels: vec![Label { + name: "host".to_string(), + value: "server1".to_string(), + }], + samples: vec![PromSample { + value: 42.0, + timestamp: 1704067200000, + }], + }; + + let body = build_write_request(vec![ts])?; + let status = send_write_request(&app, body).await?; + + assert_eq!(status, StatusCode::BAD_REQUEST); + + Ok(()) +} + +/// Test Prometheus write rejects wrong content-encoding header +#[tokio::test] +#[serial] +async fn test_prometheus_write_wrong_content_encoding() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = create_test_app(storage.clone()); + + let request = Request::builder() + .method("POST") + .uri("/api/v1/prometheus_remote_write") + .header("content-type", "application/x-protobuf") + .header("content-encoding", "gzip") // Wrong! Should be snappy + .header("x-prometheus-remote-write-version", "0.1.0") + .body(Body::from(vec![0u8; 10]))?; + + let response = app.clone().oneshot(request).await?; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + + Ok(()) +} + +/// Test Prometheus write rejects missing content-type header +#[tokio::test] +#[serial] +async fn test_prometheus_write_missing_content_type() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = create_test_app(storage.clone()); + + let request = Request::builder() + .method("POST") + .uri("/api/v1/prometheus_remote_write") + .header("content-encoding", "snappy") + .header("x-prometheus-remote-write-version", "0.1.0") + .body(Body::from(vec![0u8; 10]))?; + + let response = app.clone().oneshot(request).await?; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + + Ok(()) +} + +/// Test Prometheus write rejects missing version header +#[tokio::test] +#[serial] +async fn test_prometheus_write_missing_version_header() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = create_test_app(storage.clone()); + + let request = Request::builder() + .method("POST") + .uri("/api/v1/prometheus_remote_write") + .header("content-type", "application/x-protobuf") + .header("content-encoding", "snappy") + // Missing x-prometheus-remote-write-version + .body(Body::from(vec![0u8; 10]))?; + + let response = app.clone().oneshot(request).await?; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + + Ok(()) +} + +/// Test Prometheus write stores labels correctly +#[tokio::test] +#[serial] +async fn test_prometheus_write_preserves_labels() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = create_test_app(storage.clone()); + + let ts = TimeSeries { + labels: vec![ + Label { + name: "__name__".to_string(), + value: "http_requests_total".to_string(), + }, + Label { + name: "method".to_string(), + value: "GET".to_string(), + }, + Label { + name: "status".to_string(), + value: "200".to_string(), + }, + ], + samples: vec![PromSample { + value: 1234.0, + timestamp: 1704067200000, + }], + }; + + let body = build_write_request(vec![ts])?; + let status = send_write_request(&app, body).await?; + assert_eq!(status, StatusCode::NO_CONTENT); + + let sensor = DbHelpers::get_sensor_by_name(&storage, "http_requests_total") + .await? + .expect("Sensor should exist"); + + let label_names: Vec<&str> = sensor.labels.iter().map(|(n, _)| n.as_str()).collect(); + assert!( + label_names.contains(&"method"), + "Should have 'method' label, got: {:?}", + label_names + ); + assert!( + label_names.contains(&"status"), + "Should have 'status' label, got: {:?}", + label_names + ); + + // Check label values + let method_value = sensor + .labels + .iter() + .find(|(n, _)| n == "method") + .map(|(_, v)| v.as_str()); + assert_eq!(method_value, Some("GET")); + + let status_value = sensor + .labels + .iter() + .find(|(n, _)| n == "status") + .map(|(_, v)| v.as_str()); + assert_eq!(status_value, Some("200")); + + Ok(()) +} diff --git a/tests/query_export.rs b/tests/query_export.rs index 0c8eefc0..ff3994a1 100644 --- a/tests/query_export.rs +++ b/tests/query_export.rs @@ -83,6 +83,121 @@ mod query_tests { Ok(()) } + #[tokio::test] + #[serial] + async fn test_query_last_sample_endpoint_returns_latest_value() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + let (csv_data, sensor_name) = fixtures::temperature_sensor_csv_with_name(); + app.post_csv("/sensors/publish", &csv_data).await?; + + let sensor = DbHelpers::get_sensor_by_name(&storage, &sensor_name) + .await? + .expect("Temperature sensor should exist"); + + let response = app.get(&format!("/series/{}/last", sensor.uuid)).await?; + response.assert_status(StatusCode::OK); + + let payload: serde_json::Value = response.json()?; + assert_eq!(payload["series_uuid"], sensor.uuid.to_string()); + assert_eq!(payload["sensor_name"], sensor_name); + assert_eq!(payload["sensor_type"], "float"); + assert_eq!(payload["value"], 20.8); + assert!( + payload["timestamp"] + .as_str() + .expect("timestamp should be a string") + .contains("2024-01-01T00:04:00") + ); + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_query_last_sample_endpoint_honors_time_window() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + let (csv_data, sensor_name) = fixtures::temperature_sensor_csv_with_name(); + app.post_csv("/sensors/publish", &csv_data).await?; + + let sensor = DbHelpers::get_sensor_by_name(&storage, &sensor_name) + .await? + .expect("Temperature sensor should exist"); + + let response = app + .get(&format!( + "/series/{}/last?start=2024-01-01T00:00:00Z&end=2024-01-01T00:03:30Z", + sensor.uuid + )) + .await?; + response.assert_status(StatusCode::OK); + + let payload: serde_json::Value = response.json()?; + assert_eq!(payload["value"], 22.0); + assert!( + payload["timestamp"] + .as_str() + .expect("timestamp should be a string") + .contains("2024-01-01T00:03:00") + ); + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_series_availability_endpoint_reports_presence_and_coverage() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + let (csv_data, sensor_name) = fixtures::temperature_sensor_csv_with_name(); + app.post_csv("/sensors/publish", &csv_data).await?; + + let sensor = DbHelpers::get_sensor_by_name(&storage, &sensor_name) + .await? + .expect("Temperature sensor should exist"); + + let response = app + .get(&format!( + "/series/{}/availability?start=2024-01-01T00:00:00Z&end=2024-01-01T00:04:00Z&step=2m", + sensor.uuid + )) + .await?; + response.assert_status(StatusCode::OK); + + let payload: serde_json::Value = response.json()?; + assert_eq!(payload["series_uuid"], sensor.uuid.to_string()); + assert_eq!(payload["sensor_name"], sensor_name); + assert_eq!(payload["present"], true); + assert_eq!(payload["sample_count"], 5); + assert_eq!(payload["covered_buckets"], 3); + assert_eq!(payload["total_buckets"], 3); + assert_eq!(payload["coverage_ratio"], 1.0); + assert!( + payload["first_sample_at"] + .as_str() + .expect("first_sample_at should be a string") + .contains("2024-01-01T00:00:00") + ); + assert!( + payload["last_sample_at"] + .as_str() + .expect("last_sample_at should be a string") + .contains("2024-01-01T00:04:00") + ); + + Ok(()) + } + #[tokio::test] #[serial] async fn test_query_nonexistent_sensor() -> Result<()> { @@ -592,3 +707,120 @@ mod time_query_tests { Ok(()) } } + +mod advanced_series_query_tests { + use super::*; + + #[tokio::test] + #[serial] + async fn test_series_query_supports_bucketed_average() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + let (csv_data, sensor_name) = fixtures::temperature_sensor_csv_with_name(); + app.post_csv("/sensors/publish", &csv_data).await?; + + let sensor = DbHelpers::get_sensor_by_name(&storage, &sensor_name) + .await? + .expect("sensor should exist"); + + let response = app + .get(&format!( + "/series/{}?format=csv&step=2m&aggregation=avg", + sensor.uuid + )) + .await?; + + response.assert_status(StatusCode::OK); + + let lines: Vec<&str> = response.body().trim().lines().collect(); + assert_eq!(lines.len(), 4, "expected header + 3 aggregated rows"); + assert_eq!(lines[1], "2024-01-01T00:00:00+00:00,20.75"); + assert_eq!(lines[2], "2024-01-01T00:02:00+00:00,21.75"); + assert_eq!(lines[3], "2024-01-01T00:04:00+00:00,20.8"); + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_series_query_requires_explicit_simplify_enablement() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + let (csv_data, sensor_name) = fixtures::temperature_sensor_csv_with_name(); + app.post_csv("/sensors/publish", &csv_data).await?; + + let sensor = DbHelpers::get_sensor_by_name(&storage, &sensor_name) + .await? + .expect("sensor should exist"); + + let response = app + .get(&format!( + "/series/{}?format=csv&simplify_tolerance=0.1", + sensor.uuid + )) + .await?; + + response.assert_status(StatusCode::BAD_REQUEST); + response.assert_body_contains("simplify=true"); + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_series_query_reduces_points_with_simplify() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + let app = TestApp::new(storage.clone()).await; + + let sensor_name = format!("simplify_{}", uuid::Uuid::new_v4().simple()); + let mut csv_data = String::from("datetime,sensor_name,value,unit\n"); + for minute in 0..20u32 { + let value = 20.0 + (minute as f64 * 0.2) + if minute % 5 == 0 { 0.01 } else { 0.0 }; + csv_data.push_str(&format!( + "2024-01-01T00:{:02}:00Z,{},{}{},°C\n", + minute, sensor_name, value, "" + )); + } + app.post_csv("/sensors/publish", &csv_data).await?; + + let sensor = DbHelpers::get_sensor_by_name(&storage, &sensor_name) + .await? + .expect("sensor should exist"); + + let raw_response = app + .get(&format!("/series/{}?format=csv", sensor.uuid)) + .await?; + raw_response.assert_status(StatusCode::OK); + let raw_line_count = raw_response.body().trim().lines().count(); + + let simplified_response = app + .get(&format!( + "/series/{}?format=csv&simplify=true&simplify_tolerance=0.05", + sensor.uuid + )) + .await?; + simplified_response.assert_status(StatusCode::OK); + + let simplified_lines: Vec<&str> = simplified_response.body().trim().lines().collect(); + assert!(simplified_lines.len() < raw_line_count); + assert_eq!(simplified_lines.first().copied(), Some("timestamp,value")); + assert!(simplified_lines[1].starts_with("2024-01-01T00:00:00+00:00,")); + assert!( + simplified_lines + .last() + .copied() + .unwrap_or_default() + .starts_with("2024-01-01T00:19:00+00:00,") + ); + + Ok(()) + } +} diff --git a/tests/rrdcached_integration.rs b/tests/rrdcached_integration.rs new file mode 100644 index 00000000..51c68ce2 --- /dev/null +++ b/tests/rrdcached_integration.rs @@ -0,0 +1,621 @@ +mod common; + +#[cfg(feature = "rrdcached")] +mod rrdcached_tests { + use crate::common::{DatabaseType, TestDb}; + use anyhow::Result; + use sensapp::config::load_configuration_for_tests; + use sensapp::datamodel::{ + Sample, SensAppDateTime, Sensor, SensorType, TypedSamples, + batch::{Batch, SingleSensorBatch}, + sensapp_vec::SensAppVec, + }; + use serial_test::serial; + use smallvec::SmallVec; + use std::sync::Arc; + use uuid::Uuid; + + // Ensure configuration is loaded once for all tests in this module + static INIT: std::sync::Once = std::sync::Once::new(); + fn ensure_config() { + INIT.call_once(|| { + load_configuration_for_tests().expect("Failed to load configuration for tests"); + }); + } + + /// Test basic RRDcached connection and database setup + #[tokio::test] + #[serial] + async fn test_rrdcached_connection() -> Result<()> { + ensure_config(); + // Given: An RRDcached test database + let test_db = TestDb::new_with_type(DatabaseType::RRDcached).await?; + let storage = test_db.storage(); + + // When: We try to migrate and list series + storage.create_or_migrate().await?; + + // Then: The operations should succeed (database is accessible) + let sensors = storage.list_series(None, None, None).await?.series; + + // Database should be empty initially (or may contain existing RRD files) + println!("Found {} sensors in RRDcached database", sensors.len()); + for sensor in &sensors { + println!(" - Sensor: UUID={}, Name={}", sensor.uuid, sensor.name); + } + + Ok(()) + } + + /// Test RRDcached list functionality with data + #[tokio::test] + #[serial] + async fn test_rrdcached_list_after_publishing() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::RRDcached).await?; + let storage = test_db.storage(); + + // Given: We publish some data first + let sensor = Sensor { + uuid: Uuid::new_v4(), + name: "test_sensor_for_listing".to_string(), + sensor_type: SensorType::Float, + unit: None, + labels: SmallVec::new(), + }; + + let base_time = 1704067200.0; // 2024-01-01 00:00:00 UTC + let samples = vec![Sample { + datetime: SensAppDateTime::from_unix_seconds(base_time), + value: 42.0, + }]; + + // Create and publish batch + let sensor_arc = Arc::new(sensor); + let single_sensor_batch = + SingleSensorBatch::new(sensor_arc.clone(), TypedSamples::Float(samples.into())); + let mut sensors_vec = SensAppVec::new(); + sensors_vec.push(single_sensor_batch); + let batch = Arc::new(Batch::new(sensors_vec)); + + storage.publish(batch).await?; + + // When: We list the series + let sensors = storage.list_series(None, None, None).await?.series; + + // Then: We should find the sensor we just created + println!("Found {} sensors after publishing data", sensors.len()); + for sensor in &sensors { + println!(" - Sensor: UUID={}, Name={}", sensor.uuid, sensor.name); + } + + // Verify our sensor appears in the list (either by UUID match or by being present) + let found = sensors.iter().any(|s| s.uuid == sensor_arc.uuid); + if !found && sensors.is_empty() { + println!( + "Warning: No sensors found - this might be expected if RRDcached LIST command doesn't show recently created files" + ); + } + + Ok(()) + } + + /// Test metrics listing functionality + #[tokio::test] + #[serial] + async fn test_rrdcached_list_metrics() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::RRDcached).await?; + let storage = test_db.storage(); + + // When: We list metrics + let metrics = storage.list_metrics().await?; + + // Then: The operation should succeed + // RRDcached doesn't support metrics the same way as SQL databases + println!("Found {} metrics in RRDcached database", metrics.len()); + + Ok(()) + } + + /// Test basic data publishing to RRDcached + #[tokio::test] + #[serial] + async fn test_rrdcached_publish_float_data() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::RRDcached).await?; + let storage = test_db.storage(); + + // Given: A test sensor with float data + let sensor_uuid = Uuid::new_v4(); + let sensor = Sensor { + uuid: sensor_uuid, + name: "test_float_sensor".to_string(), + sensor_type: SensorType::Float, + unit: None, + labels: SmallVec::new(), + }; + + // Create some test float samples + let base_time = 1704067200.0; // 2024-01-01 00:00:00 UTC + let samples = vec![ + Sample { + datetime: SensAppDateTime::from_unix_seconds(base_time), + value: 23.5, + }, + Sample { + datetime: SensAppDateTime::from_unix_seconds(base_time + 1.0), + value: 24.1, + }, + Sample { + datetime: SensAppDateTime::from_unix_seconds(base_time + 2.0), + value: 22.8, + }, + ]; + + // Build batch manually + let sensor_arc = Arc::new(sensor); + let single_sensor_batch = + SingleSensorBatch::new(sensor_arc, TypedSamples::Float(samples.into())); + let mut sensors = SensAppVec::new(); + sensors.push(single_sensor_batch); + let batch = Arc::new(Batch::new(sensors)); + + // When: We publish the batch + storage.publish(batch).await?; + + // Then: The data should be stored + // Note: RRDcached doesn't support querying data back easily, + // but we can at least verify the publish operation succeeded + println!("Successfully published float data to RRDcached"); + + // Verify the sensor appears in our created sensors list + let sensors = storage.list_series(None, None, None).await?.series; + assert!(!sensors.is_empty(), "Should have at least one sensor"); + + let found_sensor = sensors.iter().find(|s| s.uuid == sensor_uuid); + assert!( + found_sensor.is_some(), + "Published sensor should be in the list" + ); + + Ok(()) + } + + /// Test publishing different data types + #[tokio::test] + #[serial] + async fn test_rrdcached_publish_multiple_types() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::RRDcached).await?; + let storage = test_db.storage(); + + let base_time = 1704067200.0; // 2024-01-01 00:00:00 UTC + // Test Integer data + let int_sensor = Sensor { + uuid: Uuid::new_v4(), + name: "test_integer_sensor".to_string(), + sensor_type: SensorType::Integer, + unit: None, + labels: SmallVec::new(), + }; + + let int_samples = vec![Sample { + datetime: SensAppDateTime::from_unix_seconds(base_time), + value: 42i64, + }]; + + let int_sensor_arc = Arc::new(int_sensor); + let single_sensor_batch = SingleSensorBatch::new( + int_sensor_arc.clone(), + TypedSamples::Integer(int_samples.into()), + ); + let mut sensors = SensAppVec::new(); + sensors.push(single_sensor_batch); + let batch = Arc::new(Batch::new(sensors)); + + storage.publish(batch).await?; + println!("Successfully published integer data to RRDcached"); + + // Test Boolean data + let bool_sensor = Sensor { + uuid: Uuid::new_v4(), + name: "test_boolean_sensor".to_string(), + sensor_type: SensorType::Boolean, + unit: None, + labels: SmallVec::new(), + }; + + let bool_samples = vec![Sample { + datetime: SensAppDateTime::from_unix_seconds(base_time + 5.0), // Ensure different timestamp + value: true, + }]; + + let bool_sensor_arc = Arc::new(bool_sensor); + let single_sensor_batch = SingleSensorBatch::new( + bool_sensor_arc.clone(), + TypedSamples::Boolean(bool_samples.into()), + ); + let mut sensors = SensAppVec::new(); + sensors.push(single_sensor_batch); + let batch = Arc::new(Batch::new(sensors)); + + storage.publish(batch).await?; + println!("Successfully published boolean data to RRDcached"); + + // Verify all sensors are tracked + let sensors = storage.list_series(None, None, None).await?.series; + assert!( + sensors.len() >= 2, + "Should have at least 2 sensors (integer and boolean)" + ); + + Ok(()) + } + + /// Test RRDcached vacuum operation + #[tokio::test] + #[serial] + async fn test_rrdcached_vacuum() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::RRDcached).await?; + let storage = test_db.storage(); + + // When: We call vacuum + storage.vacuum().await?; + + // Then: The operation should succeed (even though it's a no-op for RRDcached) + println!("RRDcached vacuum completed successfully"); + + Ok(()) + } + + /// Test that unsupported sensor types are handled gracefully + #[tokio::test] + #[serial] + async fn test_rrdcached_unsupported_types() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::RRDcached).await?; + let storage = test_db.storage(); + + // Given: A sensor with an unsupported type (String) + let string_sensor = Sensor { + uuid: Uuid::new_v4(), + name: "test_string_sensor".to_string(), + sensor_type: SensorType::String, + unit: None, + labels: SmallVec::new(), + }; + + let string_samples = vec![Sample { + datetime: SensAppDateTime::from_unix_seconds(1704067200.0), + value: "test_value".to_string(), + }]; + + let string_sensor_arc = Arc::new(string_sensor); + let single_sensor_batch = SingleSensorBatch::new( + string_sensor_arc.clone(), + TypedSamples::String(string_samples.into()), + ); + let mut sensors = SensAppVec::new(); + sensors.push(single_sensor_batch); + let batch = Arc::new(Batch::new(sensors)); + + // When: We try to publish unsupported data + storage.publish(batch).await?; + + // Then: The operation should succeed but the sensor shouldn't be created + // (since unsupported types are filtered out in the create_sensors method) + let sensors = storage.list_series(None, None, None).await?.series; + let found_sensor = sensors.iter().find(|s| s.uuid == string_sensor_arc.uuid); + assert!( + found_sensor.is_none(), + "Unsupported sensor types should not be created" + ); + + Ok(()) + } + + /// Test querying data that was previously published + #[tokio::test] + #[serial] + async fn test_rrdcached_query_published_data() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::RRDcached).await?; + let storage = test_db.storage(); + + // Given: A sensor with known data + let sensor_uuid = Uuid::new_v4(); + let sensor = Sensor { + uuid: sensor_uuid, + name: "test_query_sensor".to_string(), + sensor_type: SensorType::Float, + unit: None, + labels: SmallVec::new(), + }; + + let base_time = 1704067200.0; // 2024-01-01 00:00:00 UTC + let test_values = [23.5, 24.1, 22.8]; + let samples: Vec> = test_values + .iter() + .enumerate() + .map(|(i, &value)| Sample { + datetime: SensAppDateTime::from_unix_seconds(base_time + i as f64), + value, + }) + .collect(); + + // Publish the data + let sensor_arc = Arc::new(sensor); + let single_sensor_batch = + SingleSensorBatch::new(sensor_arc.clone(), TypedSamples::Float(samples.into())); + let mut sensors = SensAppVec::new(); + sensors.push(single_sensor_batch); + let batch = Arc::new(Batch::new(sensors)); + + storage.publish(batch).await?; + + // Give RRD time to process and consolidate the data + tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await; + + // When: We query the data back with the specific time range + let start_time = Some(SensAppDateTime::from_unix_seconds(base_time - 10.0)); // Start before our data + let end_time = Some(SensAppDateTime::from_unix_seconds(base_time + 10.0)); // End after our data + let sensor_data = storage + .query_sensor_data(&sensor_uuid.to_string(), start_time, end_time, None) + .await?; + + // Then: We should get back the data we stored + assert!(sensor_data.is_some(), "Query should return data"); + let sensor_data = sensor_data.unwrap(); + + // Verify sensor metadata (reconstructed from UUID) + assert_eq!(sensor_data.sensor.uuid, sensor_uuid); + assert_eq!(sensor_data.sensor.sensor_type, SensorType::Float); + + // Verify samples (may not be exact due to RRD time alignment and consolidation) + if let TypedSamples::Float(returned_samples) = sensor_data.samples { + assert!( + !returned_samples.is_empty(), + "Should have returned some samples" + ); + + // We may not get exact values due to RRD consolidation, but let's check we get reasonable data + for sample in &returned_samples { + assert!(!sample.value.is_nan(), "Sample values should not be NaN"); + println!( + "Retrieved sample: time={:?}, value={}", + sample.datetime, sample.value + ); + } + } else { + panic!("Expected Float samples, got: {:?}", sensor_data.samples); + } + + Ok(()) + } + + /// Test querying with time range + #[tokio::test] + #[serial] + async fn test_rrdcached_query_with_time_range() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::RRDcached).await?; + let storage = test_db.storage(); + + // Given: A sensor with data spread over time + let sensor_uuid = Uuid::new_v4(); + let sensor = Sensor { + uuid: sensor_uuid, + name: "test_time_range_sensor".to_string(), + sensor_type: SensorType::Float, + unit: None, + labels: SmallVec::new(), + }; + + let base_time = 1704067200.0; // 2024-01-01 00:00:00 UTC + let samples: Vec> = (0..10) + .map(|i| Sample { + datetime: SensAppDateTime::from_unix_seconds(base_time + i as f64 * 60.0), // Every minute + value: i as f64 * 10.0, + }) + .collect(); + + // Publish the data + let sensor_arc = Arc::new(sensor); + let single_sensor_batch = + SingleSensorBatch::new(sensor_arc.clone(), TypedSamples::Float(samples.into())); + let mut sensors = SensAppVec::new(); + sensors.push(single_sensor_batch); + let batch = Arc::new(Batch::new(sensors)); + + storage.publish(batch).await?; + + // Give RRD time to process and consolidate the data + tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await; + + // When: We query with a time range (first 5 minutes with buffer) + let start_time = Some(SensAppDateTime::from_unix_seconds(base_time - 60.0)); // Start 1 minute before + let end_time = Some(SensAppDateTime::from_unix_seconds(base_time + 360.0)); // End 6 minutes after start + + let sensor_data = storage + .query_sensor_data(&sensor_uuid.to_string(), start_time, end_time, None) + .await?; + + // Then: We should get data within the time range + assert!( + sensor_data.is_some(), + "Query with time range should return data" + ); + let sensor_data = sensor_data.unwrap(); + + if let TypedSamples::Float(returned_samples) = sensor_data.samples { + assert!( + !returned_samples.is_empty(), + "Should have returned samples in time range" + ); + + for sample in &returned_samples { + let sample_time = sample.datetime.to_unix_seconds(); + println!( + "Sample in range: time={}, value={}", + sample_time, sample.value + ); + // Note: RRD may return data slightly outside the range due to consolidation + } + } else { + panic!("Expected Float samples, got: {:?}", sensor_data.samples); + } + + Ok(()) + } + + /// Test querying non-existent sensor + #[tokio::test] + #[serial] + async fn test_rrdcached_query_nonexistent_sensor() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::RRDcached).await?; + let storage = test_db.storage(); + + // When: We query a sensor that doesn't exist + let non_existent_uuid = Uuid::new_v4(); + let sensor_data = storage + .query_sensor_data(&non_existent_uuid.to_string(), None, None, None) + .await?; + + // Then: We should get None + assert!( + sensor_data.is_none(), + "Query for non-existent sensor should return None" + ); + + Ok(()) + } + + /// Test querying sensor with no data yet + #[tokio::test] + #[serial] + async fn test_rrdcached_query_sensor_no_data() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::RRDcached).await?; + let storage = test_db.storage(); + + // Given: A sensor that exists but has no data + let sensor_uuid = Uuid::new_v4(); + let sensor = Sensor { + uuid: sensor_uuid, + name: "test_empty_sensor".to_string(), + sensor_type: SensorType::Float, + unit: None, + labels: SmallVec::new(), + }; + + // Create the sensor without publishing any data + let sensor_arc = Arc::new(sensor); + let empty_samples: Vec> = vec![]; + let single_sensor_batch = SingleSensorBatch::new( + sensor_arc.clone(), + TypedSamples::Float(empty_samples.into()), + ); + let mut sensors = SensAppVec::new(); + sensors.push(single_sensor_batch); + let batch = Arc::new(Batch::new(sensors)); + + storage.publish(batch).await?; + + // When: We query the sensor + let sensor_data = storage + .query_sensor_data(&sensor_uuid.to_string(), None, None, None) + .await?; + + // Then: We should get None (no data available) + assert!( + sensor_data.is_none(), + "Query for sensor with no data should return None" + ); + + Ok(()) + } + + /// Test querying integer data (stored as float in RRD) + #[tokio::test] + #[serial] + async fn test_rrdcached_query_integer_data() -> Result<()> { + ensure_config(); + let test_db = TestDb::new_with_type(DatabaseType::RRDcached).await?; + let storage = test_db.storage(); + + // Given: A sensor with integer data + let sensor_uuid = Uuid::new_v4(); + let sensor = Sensor { + uuid: sensor_uuid, + name: "test_integer_query_sensor".to_string(), + sensor_type: SensorType::Integer, + unit: None, + labels: SmallVec::new(), + }; + + let base_time = 1704067200.0; // 2024-01-01 00:00:00 UTC + let int_samples = vec![ + Sample { + datetime: SensAppDateTime::from_unix_seconds(base_time), + value: 42i64, + }, + Sample { + datetime: SensAppDateTime::from_unix_seconds(base_time + 1.0), + value: 84i64, + }, + ]; + + // Publish the integer data + let sensor_arc = Arc::new(sensor); + let single_sensor_batch = SingleSensorBatch::new( + sensor_arc.clone(), + TypedSamples::Integer(int_samples.into()), + ); + let mut sensors = SensAppVec::new(); + sensors.push(single_sensor_batch); + let batch = Arc::new(Batch::new(sensors)); + + storage.publish(batch).await?; + + // Give RRD time to process and consolidate the data + tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await; + + // When: We query the data back with time range + let start_time = Some(SensAppDateTime::from_unix_seconds(base_time - 10.0)); // Start before our data + let end_time = Some(SensAppDateTime::from_unix_seconds(base_time + 10.0)); // End after our data + let sensor_data = storage + .query_sensor_data(&sensor_uuid.to_string(), start_time, end_time, None) + .await?; + + // Then: We should get back Float data (since RRD stores everything as f64) + // Note: This shows a limitation - we lose the original type information + assert!(sensor_data.is_some(), "Query should return data"); + let sensor_data = sensor_data.unwrap(); + + // The sensor type will be reconstructed as Float since we don't store the original type + assert_eq!(sensor_data.sensor.sensor_type, SensorType::Float); + + if let TypedSamples::Float(returned_samples) = sensor_data.samples { + assert!( + !returned_samples.is_empty(), + "Should have returned some samples" + ); + + for sample in &returned_samples { + println!( + "Retrieved integer-as-float sample: time={:?}, value={}", + sample.datetime, sample.value + ); + // Values should be close to the original integers (42.0, 84.0) + assert!(!sample.value.is_nan(), "Sample values should not be NaN"); + } + } else { + panic!("Expected Float samples, got: {:?}", sensor_data.samples); + } + + Ok(()) + } +} diff --git a/tests/simple_promql.rs b/tests/simple_promql.rs index f3527e1b..fa83119f 100644 --- a/tests/simple_promql.rs +++ b/tests/simple_promql.rs @@ -6,6 +6,7 @@ mod common; use anyhow::Result; +use arrow_ipc::reader::StreamReader; use axum::Router; use axum::body::Body; use axum::http::{Request, StatusCode}; @@ -15,11 +16,13 @@ use sensapp::config::load_configuration_for_tests; use sensapp::datamodel::batch_builder::BatchBuilder; use sensapp::datamodel::sensapp_vec::SensAppLabels; use sensapp::datamodel::{Sample, Sensor, SensorType, TypedSamples}; -use sensapp::ingestors::http::simple_promql::simple_promql_query; -use sensapp::ingestors::http::state::HttpServerState; +use sensapp::http::metrics::HttpMetrics; +use sensapp::http::simple_promql::simple_promql_query; +use sensapp::http::state::HttpServerState; use sensapp::storage::StorageInstance; use serde_json::Value; use serial_test::serial; +use std::io::Cursor; use std::sync::Arc; use tower::ServiceExt; use uuid::Uuid; @@ -88,7 +91,9 @@ async fn create_test_app(storage: Arc) -> Router { let state = HttpServerState { name: Arc::new("SensApp Test".to_string()), storage, + metrics: Arc::new(HttpMetrics::new()), influxdb_with_numeric: false, + auth: None, }; Router::new() @@ -317,6 +322,67 @@ async fn test_simple_promql_reject_binary() -> Result<()> { Ok(()) } +/// Test binary-operation error includes guidance for hyphenated metric names +#[tokio::test] +#[serial] +async fn test_simple_promql_hyphenated_metric_name_hint() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + + let sensor = create_sensor_with_labels("demo-temperature", SensorType::Float, vec![]); + publish_test_sensors(&storage, vec![(sensor, create_float_samples(3))]).await?; + + let app = create_test_app(storage).await; + + let request = Request::builder() + .method("GET") + .uri("/api/v1/query?query=demo-temperature%5B5m%5D") + .body(Body::empty())?; + + let response = app.oneshot(request).await?; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + + let body = axum::body::to_bytes(response.into_body(), usize::MAX).await?; + let body_str = String::from_utf8_lossy(&body); + assert!(body_str.contains("demo-temperature")); + assert!(body_str.contains("__name__")); + + Ok(()) +} + +/// Test quoted-name workaround for hyphenated metric names +#[tokio::test] +#[serial] +async fn test_simple_promql_hyphenated_metric_name_via_name_matcher() -> Result<()> { + ensure_config(); + let test_db = TestDb::new().await?; + let storage = test_db.storage(); + + let sensor = create_sensor_with_labels("demo-temperature", SensorType::Float, vec![]); + publish_test_sensors(&storage, vec![(sensor, create_float_samples(3))]).await?; + + let app = create_test_app(storage).await; + + let request = Request::builder() + .method("GET") + .uri("/api/v1/query?query=%7B__name__%3D%22demo-temperature%22%7D%5B5m%5D") + .body(Body::empty())?; + + let response = app.oneshot(request).await?; + assert_eq!(response.status(), StatusCode::OK); + + let body = axum::body::to_bytes(response.into_body(), usize::MAX).await?; + let json: Value = serde_json::from_slice(&body)?; + let records = json.as_array().expect("SenML response should be an array"); + assert!( + !records.is_empty(), + "Should have records for the hyphenated sensor" + ); + + Ok(()) +} + /// Test query with regex matcher #[tokio::test] #[serial] @@ -589,10 +655,26 @@ async fn test_simple_promql_arrow_format() -> Result<()> { let body = axum::body::to_bytes(response.into_body(), usize::MAX).await?; - // Arrow files start with magic bytes "ARROW1" - assert!(body.len() > 6, "Arrow file should have content"); - // Arrow IPC file format magic number - assert_eq!(&body[0..6], b"ARROW1", "Should be valid Arrow IPC file"); + assert!(body.len() > 6, "Arrow stream should have content"); + + let mut reader = StreamReader::try_new(Cursor::new(body), None)?; + let first_batch = reader + .next() + .transpose()? + .expect("Arrow stream should contain at least one record batch"); + + assert!( + first_batch.num_rows() > 0, + "Arrow batch should contain rows" + ); + assert!( + first_batch + .schema() + .fields() + .iter() + .any(|field| field.name() == "timestamp"), + "Arrow batch should contain a timestamp column" + ); Ok(()) }