diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..6b325c773 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,79 @@ +# Dependabot configuration +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file +version: 2 + +updates: + # Gradle — main module + - package-ecosystem: "gradle" + directory: "/" + schedule: + interval: "weekly" + target-branch: "master" + open-pull-requests-limit: 10 + labels: + - "dependencies" + - "gradle" + groups: + gradle-minor-and-patch: + update-types: + - "minor" + - "patch" + + # Gradle — foundation module + - package-ecosystem: "gradle" + directory: "/foundation" + schedule: + interval: "weekly" + target-branch: "master" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "gradle" + groups: + gradle-minor-and-patch: + update-types: + - "minor" + - "patch" + + # Gradle — buildSrc (build logic / plugins) + - package-ecosystem: "gradle" + directory: "/buildSrc" + schedule: + interval: "weekly" + target-branch: "master" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "gradle" + groups: + gradle-minor-and-patch: + update-types: + - "minor" + - "patch" + + # Docker — base image in Dockerfile + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + target-branch: "master" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "docker" + + # GitHub Actions — workflow action versions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + target-branch: "master" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "github-actions" + groups: + actions-minor-and-patch: + update-types: + - "minor" + - "patch" diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index d7df5fb45..025f42764 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -8,17 +8,17 @@ jobs: publish-docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: ref: ${{ github.ref }} submodules: recursive - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'zulu' java-version: 21 - name: Setup gradle - uses: gradle/gradle-build-action@v2 + uses: gradle/gradle-build-action@v3 - name: Upload to Docker run: make jib env: @@ -28,22 +28,22 @@ jobs: publish-github: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: ref: ${{ github.ref }} submodules: recursive - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'zulu' java-version: 21 - name: Setup gradle - uses: gradle/gradle-build-action@v2 + uses: gradle/gradle-build-action@v3 - name: Build zip run: make distZip - name: Upload Release id: upload-release-asset - uses: svenstaro/upload-release-action@v1-release + uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} tag: ${{ github.ref }} diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml index 95052fb22..a83278303 100644 --- a/.github/workflows/static.yml +++ b/.github/workflows/static.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: asciidoctor-ghpages uses: manoelcampos/asciidoctor-ghpages-action@v2 env: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3311e3351..dcaa82885 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -23,22 +23,24 @@ jobs: unit-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 submodules: recursive - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'zulu' java-version: 21 - name: Install System Libs run: sudo apt-get install -y openssl libapr1 + - name: Setup gradle - uses: gradle/gradle-build-action@v2 + uses: gradle/gradle-build-action@v3 with: github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Check run: make test env: diff --git a/.gitignore b/.gitignore index 5bf3ce9c1..fa52543bc 100644 --- a/.gitignore +++ b/.gitignore @@ -20,9 +20,17 @@ env /test* /test/ Test*.kt +*_test.sh # http-client config http-client.env.json .DS_Store -mise.toml \ No newline at end of file +mise.toml +.claude/ + +# demo sandbox (local-only, regenerate with demo/response-signing/generate-keys.sh + protoc) +/demo/response-signing/ + +# superpowers scratch: specs and plans live locally only for now +/docs/superpowers/ diff --git a/.sdkmanrc b/.sdkmanrc new file mode 100644 index 000000000..1c4d66691 --- /dev/null +++ b/.sdkmanrc @@ -0,0 +1,2 @@ +gradle=9.2.0 +java=21.0.11.fx-librca \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index ec4742053..da9146281 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,3 @@ -FROM eclipse-temurin:21 +FROM eclipse-temurin:25 RUN apt-get update -y && apt-get install -y libcurl4-openssl-dev libcjson-dev diff --git a/Makefile b/Makefile index c0d84241c..bc1bac9bb 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,9 @@ all: build-foundation build-main build-foundation: cd foundation && ../gradlew build publishToMavenLocal +run-main: + ./gradlew run + build-main: ./gradlew build diff --git a/build.gradle b/build.gradle index 85c025873..1d728f365 100644 --- a/build.gradle +++ b/build.gradle @@ -10,7 +10,7 @@ buildscript { gradlePluginPortal() } dependencies { - classpath 'com.google.protobuf:protobuf-gradle-plugin:0.9.1' + classpath 'com.google.protobuf:protobuf-gradle-plugin:0.10.0' } } @@ -71,20 +71,18 @@ dependencies { implementation(libs.reactor.grpc.stub) implementation(libs.grpc.proto.util) + implementation libs.prom.exporter.server implementation libs.micrometer.registry.prometheus + implementation libs.micrometer.ctx.prop implementation libs.lettuce.core - implementation(libs.spring.cloud.starter.sleuth) { - exclude module: 'spring-security-rsa' - } implementation libs.brave.instrumentation.grpc + implementation libs.brave.ctx.slf4j implementation libs.logstash.encoder implementation libs.janino - implementation libs.spring.cloud.sleuth.zipkin - implementation libs.bitcoinj implementation libs.snake.yaml @@ -97,6 +95,7 @@ dependencies { implementation libs.javax.annotations implementation libs.auth0.jwt + implementation libs.semver4j testImplementation libs.cglib.nodep testImplementation libs.spockframework.core @@ -184,7 +183,7 @@ jib { } container { creationTime = 'USE_CURRENT_TIMESTAMP' - jvmFlags = ['-XX:+UseG1GC', '-XX:+ExitOnOutOfMemoryError', '-Xms1024M', '-XX:NativeMemoryTracking=summary', '-XX:+UnlockDiagnosticVMOptions', '-XX:GCLockerRetryAllocationCount=10', '--enable-preview'] + jvmFlags = ['-XX:+UseG1GC', '-XX:+ExitOnOutOfMemoryError', '-Xms1024M', '-XX:NativeMemoryTracking=summary', '-XX:+UnlockDiagnosticVMOptions', '--enable-preview'] mainClass = 'io.emeraldpay.dshackle.StarterKt' args = [] ports = ['2448', '2449', '8545'] diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 7f4908de4..a9b04aa8a 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -8,7 +8,7 @@ repositories { } dependencies { - implementation("org.yaml:snakeyaml:1.24") + implementation("org.yaml:snakeyaml:2.6") implementation("dshackle:foundation:1.0.0") - implementation("com.squareup:kotlinpoet:1.14.2") + implementation("com.squareup:kotlinpoet:2.3.0") } diff --git a/buildSrc/src/main/kotlin/chainsconfig.codegen.gradle.kts b/buildSrc/src/main/kotlin/chainsconfig.codegen.gradle.kts index 445d89cca..6c0c9a52d 100644 --- a/buildSrc/src/main/kotlin/chainsconfig.codegen.gradle.kts +++ b/buildSrc/src/main/kotlin/chainsconfig.codegen.gradle.kts @@ -128,6 +128,7 @@ open class CodeGen(private val config: ChainsConfig) { private fun type(type: String): String { return when(type) { + "aztec" -> "BlockchainType.AZTEC" "eth" -> "BlockchainType.ETHEREUM" "bitcoin" -> "BlockchainType.BITCOIN" "starknet" -> "BlockchainType.STARKNET" @@ -139,6 +140,9 @@ open class CodeGen(private val config: ChainsConfig) { "cosmos" -> "BlockchainType.COSMOS" "ripple" -> "BlockchainType.RIPPLE" "kadena" -> "BlockchainType.KADENA" + "avm" -> "BlockchainType.AVM" + "app" -> "BlockchainType.ETHEREUM" + "aptos" -> "BlockchainType.ETHEREUM" else -> throw IllegalArgumentException("unknown blockchain type $type") } } diff --git a/docs/04-upstream-config.adoc b/docs/04-upstream-config.adoc index 053cff6dc..7caa6f746 100644 --- a/docs/04-upstream-config.adoc +++ b/docs/04-upstream-config.adoc @@ -174,6 +174,41 @@ In case of rpc and ws connection we can specify different modes of works togethe You can specify this modes through `connector-mode` parameter in connection config. +=== Custom Headers + +Dshackle allows you to add custom HTTP headers to all requests sent to an upstream. +This is useful when connecting to providers that require API keys, authentication tokens, or other custom headers. + +Custom headers are configured at the upstream level and will be added to: + +- All HTTP JSON RPC requests +- WebSocket connection handshakes (initial HTTP upgrade request) + +NOTE: Custom headers are supported for all connection types except gRPC connections. + +==== Configuration + +Custom headers are specified using the `custom-headers` parameter in your upstream configuration: + +[source,yaml] +---- +version: v1 + +upstreams: + - id: my-ethereum-node + chain: ethereum + custom-headers: + X-API-Key: "your-api-key-here" + X-Custom-Header: "custom-value" + Authorization: "Bearer your-token" + connection: + generic: + rpc: + url: "https://api.example.com/rpc" + ws: + url: "wss://api.example.com/ws" +---- + === Bitcoin Methods .By default an ethereum upstream allows call to the following JSON RPC methods: diff --git a/docs/reference-configuration.adoc b/docs/reference-configuration.adoc index b3a40cc7b..f6a35348e 100644 --- a/docs/reference-configuration.adoc +++ b/docs/reference-configuration.adoc @@ -47,10 +47,8 @@ cache: db: 0 password: I1y0dGKy01by -signed-response: - enabled: true - algorithm: SECP256K1 - private-key: /path/key.pem +auth: + enabled: false proxy: host: 0.0.0.0 @@ -207,10 +205,10 @@ See <> section | Caching configuration. See <> section. -| `signed-response` +| `auth` | -| Signed responses -See <> section. +| Authorization and response signing. +See <> section. | `cluster` | @@ -559,37 +557,66 @@ cache: |=== -[#signed-response] -== Signed Response +[#auth] +== Authorization + +dshackle supports optional client authentication via signed JWT tokens (RS256). When +`auth.enabled` is `true`, dshackle validates tokens issued by a trusted provider and +rejects unauthenticated requests. [source,yaml] ---- -signed-response: +auth: enabled: true - algorithm: SECP256K1 - private-key: /path/key.pem + publicKeyOwner: "token-issuer-name" + server: + keys: + provider-private-key: "/etc/dshackle/auth/jwt-rsa.pem" + external-public-key: "/etc/dshackle/auth/jwt-rsa.pub" ---- -.Redis Config -[cols="2a,2,5"] |=== -| Option | Default Value | Description +| Name | Default | Description | `enabled` | `false` -| Enable/disable Signed Responses +| Enables authorization and response signing. -| `algorithm` -| `SECP256K1` -| `SECP256K1` or `NIST-P256` +| `publicKeyOwner` +| +| Expected value of the `iss` claim on inbound JWT tokens. -| `private-key` +| `server.keys.provider-private-key` | -| Path to a private key in PEM format +| Path to a PKCS#8 PEM RSA private key. Used both to sign session JWTs issued by + dshackle and to sign `NativeCall` response payloads (see <>). +| `server.keys.external-public-key` +| +| Path to a PEM-encoded RSA public key (X.509 SubjectPublicKeyInfo) used to verify + the JWTs clients present to `emerald.Auth/Authenticate`. |=== -See more details at xref:07-methods.adoc#signatures[Signed Response] in gRPC Methods. +[#response-signing] +==== Response Signing + +When `auth.enabled` is `true` and `auth.server.keys.provider-private-key` points to a +valid PKCS#8 RSA private key, dshackle automatically signs gRPC responses with +`SHA256withRSA` for any `NativeCall` request that provides a non-zero `nonce`. The same +key used for issuing JWT tokens (RS256) is reused for response signatures — no separate +configuration is required. + +The signed blob is `DSHACKLESIG///`. The +returned `NativeCallReplySignature` carries the original `nonce`, the signature bytes +and a `key_id` (first 8 bytes of the SHA-256 of the public key). Clients verify with +the public half of `provider-private-key`. + +If a client sends a nonce but the signing key is not configured (auth disabled or the +path is empty), dshackle returns an error with code `-32603` and message +"Response signing requested via nonce but signing key is not configured". + +A runnable end-to-end example (dshackle config, demo RSA keys and a Go client) lives +in `demo/response-signing/` in the repository. [#cluster] == Cluster @@ -822,6 +849,18 @@ rpc: password: "${ETH_PASSWORD}" ---- +| `rpc.bearer-auth` + `rpc.bearer-auth.token` +a| HTTP Bearer token authorization (`Authorization: Bearer ` header), if required by the remote server. + +Cannot be used together with `basic-auth`. +Value can also reference env variables, for example: +[source,yaml] +---- +rpc: + url: "https://ethereum.com:8545" + bearer-auth: + token: "${ETH_TOKEN}" +---- + | `ws.url` | WebSocket URL to connect to. Optional, but optimizes performance if it's available. @@ -832,6 +871,9 @@ Optional, but optimizes performance if it's available. | `ws.basic-auth` + ... | WebSocket Basic Auth configuration, if required by the remote server +| `ws.bearer-auth` + `ws.bearer-auth.token` +| WebSocket Bearer token authorization, if required by the remote server + | `ws.frameSize` | WebSocket frame size limit. Ex `1kb`, `1024` (same as `1kb), `2mb`, etc. diff --git a/dshackle-cli/README.md b/dshackle-cli/README.md deleted file mode 100644 index b43b6f945..000000000 --- a/dshackle-cli/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# DSHACKLE CLI - -A CLI tool to verify the [dshackle](https://github.com/emeraldpay/dshackle) installation. - -## Usage -``` -npx dshackle-cli-tool [-p|--print][--ca][--cert][--key] -``` - -### Options -```-p | --print``` will print the describe response as is - -### TLS -```--ca``` the root certificate data - -```--cert``` the client certificate key chain, if available -```--key``` the client certificate private key, if available - -## Example - -``` -> dshackle-cli-tool localhost:2449 -Connecting to: localhost:2449... -Connected to localhost:2449 -CHAIN_ETHEREUM -> AVAIL_OK -CHAIN_KOVAN -> AVAIL_OK -``` - -With TLS -``` ->dshackle-cli % dshackle-cli-tool --ca ./../out/ca.myhost.dev.crt --cert ./../out/client_1.crt --key ./../out/client_1.key 127.0.0.1:2450 -Using TLS -Connecting to: 127.0.0.1:2450... -Connected to 127.0.0.1:2450 -CHAIN_ETHEREUM -> AVAIL_OK -``` \ No newline at end of file diff --git a/dshackle-cli/bin/dshackle-cli-tool b/dshackle-cli/bin/dshackle-cli-tool deleted file mode 100755 index c2ecc4bba..000000000 --- a/dshackle-cli/bin/dshackle-cli-tool +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env node - -require = require('esm')(module /*, options*/); -require('../src/cli').cli(process.argv); \ No newline at end of file diff --git a/dshackle-cli/package-lock.json b/dshackle-cli/package-lock.json deleted file mode 100644 index 23c555c49..000000000 --- a/dshackle-cli/package-lock.json +++ /dev/null @@ -1,910 +0,0 @@ -{ - "name": "dshackle-cli-tool", - "version": "1.0.7", - "lockfileVersion": 2, - "requires": true, - "packages": { - "": { - "name": "dshackle-cli-tool", - "version": "1.0.7", - "license": "MIT", - "dependencies": { - "@grpc/grpc-js": "^1.6.12", - "@grpc/proto-loader": "^0.7.2", - "arg": "^5.0.2", - "cli-color": "^2.0.3", - "esm": "^3.2.25" - }, - "bin": { - "dshackle-cli-tool": "bin/dshackle-cli-tool" - } - }, - "node_modules/@grpc/grpc-js": { - "version": "1.6.12", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.6.12.tgz", - "integrity": "sha512-JmvQ03OTSpVd9JTlj/K3IWHSz4Gk/JMLUTtW7Zb0KvO1LcOYGATh5cNuRYzCAeDR3O8wq+q8FZe97eO9MBrkUw==", - "dependencies": { - "@grpc/proto-loader": "^0.7.0", - "@types/node": ">=12.12.47" - }, - "engines": { - "node": "^8.13.0 || >=10.10.0" - } - }, - "node_modules/@grpc/proto-loader": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.2.tgz", - "integrity": "sha512-jCdyLIT/tdQ1zhrbTQnJNK5nbDf0GoBpy5jVNywBzzMDF+Vs6uEaHnfz46dMtDxkvwrF2hzk5Z67goliceH0sA==", - "dependencies": { - "@types/long": "^4.0.1", - "lodash.camelcase": "^4.3.0", - "long": "^4.0.0", - "protobufjs": "^7.0.0", - "yargs": "^16.2.0" - }, - "bin": { - "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" - }, - "node_modules/@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" - }, - "node_modules/@protobufjs/codegen": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" - }, - "node_modules/@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" - }, - "node_modules/@protobufjs/fetch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", - "dependencies": { - "@protobufjs/aspromise": "^1.1.1", - "@protobufjs/inquire": "^1.1.0" - } - }, - "node_modules/@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" - }, - "node_modules/@protobufjs/inquire": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" - }, - "node_modules/@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" - }, - "node_modules/@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" - }, - "node_modules/@protobufjs/utf8": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" - }, - "node_modules/@types/long": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", - "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" - }, - "node_modules/@types/node": { - "version": "18.7.15", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.15.tgz", - "integrity": "sha512-XnjpaI8Bgc3eBag2Aw4t2Uj/49lLBSStHWfqKvIuXD7FIrZyMLWp8KuAFHAqxMZYTF9l08N1ctUn9YNybZJVmQ==" - }, - "node_modules/arg": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", - "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==" - }, - "node_modules/cli-color": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/cli-color/-/cli-color-2.0.3.tgz", - "integrity": "sha512-OkoZnxyC4ERN3zLzZaY9Emb7f/MhBOIpePv0Ycok0fJYT+Ouo00UBEIwsVsr0yoow++n5YWlSUgST9GKhNHiRQ==", - "dependencies": { - "d": "^1.0.1", - "es5-ext": "^0.10.61", - "es6-iterator": "^2.0.3", - "memoizee": "^0.4.15", - "timers-ext": "^0.1.7" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/cliui": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", - "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, - "node_modules/cliui/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/cliui/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, - "node_modules/cliui/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, - "node_modules/d": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", - "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", - "dependencies": { - "es5-ext": "^0.10.50", - "type": "^1.0.1" - } - }, - "node_modules/es5-ext": { - "version": "0.10.62", - "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.62.tgz", - "integrity": "sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==", - "hasInstallScript": true, - "dependencies": { - "es6-iterator": "^2.0.3", - "es6-symbol": "^3.1.3", - "next-tick": "^1.1.0" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/es6-iterator": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", - "integrity": "sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==", - "dependencies": { - "d": "1", - "es5-ext": "^0.10.35", - "es6-symbol": "^3.1.1" - } - }, - "node_modules/es6-symbol": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", - "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", - "dependencies": { - "d": "^1.0.1", - "ext": "^1.1.2" - } - }, - "node_modules/es6-weak-map": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.3.tgz", - "integrity": "sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==", - "dependencies": { - "d": "1", - "es5-ext": "^0.10.46", - "es6-iterator": "^2.0.3", - "es6-symbol": "^3.1.1" - } - }, - "node_modules/escalade": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", - "engines": { - "node": ">=6" - } - }, - "node_modules/esm": { - "version": "3.2.25", - "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", - "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==", - "engines": { - "node": ">=6" - } - }, - "node_modules/event-emitter": { - "version": "0.3.5", - "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", - "integrity": "sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==", - "dependencies": { - "d": "1", - "es5-ext": "~0.10.14" - } - }, - "node_modules/ext": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", - "integrity": "sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==", - "dependencies": { - "type": "^2.7.2" - } - }, - "node_modules/ext/node_modules/type": { - "version": "2.7.2", - "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz", - "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==" - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-promise": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz", - "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==" - }, - "node_modules/lodash.camelcase": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", - "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" - }, - "node_modules/long": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", - "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" - }, - "node_modules/lru-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/lru-queue/-/lru-queue-0.1.0.tgz", - "integrity": "sha512-BpdYkt9EvGl8OfWHDQPISVpcl5xZthb+XPsbELj5AQXxIC8IriDZIQYjBJPEm5rS420sjZ0TLEzRcq5KdBhYrQ==", - "dependencies": { - "es5-ext": "~0.10.2" - } - }, - "node_modules/memoizee": { - "version": "0.4.15", - "resolved": "https://registry.npmjs.org/memoizee/-/memoizee-0.4.15.tgz", - "integrity": "sha512-UBWmJpLZd5STPm7PMUlOw/TSy972M+z8gcyQ5veOnSDRREz/0bmpyTfKt3/51DhEBqCZQn1udM/5flcSPYhkdQ==", - "dependencies": { - "d": "^1.0.1", - "es5-ext": "^0.10.53", - "es6-weak-map": "^2.0.3", - "event-emitter": "^0.3.5", - "is-promise": "^2.2.2", - "lru-queue": "^0.1.0", - "next-tick": "^1.1.0", - "timers-ext": "^0.1.7" - } - }, - "node_modules/next-tick": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", - "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==" - }, - "node_modules/protobufjs": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.1.0.tgz", - "integrity": "sha512-rCuxKlh0UQKSMjrpIcTLbR5TtGQ52cgs1a5nUoPBAKOccdPblN67BJtjrbtudUJK6HmBvUdsmymyYOzO7lxZEA==", - "hasInstallScript": true, - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/node": ">=13.7.0", - "long": "^5.0.0" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/protobufjs/node_modules/long": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/long/-/long-5.2.0.tgz", - "integrity": "sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w==" - }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/timers-ext": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.7.tgz", - "integrity": "sha512-b85NUNzTSdodShTIbky6ZF02e8STtVVfD+fu4aXXShEELpozH+bCpJLYMPZbsABN2wDH7fJpqIoXxJpzbf0NqQ==", - "dependencies": { - "es5-ext": "~0.10.46", - "next-tick": "1" - } - }, - "node_modules/type": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", - "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" - }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs": { - "version": "16.2.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", - "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "dependencies": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs-parser": { - "version": "20.2.9", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", - "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, - "node_modules/yargs/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - } - }, - "dependencies": { - "@grpc/grpc-js": { - "version": "1.6.12", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.6.12.tgz", - "integrity": "sha512-JmvQ03OTSpVd9JTlj/K3IWHSz4Gk/JMLUTtW7Zb0KvO1LcOYGATh5cNuRYzCAeDR3O8wq+q8FZe97eO9MBrkUw==", - "requires": { - "@grpc/proto-loader": "^0.7.0", - "@types/node": ">=12.12.47" - } - }, - "@grpc/proto-loader": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.2.tgz", - "integrity": "sha512-jCdyLIT/tdQ1zhrbTQnJNK5nbDf0GoBpy5jVNywBzzMDF+Vs6uEaHnfz46dMtDxkvwrF2hzk5Z67goliceH0sA==", - "requires": { - "@types/long": "^4.0.1", - "lodash.camelcase": "^4.3.0", - "long": "^4.0.0", - "protobufjs": "^7.0.0", - "yargs": "^16.2.0" - } - }, - "@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" - }, - "@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" - }, - "@protobufjs/codegen": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" - }, - "@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" - }, - "@protobufjs/fetch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", - "requires": { - "@protobufjs/aspromise": "^1.1.1", - "@protobufjs/inquire": "^1.1.0" - } - }, - "@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" - }, - "@protobufjs/inquire": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" - }, - "@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" - }, - "@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" - }, - "@protobufjs/utf8": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" - }, - "@types/long": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", - "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" - }, - "@types/node": { - "version": "18.7.15", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.15.tgz", - "integrity": "sha512-XnjpaI8Bgc3eBag2Aw4t2Uj/49lLBSStHWfqKvIuXD7FIrZyMLWp8KuAFHAqxMZYTF9l08N1ctUn9YNybZJVmQ==" - }, - "arg": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", - "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==" - }, - "cli-color": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/cli-color/-/cli-color-2.0.3.tgz", - "integrity": "sha512-OkoZnxyC4ERN3zLzZaY9Emb7f/MhBOIpePv0Ycok0fJYT+Ouo00UBEIwsVsr0yoow++n5YWlSUgST9GKhNHiRQ==", - "requires": { - "d": "^1.0.1", - "es5-ext": "^0.10.61", - "es6-iterator": "^2.0.3", - "memoizee": "^0.4.15", - "timers-ext": "^0.1.7" - } - }, - "cliui": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", - "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - }, - "dependencies": { - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" - }, - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "requires": { - "color-convert": "^2.0.1" - } - }, - "emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, - "string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, - "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - } - }, - "wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - } - } - }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, - "d": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", - "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", - "requires": { - "es5-ext": "^0.10.50", - "type": "^1.0.1" - } - }, - "es5-ext": { - "version": "0.10.62", - "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.62.tgz", - "integrity": "sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==", - "requires": { - "es6-iterator": "^2.0.3", - "es6-symbol": "^3.1.3", - "next-tick": "^1.1.0" - } - }, - "es6-iterator": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", - "integrity": "sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==", - "requires": { - "d": "1", - "es5-ext": "^0.10.35", - "es6-symbol": "^3.1.1" - } - }, - "es6-symbol": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", - "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", - "requires": { - "d": "^1.0.1", - "ext": "^1.1.2" - } - }, - "es6-weak-map": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.3.tgz", - "integrity": "sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==", - "requires": { - "d": "1", - "es5-ext": "^0.10.46", - "es6-iterator": "^2.0.3", - "es6-symbol": "^3.1.1" - } - }, - "escalade": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" - }, - "esm": { - "version": "3.2.25", - "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", - "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==" - }, - "event-emitter": { - "version": "0.3.5", - "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", - "integrity": "sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==", - "requires": { - "d": "1", - "es5-ext": "~0.10.14" - } - }, - "ext": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", - "integrity": "sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==", - "requires": { - "type": "^2.7.2" - }, - "dependencies": { - "type": { - "version": "2.7.2", - "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz", - "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==" - } - } - }, - "get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" - }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" - }, - "is-promise": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz", - "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==" - }, - "lodash.camelcase": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", - "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" - }, - "long": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", - "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" - }, - "lru-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/lru-queue/-/lru-queue-0.1.0.tgz", - "integrity": "sha512-BpdYkt9EvGl8OfWHDQPISVpcl5xZthb+XPsbELj5AQXxIC8IriDZIQYjBJPEm5rS420sjZ0TLEzRcq5KdBhYrQ==", - "requires": { - "es5-ext": "~0.10.2" - } - }, - "memoizee": { - "version": "0.4.15", - "resolved": "https://registry.npmjs.org/memoizee/-/memoizee-0.4.15.tgz", - "integrity": "sha512-UBWmJpLZd5STPm7PMUlOw/TSy972M+z8gcyQ5veOnSDRREz/0bmpyTfKt3/51DhEBqCZQn1udM/5flcSPYhkdQ==", - "requires": { - "d": "^1.0.1", - "es5-ext": "^0.10.53", - "es6-weak-map": "^2.0.3", - "event-emitter": "^0.3.5", - "is-promise": "^2.2.2", - "lru-queue": "^0.1.0", - "next-tick": "^1.1.0", - "timers-ext": "^0.1.7" - } - }, - "next-tick": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", - "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==" - }, - "protobufjs": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.1.0.tgz", - "integrity": "sha512-rCuxKlh0UQKSMjrpIcTLbR5TtGQ52cgs1a5nUoPBAKOccdPblN67BJtjrbtudUJK6HmBvUdsmymyYOzO7lxZEA==", - "requires": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/node": ">=13.7.0", - "long": "^5.0.0" - }, - "dependencies": { - "long": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/long/-/long-5.2.0.tgz", - "integrity": "sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w==" - } - } - }, - "require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" - }, - "timers-ext": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.7.tgz", - "integrity": "sha512-b85NUNzTSdodShTIbky6ZF02e8STtVVfD+fu4aXXShEELpozH+bCpJLYMPZbsABN2wDH7fJpqIoXxJpzbf0NqQ==", - "requires": { - "es5-ext": "~0.10.46", - "next-tick": "1" - } - }, - "type": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", - "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" - }, - "y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" - }, - "yargs": { - "version": "16.2.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", - "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "requires": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - }, - "dependencies": { - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" - }, - "emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, - "string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, - "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - } - } - } - }, - "yargs-parser": { - "version": "20.2.9", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", - "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==" - } - } -} diff --git a/dshackle-cli/package.json b/dshackle-cli/package.json deleted file mode 100644 index 495305f4e..000000000 --- a/dshackle-cli/package.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "dshackle-cli-tool", - "version": "1.0.7", - "description": "", - "main": "src/index.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "bin": { - "@mfomenkov/dshackle-cli-tool": "bin/dshackle-cli-tool", - "dshackle-cli-tool": "bin/dshackle-cli-tool" - }, - "keywords": [], - "author": "", - "license": "MIT", - "dependencies": { - "@grpc/grpc-js": "^1.6.12", - "@grpc/proto-loader": "^0.7.2", - "arg": "^5.0.2", - "cli-color": "^2.0.3", - "esm": "^3.2.25" - }, - "files": [ - "bin/", - "src/", - "grpc/" - ] -} diff --git a/dshackle-cli/src/cli.js b/dshackle-cli/src/cli.js deleted file mode 100644 index b0eee8966..000000000 --- a/dshackle-cli/src/cli.js +++ /dev/null @@ -1,144 +0,0 @@ -import {describe, connect, nativeCall} from "./grpc-clent"; -import arg from 'arg'; -import clc from "cli-color"; -import util from "util"; - -const path = require('path') -const protoLoader = require("@grpc/proto-loader"); -const grpc = require("@grpc/grpc-js"); - -const options = { - keepCase: true, - longs: String, - enums: String, - defaults: true, - oneofs: true, -}; - -const PROTO_PATH = path.join(__dirname, "../../emerald-grpc/proto/blockchain.proto"); -const packageDefinition = protoLoader.loadSync(PROTO_PATH, options); -const emerald = grpc.loadPackageDefinition(packageDefinition).emerald - -export function cli(args) { - let opts = parseArgumentsIntoOptions(args); - if (!opts.url) { - console.log("Err: URL not specified!!!") - return - } - - const chains = mapChains() - - const client = connect(opts.url, opts.ca, opts.cert, opts.key, emerald) - describe(client, (error, response) => { - if (error) { - console.error(clc.red('Connection to ' + opts.url + ' failed: ' + error.message)); - return - } - console.log(clc.green('Connected to ', opts.url)); - if (opts.print) { - console.log(util.inspect(response, false, null, true /* enable colors */)); - } - processDescribe(client, response, opts.testRun, chains) - }) -} - -function processDescribe(client, response, testRun, chains) { - let promises = [] - let statuses = new Map() - - response.chains.forEach((item) => { - const state = item.status.availability - const chain = item.status.chain - let status = { - state: 'AVAIL_UNKNOWN', - grpc: clc.yellow('UNKNOWN'), - failed: false - } - - switch (state) { - case 'AVAIL_OK': - status.state = clc.green(state); - break - case 'AVAIL_UNKNOWN': - case 'AVAIL_UNAVAILABLE': - status.state = clc.red(state); - break - default: - status.state = clc.yellow(state); - } - - if (state === 'AVAIL_OK') { - promises.push(nativeCall(client, chains.get(chain), chain)) - } else { - status.failed = true - } - statuses.set(chain, status) - }) - - Promise.all(promises).then(responses => { - responses.forEach((resp) => { - let status = statuses.get(resp.chain) - if (resp.error) { - status.failed = true - status.grpc = clc.red(resp.error.message) - } else { - if (resp.payload.succeed) { - status.grpc = clc.green('OK') - } else { - status.grpc = clc.red('FAILED') - status.failed = true - } - } - }) - - let hasError = false - statuses.forEach((status, chain) => { - printState(chain, status) - if (status.failed) { - hasError = true - } - }) - - if (hasError && testRun) { - process.exit(1) - } - }) -} - -function mapChains() { - return new Map( - emerald.ChainRef.type.value.map(obj => { - return [obj.name, obj.number] - }) - ) -} - -function printState(chain, status) { - console.log(chain + ' -> ' + "state: " + clc.bold(status.state) + " gRPC: " + clc.bold(status.grpc)) -} - -function parseArgumentsIntoOptions(rawArgs) { - const args = arg( - { - '--print': Boolean, - '--test-run': Boolean, - '--ca': String, - '--cert': String, - '--key': String, - '-p': '--print' - }, - { - argv: rawArgs.slice(2), - } - ); - return { - print: args['--print'] || false, - testRun: args['--test-run'] || false, - url: args._[0], - ca: args['--ca'], - cert: args['--cert'], - key: args['--key'] - }; -} - - diff --git a/dshackle-cli/src/grpc-clent.js b/dshackle-cli/src/grpc-clent.js deleted file mode 100644 index b6ed75b1a..000000000 --- a/dshackle-cli/src/grpc-clent.js +++ /dev/null @@ -1,53 +0,0 @@ -const grpc = require("@grpc/grpc-js"); -const fs = require('fs'); - -var id = 100 - -export function connect(url, ca, cert, key, emerald) { - let credentials = grpc.credentials.createInsecure() - if (ca || cert || key) { - console.log("Using TLS") - credentials = grpc.credentials.createSsl( - ca ? fs.readFileSync(ca) : null, - key ? fs.readFileSync(key) : null, - cert ? fs.readFileSync(cert) : null - ); - } - console.log('Connecting to: ' + url + '...') - return new emerald.Blockchain( - url, - credentials - ); -} - -export function describe(client, handler) { - client.Describe({}, handler); -} - -export function nativeCall(client, chainCode, chain) { - return new Promise((resolve, reject) => { - const call = client.NativeCall({ - chain: chainCode, - items: [{ - id: id++, - method: "eth_getBalance", - payload: "WyIweDhEOTc2ODlDOTgxODg5MkI3MDBlMjdGMzE2Y2MzRTQxZTE3ZkJlYjkiLCAibGF0ZXN0Il0=" - }], - quorum: 1, - min_availability: 0 - }) - call.on('data', (item) => { - resolve(toResult(chain, item, null)) - }) - call.on('end', () => resolve(toResult(chain, null, null))) - call.on('error', (e) => reject(toResult(chain, null, e))) - }) -} - -function toResult(chain, obj, err) { - return { - chain: chain, - payload: obj, - error: err - } -} diff --git a/emerald-grpc b/emerald-grpc index 1698b6f9e..519173182 160000 --- a/emerald-grpc +++ b/emerald-grpc @@ -1 +1 @@ -Subproject commit 1698b6f9e332548fe263098f877cd8e08c9963f8 +Subproject commit 519173182997d0cff06c5338982b18e81aba99ea diff --git a/foundation/build.gradle b/foundation/build.gradle index 5828bb866..e2cc05d00 100644 --- a/foundation/build.gradle +++ b/foundation/build.gradle @@ -1,6 +1,9 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget + plugins { - id 'org.jetbrains.kotlin.jvm' version '1.9.25' + id 'org.jetbrains.kotlin.jvm' version '2.3.21' id 'maven-publish' + id 'java' } repositories { @@ -10,9 +13,22 @@ repositories { group = 'dshackle' +java { + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 +} + +compileKotlin { + compilerOptions.jvmTarget.set(JvmTarget.JVM_21) +} +compileTestKotlin { + compilerOptions.jvmTarget.set(JvmTarget.JVM_21) +} + dependencies { - implementation 'org.yaml:snakeyaml:1.24' - testImplementation 'org.junit.jupiter:junit-jupiter:5.9.1' + implementation 'org.yaml:snakeyaml:2.6' + testImplementation 'org.junit.jupiter:junit-jupiter:6.0.3' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' } test { diff --git a/foundation/gradle/wrapper/gradle-wrapper.jar b/foundation/gradle/wrapper/gradle-wrapper.jar index 7454180f2..f8e1ee312 100644 Binary files a/foundation/gradle/wrapper/gradle-wrapper.jar and b/foundation/gradle/wrapper/gradle-wrapper.jar differ diff --git a/foundation/gradle/wrapper/gradle-wrapper.properties b/foundation/gradle/wrapper/gradle-wrapper.properties index a59520664..cf71939ed 100644 --- a/foundation/gradle/wrapper/gradle-wrapper.properties +++ b/foundation/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-9.5.1-bin.zip +networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/foundation/src/main/kotlin/io/emeraldpay/dshackle/BlockchainType.kt b/foundation/src/main/kotlin/io/emeraldpay/dshackle/BlockchainType.kt index 1f10d9bcd..a81006c03 100644 --- a/foundation/src/main/kotlin/io/emeraldpay/dshackle/BlockchainType.kt +++ b/foundation/src/main/kotlin/io/emeraldpay/dshackle/BlockchainType.kt @@ -5,6 +5,7 @@ enum class BlockchainType( ) { UNKNOWN(ApiType.JSON_RPC), BITCOIN(ApiType.JSON_RPC), + AZTEC(ApiType.JSON_RPC), ETHEREUM(ApiType.JSON_RPC), STARKNET(ApiType.JSON_RPC), POLKADOT(ApiType.JSON_RPC), @@ -14,9 +15,10 @@ enum class BlockchainType( COSMOS(ApiType.JSON_RPC), TON(ApiType.REST), RIPPLE(ApiType.JSON_RPC), - KADENA(ApiType.REST),; + KADENA(ApiType.REST), + AVM(ApiType.REST),; } enum class ApiType { JSON_RPC, REST; -} \ No newline at end of file +} diff --git a/foundation/src/main/kotlin/io/emeraldpay/dshackle/foundation/ChainOptions.kt b/foundation/src/main/kotlin/io/emeraldpay/dshackle/foundation/ChainOptions.kt index 16470c705..6a1aae22a 100644 --- a/foundation/src/main/kotlin/io/emeraldpay/dshackle/foundation/ChainOptions.kt +++ b/foundation/src/main/kotlin/io/emeraldpay/dshackle/foundation/ChainOptions.kt @@ -20,6 +20,7 @@ class ChainOptions { val disableBoundValidation: Boolean = false, val valdateErigonBug: Boolean, val disableLogIndexValidation: Boolean = false, + val disablePendingTxValidation: Boolean = false, ) data class DefaultOptions( @@ -43,7 +44,8 @@ class ChainOptions { var disableLivenessSubscriptionValidation: Boolean? = null, var disableBoundValidation: Boolean? = null, var validateErigonBug: Boolean? = null, - var disableLogIndexValidation: Boolean? = null + var disableLogIndexValidation: Boolean? = null, + var disablePendingTxValidation: Boolean? = null ) { companion object { @JvmStatic @@ -76,6 +78,7 @@ class ChainOptions { copy.disableBoundValidation = overwrites.disableBoundValidation ?: this.disableBoundValidation copy.validateErigonBug = overwrites.validateErigonBug ?: this.validateErigonBug copy.disableLogIndexValidation = overwrites.disableLogIndexValidation ?: this.disableLogIndexValidation + copy.disablePendingTxValidation = overwrites.disablePendingTxValidation ?: this.disablePendingTxValidation return copy } @@ -97,6 +100,7 @@ class ChainOptions { this.disableBoundValidation ?: false, this.validateErigonBug ?: true, this.disableLogIndexValidation ?: false, + this.disablePendingTxValidation ?: false, ) } } diff --git a/foundation/src/main/kotlin/io/emeraldpay/dshackle/foundation/ChainOptionsReader.kt b/foundation/src/main/kotlin/io/emeraldpay/dshackle/foundation/ChainOptionsReader.kt index b2b951b83..7c7cfc335 100644 --- a/foundation/src/main/kotlin/io/emeraldpay/dshackle/foundation/ChainOptionsReader.kt +++ b/foundation/src/main/kotlin/io/emeraldpay/dshackle/foundation/ChainOptionsReader.kt @@ -61,6 +61,9 @@ class ChainOptionsReader : YamlConfigReader() { getValueAsBool(values, "disable-log-index-validation")?.let { options.disableLogIndexValidation = it } + getValueAsBool(values, "disable-pending-tx-validation")?.let { + options.disablePendingTxValidation = it + } return options } } diff --git a/foundation/src/main/kotlin/org/drpc/chainsconfig/ChainsConfig.kt b/foundation/src/main/kotlin/org/drpc/chainsconfig/ChainsConfig.kt index 7ffa3c7b0..4d174a6ff 100644 --- a/foundation/src/main/kotlin/org/drpc/chainsconfig/ChainsConfig.kt +++ b/foundation/src/main/kotlin/org/drpc/chainsconfig/ChainsConfig.kt @@ -44,6 +44,25 @@ data class ChainsConfig(private val chains: List) : Iterable "$op $limit" } } + enum class LowerBoundType { + UNKNOWN, STATE, SLOT, BLOCK, TX, LOGS, TRACE, PROOF, BLOB, EPOCH, RECEIPTS; + + companion object { + fun byName(name: String): LowerBoundType { + return LowerBoundType.entries.firstOrNull { it.name.equals(name, ignoreCase = true) } ?: UNKNOWN + } + } + } + + open class GoldLowerBound( + val block: Long, + ) + + class GoldLowerBoundWithHash( + block: Long, + val hash: String, + ) : GoldLowerBound(block) + data class ChainConfig( val expectedBlockTime: Duration, val syncingLagSize: Int, @@ -59,6 +78,7 @@ data class ChainsConfig(private val chains: List) : Iterable ) { companion object { @JvmStatic @@ -80,6 +100,7 @@ data class ChainsConfig(private val chains: List) : Iterable) : Iterable { + return chainMap.values + } } diff --git a/foundation/src/main/kotlin/org/drpc/chainsconfig/ChainsConfigReader.kt b/foundation/src/main/kotlin/org/drpc/chainsconfig/ChainsConfigReader.kt index 19c0666b7..e79340047 100644 --- a/foundation/src/main/kotlin/org/drpc/chainsconfig/ChainsConfigReader.kt +++ b/foundation/src/main/kotlin/org/drpc/chainsconfig/ChainsConfigReader.kt @@ -137,9 +137,47 @@ class ChainsConfigReader( blockchain = blockchain, type = type, gasPriceCondition = ChainsConfig.GasPriceCondition(gasPriceConditions), + goldLowerBounds = readGoldLowerBounds(node), ) } + private fun readGoldLowerBounds(node: MappingNode): Map { + val goldLowerBounds = getMapping(node, "lower-bounds") ?: return emptyMap() + + return goldLowerBounds.value.asSequence() + .filter { + (it.keyNode.valueAsString()?.isNotBlank() ?: false) && it.valueNode.tag == Tag.MAP + }.map { + val type = ChainsConfig.LowerBoundType.byName(it.keyNode.valueAsString()!!) + val bound = if (type == ChainsConfig.LowerBoundType.TX || type == ChainsConfig.LowerBoundType.RECEIPTS) { + readHashGoldLowerBound(it.valueNode as MappingNode) + } else { + readGoldLowerBound(it.valueNode as MappingNode) + } + + if (bound != null) { + type to bound + } else { + null + } + } + .filterNotNull() + .associate { it } + } + + private fun readGoldLowerBound(node: MappingNode): ChainsConfig.GoldLowerBound? { + val block = getValueAsLong(node, "block") ?: return null + + return ChainsConfig.GoldLowerBound(block) + } + + private fun readHashGoldLowerBound(node: MappingNode): ChainsConfig.GoldLowerBound? { + val hash = getValueAsString(node, "hash") ?: return null + val block = getValueAsLong(node, "block") ?: return null + + return ChainsConfig.GoldLowerBoundWithHash(block, hash) + } + override fun read(input: MappingNode?): ChainsConfig { val default = readChains(readNode(defaultConfig)) val current = readChains(input) diff --git a/foundation/src/main/resources/public b/foundation/src/main/resources/public index 878ce981c..0a070aaca 160000 --- a/foundation/src/main/resources/public +++ b/foundation/src/main/resources/public @@ -1 +1 @@ -Subproject commit 878ce981c7a0f724226969f92001225213123ebc +Subproject commit 0a070aaca7b5e80541d277ebadf172a66629ec8e diff --git a/foundation/src/test/kotlin/org/drpc/chainsconfig/ChainsConfigReaderTest.kt b/foundation/src/test/kotlin/org/drpc/chainsconfig/ChainsConfigReaderTest.kt index abc1f6a12..c1a529d3f 100644 --- a/foundation/src/test/kotlin/org/drpc/chainsconfig/ChainsConfigReaderTest.kt +++ b/foundation/src/test/kotlin/org/drpc/chainsconfig/ChainsConfigReaderTest.kt @@ -1,12 +1,13 @@ package org.drpc.chainsconfig +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.config.ChainsConfigReader import io.emeraldpay.dshackle.foundation.ChainOptionsReader import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import java.math.BigInteger -internal class ChainsConfigReaderTest { +class ChainsConfigReaderTest { @Test fun `read standard config without custom`() { @@ -41,5 +42,17 @@ internal class ChainsConfigReaderTest { assertEquals(config.code, "FTA") assertEquals(config.grpcId, 102) assertEquals(config.netVersion, BigInteger.valueOf(251)) + assertEquals(15L, config.goldLowerBounds[ChainsConfig.LowerBoundType.STATE]!!.block) + assertEquals(150L, config.goldLowerBounds[ChainsConfig.LowerBoundType.BLOCK]!!.block) + assertEquals(1L, config.goldLowerBounds[ChainsConfig.LowerBoundType.TX]!!.block) + assertEquals( + "0x5e77a04531c7c107af1882d76cbff9486d0a9aa53701c30888509d4f5f2b003a", + (config.goldLowerBounds[ChainsConfig.LowerBoundType.TX] as ChainsConfig.GoldLowerBoundWithHash).hash, + ) + assertEquals(1000L, config.goldLowerBounds[ChainsConfig.LowerBoundType.RECEIPTS]!!.block) + assertEquals( + "0x4e72a04531c7c107af1882d76cbff9486d0a9aa53701c30888509d4f5f2b003a", + (config.goldLowerBounds[ChainsConfig.LowerBoundType.RECEIPTS] as ChainsConfig.GoldLowerBoundWithHash).hash, + ) } } diff --git a/foundation/src/test/resources/configs/chains-basic.yaml b/foundation/src/test/resources/configs/chains-basic.yaml index 8ba3e3af9..cae9937fa 100644 --- a/foundation/src/test/resources/configs/chains-basic.yaml +++ b/foundation/src/test/resources/configs/chains-basic.yaml @@ -16,4 +16,15 @@ chain-settings: short-names: [fantom] code: FTA grpcId: 102 - chain-id: 0xfb \ No newline at end of file + chain-id: 0xfb + lower-bounds: + tx: + block: 1 + hash: "0x5e77a04531c7c107af1882d76cbff9486d0a9aa53701c30888509d4f5f2b003a" + receipts: + block: 1000 + hash: "0x4e72a04531c7c107af1882d76cbff9486d0a9aa53701c30888509d4f5f2b003a" + state: + block: 15 + block: + block: 150 diff --git a/gradle.properties b/gradle.properties index fd99adb0b..eacee52bf 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ kotlin.code.style=official grpcVersion=1.49.2 -reactiveGrpcVersion=1.2.0 +reactiveGrpcVersion=1.2.4 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 8ca6212b5..217a3f164 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,53 +1,49 @@ [versions] -detekt = "1.23.1" -groovy = "4.0.15" -protoc = "4.29.2" -jackson = "2.11.0" -grpc = "1.57.0" -reactive-grpc = "1.2.0" -spring-boot = "2.6.15" -spring-security = "5.5.3" -reactor = "3.4.32" -netty = "4.1.110.Final" -netty-tcnative = "2.0.66.Final" -kotlin = "1.9.25" -httpcomponents = "4.5.8" +groovy = "5.0.6" +protoc = "4.34.1" +jackson = "2.21.3" +grpc = "1.81.0" +spring-boot = "4.0.6" +reactor = "3.8.5" +netty = "4.2.13.Final" +netty-tcnative = "2.0.77.Final" +kotlin = "2.3.21" +httpcomponents = "5.6.1" [libraries] -apache-commons-lang3 = "org.apache.commons:commons-lang3:3.9" -apache-commons-collections4 = "org.apache.commons:commons-collections4:4.3" +apache-commons-lang3 = "org.apache.commons:commons-lang3:3.20.0" +apache-commons-collections4 = "org.apache.commons:commons-collections4:4.5.0" apache-commons-math3 = "org.apache.commons:commons-math3:3.6.1" +commons-codec = "commons-codec:commons-codec:1.22.0" -bitcoinj = "org.bitcoinj:bitcoinj-core:0.15.8" +bitcoinj = "org.bitcoinj:bitcoinj-core:0.17.1" -logstash-encoder = "net.logstash.logback:logstash-logback-encoder:7.2" +logstash-encoder = "net.logstash.logback:logstash-logback-encoder:9.0" -janino = "org.codehaus.janino:janino:3.1.9" +janino = "org.codehaus.janino:janino:3.1.12" -bouncycastle-prov = "org.bouncycastle:bcprov-jdk15on:1.61" -bouncycastle-pkix = 'org.bouncycastle:bcpkix-jdk15on:1.61' +bouncycastle-prov = "org.bouncycastle:bcprov-jdk15on:1.70" +bouncycastle-pkix = 'org.bouncycastle:bcpkix-jdk15on:1.70' caffeine = "com.github.ben-manes.caffeine:caffeine:2.8.5" -commons-io = "commons-io:commons-io:2.6" +commons-io = "commons-io:commons-io:2.22.0" cglib-nodep = "cglib:cglib-nodep:3.3.0" -detekt-formatting = { module = "io.gitlab.arturbosch.detekt:detekt-formatting", version.ref = "detekt" } - -equals-verifier = "nl.jqno.equalsverifier:equalsverifier:3.10.1" +equals-verifier = "nl.jqno.equalsverifier:equalsverifier:4.5" groovy = { module = "org.apache.groovy:groovy", version.ref = "groovy" } grpc-proto-util = { module = "com.google.protobuf:protobuf-java-util", version.ref = "protoc" } grpc-protobuf = { module = "io.grpc:grpc-protobuf", version.ref = "grpc" } +grpc-inprocess = { module = "io.grpc:grpc-inprocess", version.ref = "grpc" } grpc-stub = { module = "io.grpc:grpc-stub", version.ref = "grpc" } grpc-netty = { module = "io.grpc:grpc-netty", version.ref = "grpc" } grpc-testing = { module = "io.grpc:grpc-testing", version.ref = "grpc" } grpc-services = { module = "io.grpc:grpc-services", version.ref = "grpc" } -httpcomponents-httpmime = { module = "org.apache.httpcomponents:httpmime", version.ref = "httpcomponents" } -httpcomponents-httpclient = { module = "org.apache.httpcomponents:httpclient", version.ref = "httpcomponents" } +httpcomponents-httpclient = { module = "org.apache.httpcomponents.client5:httpclient5", version.ref = "httpcomponents" } jackson-core = { module = "com.fasterxml.jackson.core:jackson-core", version.ref = "jackson" } jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" } @@ -56,20 +52,22 @@ jackson-datatype-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-dat jackson-module-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version.ref = "jackson" } jackson-yaml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml", version.ref = "jackson" } -java-websocket = "org.java-websocket:Java-WebSocket:1.5.1" +java-websocket = "org.java-websocket:Java-WebSocket:1.6.0" javax-annotations = "javax.annotation:javax.annotation-api:1.3.2" kotlin-stdlib-jdk8 = { module = "org.jetbrains.kotlin:kotlin-stdlib-jdk8", version.ref = "kotlin" } kotlin-reflect = { module = "org.jetbrains.kotlin:kotlin-reflect", version.ref = "kotlin" } -lettuce-core = "io.lettuce:lettuce-core:5.2.2.RELEASE" +lettuce-core = "io.lettuce:lettuce-core:7.5.2.RELEASE" -micrometer-registry-prometheus = "io.micrometer:micrometer-registry-prometheus:1.10.0" +prom-exporter-server = "io.prometheus:prometheus-metrics-exporter-httpserver:1.6.1" +micrometer-registry-prometheus = "io.micrometer:micrometer-registry-prometheus:1.16.5" +micrometer-ctx-prop = "io.micrometer:context-propagation:1.2.1" -mockserver-netty = "org.mock-server:mockserver-netty:5.11.2" +mockserver-netty = "org.mock-server:mockserver-netty:5.15.0" -mockito-kotlin = "org.mockito.kotlin:mockito-kotlin:5.1.0" +mockito-kotlin = "org.mockito.kotlin:mockito-kotlin:6.3.0" netty-common = { module = "io.netty:netty-common", version.ref = "netty" } netty-transport = { module = "io.netty:netty-transport", version.ref = "netty" } @@ -85,62 +83,60 @@ netty-tcnative-core = { module = "io.netty:netty-tcnative", version.ref = "netty netty-tcnative-boringssl = { module = "io.netty:netty-tcnative-boringssl-static", version.ref = "netty-tcnative"} netty-macos = { module = "io.netty:netty-resolver-dns-native-macos", version.ref = "netty" } -zeromq = "org.zeromq:jeromq:0.5.2" +zeromq = "org.zeromq:jeromq:0.6.0" -objgenesis = "org.objenesis:objenesis:3.1" +objgenesis = "org.objenesis:objenesis:3.5" reactor-core = { module = "io.projectreactor:reactor-core", version.ref = "reactor" } -reactor-netty = { module = "io.projectreactor.netty:reactor-netty", version = "1.1.10" } -reactor-extra = { module = "io.projectreactor.addons:reactor-extra", version = "3.5.1" } -reactor-kotlin = { module = "io.projectreactor.kotlin:reactor-kotlin-extensions", version = "1.1.7" } +reactor-micrometer = { module = "io.projectreactor:reactor-core-micrometer", version.ref = "reactor" } +reactor-netty = { module = "io.projectreactor.netty:reactor-netty", version = "1.3.5" } +reactor-extra = { module = "io.projectreactor.addons:reactor-extra", version = "3.6.0" } +reactor-kotlin = { module = "io.projectreactor.kotlin:reactor-kotlin-extensions", version = "1.3.0" } reactor-test = { module = "io.projectreactor:reactor-test", version.ref = "reactor" } -reactor-grpc-stub = "com.salesforce.servicelibs:reactor-grpc-stub:1.2.0" +reactor-grpc-stub = "com.salesforce.servicelibs:reactor-grpc-stub:1.2.4" -snake-yaml = "org.yaml:snakeyaml:1.24" +snake-yaml = "org.yaml:snakeyaml:2.6" -spockframework-core = "org.spockframework:spock-core:2.3-groovy-4.0" +spockframework-core = "org.spockframework:spock-core:2.4-groovy-5.0" spring-boot-starter = { module = "org.springframework.boot:spring-boot-starter", version.ref = "spring-boot" } -spring-security-core = { module = "org.springframework.security:spring-security-core", version.ref = "spring-security" } -spring-security-web = { module = "org.springframework.security:spring-security-web", version.ref = "spring-security" } -spring-security-config = { module = "org.springframework.security:spring-security-config", version.ref = "spring-security" } +spring-boot-starter-actuator = { module = "org.springframework.boot:spring-boot-starter-actuator", version.ref = "spring-boot" } spring-boot-starter-test = { module = "org.springframework.boot:spring-boot-starter-test", version.ref = "spring-boot" } -testcontainers = "org.testcontainers:testcontainers:1.21.3" +testcontainers = "org.testcontainers:testcontainers:2.0.5" testcontainers-ganache = "io.github.ganchix:testcontainers-java-module-ganache:0.0.4" -junit-jupiter = "org.junit.jupiter:junit-jupiter:5.9.1" -assertj = "org.assertj:assertj-core:3.23.1" +assertj = "org.assertj:assertj-core:3.27.7" + +brave-instrumentation-grpc = "io.zipkin.brave:brave-instrumentation-grpc:6.3.1" +brave-ctx-slf4j = "io.zipkin.brave:brave-context-slf4j:6.3.1" -spring-cloud-starter-sleuth = "org.springframework.cloud:spring-cloud-starter-sleuth:3.1.6" -spring-cloud-sleuth-zipkin = "org.springframework.cloud:spring-cloud-sleuth-zipkin:3.1.6" -brave-instrumentation-grpc = "io.zipkin.brave:brave-instrumentation-grpc:5.15.0" +auth0-jwt = "com.auth0:java-jwt:4.5.2" -auth0-jwt = "com.auth0:java-jwt:4.4.0" +mockito-inline = "org.mockito:mockito-inline:5.2.0" -mockito-inline = "org.mockito:mockito-inline:4.0.0" +semver4j = "com.vdurmont:semver4j:3.1.0" [bundles] -apache-commons = ["commons-io", "apache-commons-lang3", "apache-commons-collections4", "apache-commons-math3"] -grpc = ["grpc-protobuf", "grpc-stub", "grpc-netty", "grpc-proto-util", "grpc-services"] -httpcomponents = ["httpcomponents-httpmime", "httpcomponents-httpclient"] +apache-commons = ["commons-io", "apache-commons-lang3", "apache-commons-collections4", "apache-commons-math3", "commons-codec"] +grpc = ["grpc-protobuf", "grpc-stub", "grpc-netty", "grpc-proto-util", "grpc-services", "grpc-inprocess"] +httpcomponents = ["httpcomponents-httpclient"] jackson = ["jackson-core", "jackson-databind", "jackson-datatype-jdk8", "jackson-datatype-jsr310", "jackson-module-kotlin", "jackson-yaml"] kotlin = ["kotlin-stdlib-jdk8", "kotlin-reflect"] netty = ["netty-common", "netty-transport", "netty-handler-core", "netty-handler-proxy", "netty-resolver-core", "netty-resolver-dns", "netty-codec-core", "netty-codec-http", "netty-codec-http2", "netty-buffer", "netty-tcnative-core"] -reactor = ["reactor-core", "reactor-netty", "reactor-extra", "reactor-kotlin"] -spring-framework = ["spring-boot-starter", "spring-security-core", "spring-security-web", "spring-security-config"] +reactor = ["reactor-core", "reactor-netty", "reactor-extra", "reactor-kotlin", "reactor-micrometer"] +spring-framework = ["spring-boot-starter", "spring-boot-starter-actuator"] testcontainers = ["testcontainers", "testcontainers-ganache"] -junit = ["junit-jupiter", "assertj"] +junit = ["assertj"] bouncycastle = ["bouncycastle-pkix", "bouncycastle-prov"] [plugins] kotlin = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } -jib = { id = "com.google.cloud.tools.jib", version = "3.4.3" } -spring = { id = "org.springframework.boot", version = "2.6.0" } -git = { id = "com.palantir.git-version", version = "0.12.3" } -protobuf = { id = "com.google.protobuf", version = "0.9.1" } +jib = { id = "com.google.cloud.tools.jib", version = "3.5.3" } +spring = { id = "org.springframework.boot", version = "4.0.6" } +git = { id = "com.palantir.git-version", version = "5.0.0" } +protobuf = { id = "com.google.protobuf", version = "0.10.0" } ktlint = { id = "org.jlleitschuh.gradle.ktlint", version = "11.6.0" } -detekt = { id = "io.gitlab.arturbosch.detekt", version.ref = "detekt" } \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 7454180f2..b1b8ef56b 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a59520664..4dcb8425b 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,8 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-9.5.1-bin.zip +networkTimeout=10000 +retries=0 +retryBackOffMs=500 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 1b6c78733..b9bb139f7 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ #!/bin/sh # -# Copyright © 2015-2021 the original authors. +# Copyright © 2015 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -55,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/3d91ce3b8caaf77ad09f381f43615b715b53f72c/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -80,13 +82,11 @@ do esac done -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" +# This is normally unused +# shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -114,7 +114,6 @@ case "$( uname )" in #( NONSTOP* ) nonstop=true ;; esac -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar # Determine the Java command to use to start the JVM. @@ -133,22 +132,29 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -165,7 +171,6 @@ fi # For Cygwin or MSYS, switch paths to Windows format before running java if "$cygwin" || "$msys" ; then APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) - CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) JAVACMD=$( cygpath --unix "$JAVACMD" ) @@ -193,18 +198,27 @@ if "$cygwin" || "$msys" ; then done fi -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ - -classpath "$CLASSPATH" \ - org.gradle.wrapper.GradleWrapperMain \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ "$@" +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + # Use "xargs" to parse quoted args. # # With -n1 it outputs one arg per line, with the quotes and backslashes removed. diff --git a/gradlew.bat b/gradlew.bat index ac1b06f93..aa5f10b06 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -13,19 +13,22 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @rem @rem ########################################################################## -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal +@rem Set local scope for the variables, and ensure extensions are enabled +setlocal EnableExtensions set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,15 +43,15 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute +if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 -goto fail +"%COMSPEC%" /c exit 1 :findJavaFromJavaHome set JAVA_HOME=%JAVA_HOME:"=% @@ -56,34 +59,24 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 -goto fail +"%COMSPEC%" /c exit 1 :execute @rem Setup the command line -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal +@rem endlocal doesn't take effect until after the line is parsed and variables are expanded +@rem which allows us to clear the local environment before executing the java command +endlocal & "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* & call :exitWithErrorLevel -:omega +:exitWithErrorLevel +@rem Use "%COMSPEC%" /c exit to allow operators to work properly in scripts +"%COMSPEC%" /c exit %ERRORLEVEL% diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt index b33ab61b4..51e8f08d5 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Config.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Config.kt @@ -24,7 +24,6 @@ import io.emeraldpay.dshackle.config.HealthConfig import io.emeraldpay.dshackle.config.MainConfig import io.emeraldpay.dshackle.config.MainConfigReader import io.emeraldpay.dshackle.config.MonitoringConfig -import io.emeraldpay.dshackle.config.SignatureConfig import io.emeraldpay.dshackle.config.TokensConfig import io.emeraldpay.dshackle.config.UpstreamsConfig import org.bouncycastle.jce.provider.BouncyCastleProvider @@ -130,11 +129,6 @@ open class Config( return mainConfig.cache ?: CacheConfig() } - @Bean - open fun signatureConfig(@Autowired mainConfig: MainConfig): SignatureConfig { - return mainConfig.signature ?: SignatureConfig() - } - @Bean open fun tokensConfig(@Autowired mainConfig: MainConfig): TokensConfig { return mainConfig.tokens ?: TokensConfig(emptyList()) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Global.kt b/src/main/kotlin/io/emeraldpay/dshackle/Global.kt index 8d5ee45f7..f156558c9 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Global.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Global.kt @@ -35,6 +35,7 @@ import io.emeraldpay.dshackle.upstream.ethereum.domain.TransactionId import io.emeraldpay.dshackle.upstream.ethereum.subscribe.json.TransactionIdSerializer import io.emeraldpay.dshackle.upstream.ton.TonMasterchainInfo import io.emeraldpay.dshackle.upstream.ton.TonMasterchainInfoDeserializer +import reactor.netty.resources.LoopResources import java.math.BigInteger import java.text.SimpleDateFormat import java.util.Locale @@ -46,6 +47,8 @@ class Global { companion object { + val wsLoops: LoopResources = LoopResources.create("reactor-ws") + val nullValue: ByteArray = "null".toByteArray() var metricsExtended = false @@ -75,6 +78,18 @@ class Global { } } + fun getErrorValueAsIs(errorResp: ByteArray): ByteArray? { + return runCatching { + Global.objectMapper.readTree(errorResp) + ?.get("error") + ?.let { + Global.objectMapper.writeValueAsBytes(it) + } + }.getOrElse { + null + } + } + private fun isSolana(chain: Chain): Boolean { return chain == Chain.SOLANA__MAINNET || chain == Chain.SOLANA__DEVNET || chain == Chain.SOLANA__TESTNET } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt b/src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt index 3dd03cb78..e144620b1 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/GrpcServer.kt @@ -25,10 +25,11 @@ import io.grpc.ServerCall import io.grpc.ServerCallHandler import io.grpc.ServerInterceptor import io.grpc.netty.NettyServerBuilder -import io.grpc.protobuf.services.ProtoReflectionService +import io.grpc.protobuf.services.ProtoReflectionServiceV1 import io.micrometer.core.instrument.Metrics -import io.micrometer.core.instrument.Tag import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics +import jakarta.annotation.PostConstruct +import jakarta.annotation.PreDestroy import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Value import org.springframework.scheduling.concurrent.CustomizableThreadFactory @@ -36,8 +37,6 @@ import org.springframework.stereotype.Service import java.net.InetSocketAddress import java.util.concurrent.Executors import java.util.concurrent.TimeUnit -import javax.annotation.PostConstruct -import javax.annotation.PreDestroy @Service open class GrpcServer( @@ -103,21 +102,16 @@ open class GrpcServer( serverBuilder.addService(it) } - serverBuilder.addService(ProtoReflectionService.newInstance()) + serverBuilder.addService(ProtoReflectionServiceV1.newInstance()) val pool = Executors.newFixedThreadPool(20, CustomizableThreadFactory("fixed-grpc-")) serverBuilder.executor( - if (mainConfig.monitoring.enableExtended) { - ExecutorServiceMetrics.monitor( - Metrics.globalRegistry, - pool, - "fixed-grpc-executor", - Tag.of("reactor_scheduler_id", "_"), - ) - } else { - pool - }, + ExecutorServiceMetrics.monitor( + Metrics.globalRegistry, + pool, + "fixed-grpc-executor", + ), ) val server = serverBuilder.build() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/ProxyStarter.kt b/src/main/kotlin/io/emeraldpay/dshackle/ProxyStarter.kt index baa81121a..84d367a33 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/ProxyStarter.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/ProxyStarter.kt @@ -17,32 +17,28 @@ package io.emeraldpay.dshackle import io.emeraldpay.dshackle.config.MainConfig -import io.emeraldpay.dshackle.monitoring.MonitoringSetup import io.emeraldpay.dshackle.monitoring.accesslog.AccessHandlerHttp import io.emeraldpay.dshackle.proxy.ProxyServer import io.emeraldpay.dshackle.proxy.ReadRpcJson import io.emeraldpay.dshackle.proxy.WriteRpcJson import io.emeraldpay.dshackle.rpc.NativeCall import io.emeraldpay.dshackle.rpc.NativeSubscribe +import jakarta.annotation.PostConstruct import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import javax.annotation.PostConstruct /** * Starts HTTP proxy endpoint, if configured */ @Service class ProxyStarter( - @Autowired private val mainConfig: MainConfig, - @Autowired private val readRpcJson: ReadRpcJson, - @Autowired private val writeRpcJson: WriteRpcJson, - @Autowired private val nativeCall: NativeCall, - @Autowired private val nativeSubscribe: NativeSubscribe, - @Autowired private val tlsSetup: TlsSetup, - @Autowired private val accessHandlerHttp: AccessHandlerHttp, - // depend on Monitoring, declared here just to ensure it's properly initialized before the Proxy - @Autowired private val monitoringSetup: MonitoringSetup, + private val mainConfig: MainConfig, + private val readRpcJson: ReadRpcJson, + private val writeRpcJson: WriteRpcJson, + private val nativeCall: NativeCall, + private val nativeSubscribe: NativeSubscribe, + private val tlsSetup: TlsSetup, + private val accessHandlerHttp: AccessHandlerHttp, ) { companion object { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/Starter.kt b/src/main/kotlin/io/emeraldpay/dshackle/Starter.kt index d63abfb57..f1d00c8a1 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/Starter.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/Starter.kt @@ -16,6 +16,10 @@ */ package io.emeraldpay.dshackle +import io.micrometer.context.ContextRegistry +import io.micrometer.context.integration.Slf4jThreadLocalAccessor +import io.micrometer.core.instrument.Metrics +import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics import io.netty.handler.ssl.OpenSsl import org.slf4j.LoggerFactory import org.springframework.boot.ResourceBanner @@ -23,6 +27,9 @@ import org.springframework.boot.SpringApplication import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.core.io.ClassPathResource import org.springframework.core.io.support.ResourcePropertySource +import reactor.core.Scannable +import reactor.core.Scannable.Attr +import reactor.core.publisher.Hooks import reactor.core.scheduler.Schedulers @SpringBootApplication(scanBasePackages = ["io.emeraldpay.dshackle"]) @@ -31,9 +38,22 @@ open class Starter private val log = LoggerFactory.getLogger(Starter::class.java) fun main(args: Array) { + ContextRegistry.getInstance().registerThreadLocalAccessor(Slf4jThreadLocalAccessor()) + Hooks.enableAutomaticContextPropagation() OpenSsl.ensureAvailability() - Schedulers.enableMetrics() + // add metrics for internal reactor schedulers + Schedulers.addExecutorServiceDecorator("key") { scheduler, execService -> + val schedulerName = Scannable.from(scheduler).scanOrDefault(Attr.NAME, scheduler.javaClass.name) + if (schedulerName.contains("single") || schedulerName.contains("parallel")) { + ExecutorServiceMetrics.monitor( + Metrics.globalRegistry, + execService, + schedulerName, + ) + } + execService + } HeapDumpCreator.init() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/TlsSetup.kt b/src/main/kotlin/io/emeraldpay/dshackle/TlsSetup.kt index 2a16d6cac..f3dd0ce7b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/TlsSetup.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/TlsSetup.kt @@ -24,13 +24,12 @@ import io.netty.handler.ssl.SslContextBuilder import io.netty.handler.ssl.SslProvider import org.apache.commons.lang3.StringUtils import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import java.security.cert.CertificateFactory @Service open class TlsSetup( - @Autowired val fileResolver: FileResolver, + val fileResolver: FileResolver, ) { companion object { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt index 6e4569a17..d88e6d40f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/CachesFactory.kt @@ -24,16 +24,15 @@ import io.lettuce.core.api.StatefulRedisConnection import io.lettuce.core.codec.ByteArrayCodec import io.lettuce.core.codec.RedisCodec import io.lettuce.core.codec.StringCodec +import jakarta.annotation.PostConstruct import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Repository import java.util.EnumMap -import javax.annotation.PostConstruct import kotlin.system.exitProcess @Repository open class CachesFactory( - @Autowired private val cacheConfig: CacheConfig, + private val cacheConfig: CacheConfig, ) { companion object { @@ -91,7 +90,13 @@ open class CachesFactory( private fun initCache(chain: Chain): Caches { val caches = Caches.newBuilder() - if (chain == Chain.ZIRCUIT__MAINNET || chain == Chain.ZIRCUIT__TESTNET) { + val disableCacheNetworks = setOf( + Chain.ZIRCUIT__MAINNET, + Chain.ZIRCUIT__TESTNET, + Chain.HYPERLIQUID__MAINNET, + Chain.HYPERLIQUID__TESTNET, + ) + if (chain in disableCacheNetworks) { caches.setCacheEnabled(false) } redis?.let { redis -> diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/CurrentBlockCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/CurrentBlockCache.kt index c1bf3b178..ecac4f591 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/CurrentBlockCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/CurrentBlockCache.kt @@ -5,7 +5,7 @@ import reactor.core.publisher.Mono import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicReference -class CurrentBlockCache : Reader { +class CurrentBlockCache : Reader { private val cache = AtomicReference(ConcurrentHashMap()) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/OnBlockRedisCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/OnBlockRedisCache.kt index 23e88b76c..7ecf370b6 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/OnBlockRedisCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/OnBlockRedisCache.kt @@ -29,7 +29,7 @@ import java.time.Instant import java.util.concurrent.TimeUnit import kotlin.math.min -abstract class OnBlockRedisCache( +abstract class OnBlockRedisCache( private val redis: RedisReactiveCommands, private val chain: Chain, private val valueType: ValueContainer.ValueType, diff --git a/src/main/kotlin/io/emeraldpay/dshackle/cache/OnTxRedisCache.kt b/src/main/kotlin/io/emeraldpay/dshackle/cache/OnTxRedisCache.kt index 8ed45cd0a..538305d31 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/cache/OnTxRedisCache.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/cache/OnTxRedisCache.kt @@ -29,7 +29,7 @@ import java.time.Instant import java.util.concurrent.TimeUnit import kotlin.math.min -abstract class OnTxRedisCache( +abstract class OnTxRedisCache( private val redis: RedisReactiveCommands, private val chain: Chain, private val valueType: CachesProto.ValueContainer.ValueType, diff --git a/src/main/kotlin/io/emeraldpay/dshackle/commons/DurableFlux.kt b/src/main/kotlin/io/emeraldpay/dshackle/commons/DurableFlux.kt index 3505ecee8..6d227164b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/commons/DurableFlux.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/commons/DurableFlux.kt @@ -13,7 +13,7 @@ import java.time.Duration /** * A flux holder that reconnects to it on failure taking into account a back off strategy */ -class DurableFlux( +class DurableFlux( private val provider: () -> Flux, private val errorBackOff: BackOff, private val log: Logger, @@ -55,7 +55,7 @@ class DurableFlux( } } - class Builder { + class Builder { private var provider: (() -> Flux)? = null @@ -63,7 +63,7 @@ class DurableFlux( protected var log: Logger = DurableFlux.defaultLog @Suppress("UNCHECKED_CAST") - fun using(provider: () -> Flux): Builder { + fun using(provider: () -> Flux): Builder { this.provider = provider as () -> Flux return this as Builder } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/commons/DynamicMergeFlux.kt b/src/main/kotlin/io/emeraldpay/dshackle/commons/DynamicMergeFlux.kt index f11432602..1ea611edb 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/commons/DynamicMergeFlux.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/commons/DynamicMergeFlux.kt @@ -6,7 +6,7 @@ import reactor.core.publisher.Sinks import reactor.core.scheduler.Scheduler import java.util.concurrent.ConcurrentHashMap -class DynamicMergeFlux(private val scheduler: Scheduler) { +class DynamicMergeFlux(private val scheduler: Scheduler) { private val merge = Sinks.many().multicast().directBestEffort() private val sources = ConcurrentHashMap() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/commons/SharedFluxHolder.kt b/src/main/kotlin/io/emeraldpay/dshackle/commons/SharedFluxHolder.kt index 1dd0f386f..59d155e25 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/commons/SharedFluxHolder.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/commons/SharedFluxHolder.kt @@ -11,7 +11,7 @@ import kotlin.concurrent.write * A flux holder that that creates it only if requested. Keeps it for the following calls, so all the following calls will * reuse it. Forgets as soon as it completes/cancelled, so it will be recreated again if needed. */ -class SharedFluxHolder( +class SharedFluxHolder( /** * Provider for the flux. Note that it can be called multiple times but only one is used at the same time. * I.e., if there is a few calls because of a thread-race only one is kept. @@ -66,7 +66,7 @@ class SharedFluxHolder( } } - data class Holder( + data class Holder( val flux: Flux, val id: Long, ) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfig.kt index 8c30950d5..21d0c69cb 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfig.kt @@ -33,6 +33,10 @@ class AuthConfig { val password: String, ) : ClientAuth() + class ClientBearerAuth( + val token: String, + ) : ClientAuth() + class ClientTlsAuth( var ca: String? = null, var certificate: String? = null, diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfigReader.kt index d886c0fd4..027c5a3d5 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/AuthConfigReader.kt @@ -40,6 +40,18 @@ class AuthConfigReader : YamlConfigReader() { } } + fun readClientBearerAuth(node: MappingNode?): AuthConfig.ClientBearerAuth? { + return getMapping(node, "bearer-auth")?.let { authNode -> + val token = getValueAsString(authNode, "token") + if (token != null) { + AuthConfig.ClientBearerAuth(token) + } else { + log.warn("Bearer auth is not fully configured, token is required") + null + } + } + } + fun readClientTls(node: MappingNode?): AuthConfig.ClientTlsAuth? { return getMapping(node, "tls")?.let { authNode -> val auth = AuthConfig.ClientTlsAuth() @@ -91,7 +103,6 @@ class AuthConfigReader : YamlConfigReader() { auth.clientCAs.add(it) } getList(clientNode, "cas")?.let { - println(it) it.value?.let { crt -> auth.clientCAs.addAll(crt.map { v -> v.value }) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt index a1d8e4b4d..0d579aa36 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfig.kt @@ -41,7 +41,6 @@ class MainConfig { var monitoring: MonitoringConfig = MonitoringConfig.default() var accessLogConfig: AccessLogConfig = AccessLogConfig.default() var health: HealthConfig = HealthConfig.default() - var signature: SignatureConfig? = null var compression: CompressionConfig = CompressionConfig.default() var chains: ChainsConfig = ChainsConfig.default() var authorization: AuthorizationConfig = AuthorizationConfig.default() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt index 90cdef656..5664eaf6a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/MainConfigReader.kt @@ -33,7 +33,6 @@ class MainConfigReader( private val monitoringConfigReader = MonitoringConfigReader() private val accessLogReader = AccessLogReader() private val healthConfigReader = HealthConfigReader() - private val signatureConfigReader = SignatureConfigReader(fileResolver) private val compressionConfigReader = CompressionConfigReader() private val chainsConfigReader = ChainsConfigReader(optionsReader) private val authorizationConfigReader = AuthorizationConfigReader() @@ -75,9 +74,6 @@ class MainConfigReader( healthConfigReader.read(input).let { config.health = it } - signatureConfigReader.read(input).let { - config.signature = it - } compressionConfigReader.read(input).let { config.compression = it } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/SignatureConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/SignatureConfig.kt deleted file mode 100644 index 3239382af..000000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/SignatureConfig.kt +++ /dev/null @@ -1,41 +0,0 @@ -package io.emeraldpay.dshackle.config - -import java.util.Locale - -class SignatureConfig { - - enum class Algorithm { - NIST_P256, - ; - - fun getCurveName(): String { - return if (this == NIST_P256) { - "secp256r1" - } else { - throw IllegalStateException() - } - } - } - - companion object { - fun algorithmOfString(algo: String): Algorithm { - val algorithm = when (algo.uppercase(Locale.getDefault())) { - "NIST_P256", "NIST-P256", "NISTP256", "SECP256R1" -> Algorithm.NIST_P256 - else -> throw IllegalArgumentException("Unknown algorithm or not allowed") - } - return algorithm - } - } - - /** - * Signature scheme that we should use - */ - var algorithm: Algorithm = Algorithm.NIST_P256 - - /** - * Should we generate signature on this instance if it's not already present - */ - var enabled: Boolean = false - - var privateKey: String? = null -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/SignatureConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/SignatureConfigReader.kt deleted file mode 100644 index 21fe1b6df..000000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/SignatureConfigReader.kt +++ /dev/null @@ -1,29 +0,0 @@ -package io.emeraldpay.dshackle.config - -import io.emeraldpay.dshackle.FileResolver -import io.emeraldpay.dshackle.foundation.YamlConfigReader -import org.yaml.snakeyaml.nodes.MappingNode - -class SignatureConfigReader(val fileResolver: FileResolver) : YamlConfigReader() { - override fun read(input: MappingNode?): SignatureConfig? { - return getMapping(input, "signed-response")?.let { node -> - val config = SignatureConfig() - getValueAsBool(node, "enabled")?.let { - config.enabled = it - } - if (config.enabled) { - getValueAsString(node, "algorithm")?.let { - config.algorithm = SignatureConfig.algorithmOfString(it) - } - getValueAsString(node, "private-key")?.let { - val key = fileResolver.resolve(it) - config.privateKey = key.absolutePath - } - } - if (config.enabled && config.privateKey == null) { - throw IllegalStateException("Path to a private key (`signature.private-key`) is required when Response signature is enabled.") - } - config - } - } -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfig.kt index d89138ac2..988070605 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/TokensConfig.kt @@ -38,7 +38,7 @@ class TokensConfig( id.isNullOrBlank() -> "id" blockchain == null -> "blockchain" name.isNullOrBlank() -> "name" - type == null -> type + type == null -> "type" address.isNullOrBlank() -> "address" blockchain != null && (blockchain!!.type == BlockchainType.ETHEREUM) && diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt index f113f8864..5c333c780 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt @@ -20,6 +20,8 @@ import com.fasterxml.jackson.annotation.JsonSubTypes import com.fasterxml.jackson.annotation.JsonTypeInfo import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.upstream.generic.connectors.GenericConnectorFactory.ConnectorMode +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType +import io.emeraldpay.dshackle.upstream.lowerbound.ManualLowerBoundType import java.net.URI import java.util.Arrays import java.util.Locale @@ -41,6 +43,8 @@ data class UpstreamsConfig( var methods: Methods? = null, var methodGroups: MethodGroups? = null, var role: UpstreamRole = UpstreamRole.PRIMARY, + var customHeaders: Map = emptyMap(), + var additionalSettings: AdditionalSettings? = null, ) { @Suppress("UNCHECKED_CAST") @@ -132,12 +136,14 @@ data class UpstreamsConfig( constructor(url: URI) : this(url, DEFAULT_MAX_CONNECTIONS, DEFAULT_QUEUE_SIZE) var basicAuth: AuthConfig.ClientBasicAuth? = null + var bearerAuth: AuthConfig.ClientBearerAuth? = null var tls: AuthConfig.ClientTlsAuth? = null } data class WsEndpoint(val url: URI) { var origin: URI? = null var basicAuth: AuthConfig.ClientBasicAuth? = null + var bearerAuth: AuthConfig.ClientBearerAuth? = null var frameSize: Int? = null var msgSize: Int? = null var connections: Int = 1 @@ -187,6 +193,15 @@ data class UpstreamsConfig( } } + data class AdditionalSettings( + val manualLowerBounds: Map, + ) + + data class ManualBoundSetting( + val type: ManualLowerBoundType, + val value: Long, + ) + data class Methods( val enabled: Set, val disabled: Set, diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt index 982a1b5ce..c51a493cd 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReader.kt @@ -17,12 +17,16 @@ package io.emeraldpay.dshackle.config import io.emeraldpay.dshackle.FileResolver +import io.emeraldpay.dshackle.config.UpstreamsConfig.ManualBoundSetting import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.foundation.ChainOptionsReader import io.emeraldpay.dshackle.foundation.YamlConfigReader +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType +import io.emeraldpay.dshackle.upstream.lowerbound.ManualLowerBoundType import org.apache.commons.lang3.StringUtils import org.slf4j.LoggerFactory import org.yaml.snakeyaml.nodes.MappingNode +import org.yaml.snakeyaml.nodes.Tag import java.io.InputStream import java.net.URI import java.util.Locale @@ -143,6 +147,7 @@ class UpstreamsConfigReader( getValueAsString(node, "url")?.let { url -> val http = UpstreamsConfig.HttpEndpoint(URI(url), DEFAULT_MAX_CONNECTIONS, DEFAULT_QUEUE_SIZE) http.basicAuth = authConfigReader.readClientBasicAuth(node) + http.bearerAuth = readBearerAuth(node, http.basicAuth, url) http.tls = authConfigReader.readClientTls(node) connection.esplora = http } @@ -169,6 +174,19 @@ class UpstreamsConfigReader( return connection } + private fun readBearerAuth( + node: MappingNode?, + basicAuth: AuthConfig.ClientBasicAuth?, + url: String, + ): AuthConfig.ClientBearerAuth? { + val bearerAuth = authConfigReader.readClientBearerAuth(node) + if (bearerAuth != null && basicAuth != null) { + log.warn("Both basic-auth and bearer-auth are configured for $url, basic-auth is used") + return null + } + return bearerAuth + } + private fun readRpcConfig(connConfigNode: MappingNode): UpstreamsConfig.HttpEndpoint? { return getMapping(connConfigNode, "rpc")?.let { node -> val maxConnections = getValueAsInt(node, "max-connections") ?: DEFAULT_MAX_CONNECTIONS @@ -177,6 +195,7 @@ class UpstreamsConfigReader( getValueAsString(node, "url")?.let { url -> val http = UpstreamsConfig.HttpEndpoint(URI(url), maxConnections, queueSize) http.basicAuth = authConfigReader.readClientBasicAuth(node) + http.bearerAuth = readBearerAuth(node, http.basicAuth, url) http.tls = authConfigReader.readClientTls(node) http } @@ -220,6 +239,7 @@ class UpstreamsConfigReader( ws.origin = URI(origin) } ws.basicAuth = authConfigReader.readClientBasicAuth(node) + ws.bearerAuth = readBearerAuth(node, ws.basicAuth, url) getValueAsBytes(node, "frameSize")?.let { if (it < 65_535) { @@ -287,6 +307,7 @@ class UpstreamsConfigReader( upstream.options = optionsReader.read(upNode) upstream.methods = tryReadMethods(upNode) upstream.methodGroups = tryReadMethodGroups(upNode) + upstream.additionalSettings = readAdditionalSettings(upNode) getValueAsBool(upNode, "enabled")?.let { upstream.isEnabled = it } @@ -300,6 +321,47 @@ class UpstreamsConfigReader( } } } + if (hasAny(upNode, "custom-headers")) { + getMapping(upNode, "custom-headers")?.let { headers -> + val headersMap = headers.value + .map { it.keyNode.valueAsString() to it.valueNode.valueAsString() } + .filter { StringUtils.isNotBlank(it.first) && StringUtils.isNotBlank(it.second) } + .associate { it.first!!.trim() to it.second!!.trim() } + upstream.customHeaders = headersMap + } + } + } + + private fun readAdditionalSettings(upNode: MappingNode): UpstreamsConfig.AdditionalSettings? { + return getMapping(upNode, "additional-settings") + ?.let { settings -> + val manualLowerBounds = getMapping(settings, "manual-lower-bounds") + ?.let { bounds -> + bounds.value.asSequence() + .filter { + StringUtils.isNotBlank(it.keyNode.valueAsString()) && it.valueNode.tag == Tag.MAP + } + .map { + val setting = readManualBoundSetting(it.valueNode as MappingNode) + + if (setting != null) { + LowerBoundType.byName(it.keyNode.valueAsString()!!) to setting + } else { + null + } + } + .filterNotNull() + .associate { it.first to it.second } + } ?: emptyMap() + UpstreamsConfig.AdditionalSettings(manualLowerBounds) + } + } + + private fun readManualBoundSetting(node: MappingNode): ManualBoundSetting? { + val type = getValueAsString(node, "type") + ?.let { ManualLowerBoundType.byName(it) } ?: return null + val value = getValueAsLong(node, "value") ?: return null + return ManualBoundSetting(type, value) } private fun readUpstreamGrpc( diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/context/MultistreamsConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/context/MultistreamsConfig.kt index 7e22c9288..37780b5de 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/context/MultistreamsConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/context/MultistreamsConfig.kt @@ -10,7 +10,6 @@ import io.emeraldpay.dshackle.upstream.generic.ChainSpecificRegistry import io.emeraldpay.dshackle.upstream.generic.GenericMultistream import org.springframework.beans.factory.annotation.Qualifier import org.springframework.beans.factory.config.ConfigurableListableBeanFactory -import org.springframework.cloud.sleuth.Tracer import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import reactor.core.scheduler.Scheduler @@ -26,7 +25,6 @@ open class MultistreamsConfig(val beanFactory: ConfigurableListableBeanFactory) headScheduler: Scheduler, @Qualifier("subScheduler") subScheduler: Scheduler, - tracer: Tracer, multistreamEventsScheduler: Scheduler, ): List { return Chain.entries @@ -40,7 +38,6 @@ open class MultistreamsConfig(val beanFactory: ConfigurableListableBeanFactory) cachesFactory, headScheduler, subScheduler, - tracer, multistreamEventsScheduler, ) } @@ -52,7 +49,6 @@ open class MultistreamsConfig(val beanFactory: ConfigurableListableBeanFactory) cachesFactory: CachesFactory, headScheduler: Scheduler, subScheduler: Scheduler, - tracer: Tracer, multistreamEventsScheduler: Scheduler, ): Multistream { val name = "multi-$chain" @@ -65,7 +61,7 @@ open class MultistreamsConfig(val beanFactory: ConfigurableListableBeanFactory) CopyOnWriteArrayList(), caches, headScheduler, - cs.makeCachingReaderBuilder(tracer), + cs.makeCachingReaderBuilder(), cs::localReaderBuilder, cs.subscriptionBuilder(subScheduler), ).also { register(it, name) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/context/SchedulersConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/context/SchedulersConfig.kt index 4ad9a48e1..4c5ca2bf2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/context/SchedulersConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/context/SchedulersConfig.kt @@ -1,12 +1,11 @@ package io.emeraldpay.dshackle.config.context -import io.emeraldpay.dshackle.config.MonitoringConfig import io.micrometer.core.instrument.Metrics -import io.micrometer.core.instrument.Tag import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics import org.slf4j.LoggerFactory import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.DependsOn import org.springframework.scheduling.concurrent.CustomizableThreadFactory import reactor.core.scheduler.Scheduler import reactor.core.scheduler.Schedulers @@ -17,6 +16,7 @@ import java.util.concurrent.ThreadPoolExecutor import java.util.concurrent.TimeUnit @Configuration +@DependsOn("monitoringSetup") open class SchedulersConfig { companion object { @@ -32,55 +32,65 @@ open class SchedulersConfig { } @Bean - open fun rpcScheduler(monitoringConfig: MonitoringConfig): Scheduler { - return makeScheduler("blockchain-rpc-scheduler", 20, monitoringConfig) + open fun rpcScheduler(): Scheduler { + return makeScheduler("blockchain-rpc-scheduler", 20) } @Bean - open fun headScheduler(monitoringConfig: MonitoringConfig): Scheduler { - return makeScheduler("head-scheduler", 4, monitoringConfig) + open fun headScheduler(): Scheduler { + return makeScheduler("head-scheduler", 4) } @Bean - open fun subScheduler(monitoringConfig: MonitoringConfig): Scheduler { - return makeScheduler("sub-scheduler", 4, monitoringConfig) + open fun subScheduler(): Scheduler { + return makeScheduler("sub-scheduler", 4) } @Bean - open fun multistreamEventsScheduler(monitoringConfig: MonitoringConfig): Scheduler { - return makeScheduler("events-scheduler", 4, monitoringConfig) + open fun multistreamEventsScheduler(): Scheduler { + return makeScheduler("events-scheduler", 4) } @Bean - open fun wsConnectionResubscribeScheduler(monitoringConfig: MonitoringConfig): Scheduler { - return makeScheduler("ws-connection-resubscribe-scheduler", 2, monitoringConfig) + open fun wsConnectionResubscribeScheduler(): Scheduler { + return makeScheduler("ws-connection-resubscribe-scheduler", 2) } @Bean - open fun wsScheduler(monitoringConfig: MonitoringConfig): Scheduler { - return makeScheduler("ws-scheduler", 4, monitoringConfig) + open fun wsScheduler(): Scheduler { + return makeScheduler("ws-scheduler", 4) } @Bean - open fun headLivenessScheduler(monitoringConfig: MonitoringConfig): Scheduler { - return makeScheduler("head-liveness-scheduler", 4, monitoringConfig) + open fun headLivenessScheduler(): Scheduler { + return makeScheduler("head-liveness-scheduler", 4) } @Bean - open fun grpcChannelExecutor(monitoringConfig: MonitoringConfig): Executor { - return makePool("grpc-client-channel", 10, monitoringConfig) + open fun grpcChannelExecutor(): Executor { + return makePool("grpc-client-channel", 10) } @Bean - open fun authScheduler(monitoringConfig: MonitoringConfig): Scheduler { - return makeScheduler("auth-scheduler", 4, monitoringConfig) + open fun authScheduler(): Scheduler { + return makeScheduler("auth-scheduler", 4) } - private fun makeScheduler(name: String, size: Int, monitoringConfig: MonitoringConfig): Scheduler { - return Schedulers.fromExecutorService(makePool(name, size, monitoringConfig)) + @Bean + open fun httpScheduler(): Scheduler { + return makeScheduler("http-scheduler", 30) + } + + @Bean + open fun eventsScheduler(): Scheduler { + return makeScheduler("ws-events-scheduler", 30) } - private fun makePool(name: String, size: Int, monitoringConfig: MonitoringConfig): ExecutorService { + private fun makeScheduler(name: String, size: Int): Scheduler { + return Schedulers.fromExecutorService(makePool(name, size)) + } + + private fun makePool(name: String, size: Int): ExecutorService { val cachedPool = ThreadPoolExecutor( size, size * threadsMultiplier, @@ -90,15 +100,10 @@ open class SchedulersConfig { CustomizableThreadFactory("$name-"), ) - return if (monitoringConfig.enableExtended) { - ExecutorServiceMetrics.monitor( - Metrics.globalRegistry, - cachedPool, - name, - Tag.of("reactor_scheduler_id", "_"), - ) - } else { - cachedPool - } + return ExecutorServiceMetrics.monitor( + Metrics.globalRegistry, + cachedPool, + name, + ) } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/context/TraceConfiguration.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/context/TraceConfiguration.kt index 5c06db891..a950cf46c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/context/TraceConfiguration.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/context/TraceConfiguration.kt @@ -1,6 +1,9 @@ package io.emeraldpay.dshackle.config.context +import brave.Tracing +import brave.context.slf4j.MDCScopeDecorator import brave.grpc.GrpcTracing +import brave.propagation.CurrentTraceContext import brave.rpc.RpcTracing import brave.sampler.Sampler import io.grpc.ServerInterceptor @@ -9,14 +12,28 @@ import org.springframework.context.annotation.Configuration @Configuration open class TraceConfiguration { + @Bean + open fun defaultSampler(): Sampler = Sampler.ALWAYS_SAMPLE @Bean - open fun grpcTracing(rpcTracing: RpcTracing): GrpcTracing = GrpcTracing.create(rpcTracing) + open fun braveTracing(defaultSampler: Sampler): Tracing { + val currentTraceContext = + CurrentTraceContext.Default.newBuilder() + .addScopeDecorator(MDCScopeDecorator.newBuilder().build()) + .build() + + return Tracing.newBuilder() + .localServiceName("grpc-server") + .currentTraceContext(currentTraceContext) + .build() + } @Bean - open fun grpcServerBraveInterceptor(grpcTracing: GrpcTracing): ServerInterceptor = - grpcTracing.newServerInterceptor() + open fun rpcTracing(tracing: Tracing): RpcTracing = RpcTracing.create(tracing) @Bean - open fun defaultSampler(): Sampler = Sampler.ALWAYS_SAMPLE + open fun grpcTracing(rpcTracing: RpcTracing): GrpcTracing = GrpcTracing.create(rpcTracing) + + @Bean + open fun grpcServerBraveInterceptor(grpcTracing: GrpcTracing): ServerInterceptor = grpcTracing.newServerInterceptor() } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/hot/AutoReloadbleConfig.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/hot/AutoReloadbleConfig.kt index be60bc7bf..73832205b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/hot/AutoReloadbleConfig.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/hot/AutoReloadbleConfig.kt @@ -1,8 +1,10 @@ package io.emeraldpay.dshackle.config.hot import io.emeraldpay.dshackle.Global +import org.apache.hc.client5.http.classic.methods.HttpGet +import org.apache.hc.client5.http.impl.classic.HttpClients +import org.apache.hc.core5.http.io.entity.EntityUtils import org.slf4j.LoggerFactory -import org.springframework.web.client.RestTemplate import reactor.core.publisher.Flux import java.time.Duration import java.util.concurrent.atomic.AtomicReference @@ -17,12 +19,13 @@ class AutoReloadbleConfig( private val log = LoggerFactory.getLogger(AutoReloadbleConfig::class.java) } - private val restTemplate = RestTemplate() + private val httpClient = HttpClients.createDefault() private val instance = AtomicReference() fun reload() { try { - val response = restTemplate.getForObject(configUrl, String::class.java) + val getReq = HttpGet(configUrl) + val response = httpClient.execute(getReq) { resp -> EntityUtils.toString(resp.entity) } if (response != null) { instance.set(parseConfig(response)) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/hot/CompatibleVersionsRules.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/hot/CompatibleVersionsRules.kt index 1d2445675..3f5683c72 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/hot/CompatibleVersionsRules.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/config/hot/CompatibleVersionsRules.kt @@ -5,16 +5,18 @@ import com.fasterxml.jackson.annotation.JsonProperty @JsonIgnoreProperties(ignoreUnknown = true) data class CompatibleVersionsRules( - @JsonProperty("rules") + @param:JsonProperty("rules") val rules: List, ) @JsonIgnoreProperties(ignoreUnknown = true) data class CompatibleVersionsRule( - @JsonProperty("client") + @param:JsonProperty("client") val client: String, - @JsonProperty("blacklist") + @param:JsonProperty("networks") + val networks: List?, + @param:JsonProperty("blacklist") val blacklist: List?, - @JsonProperty("whitelist") + @param:JsonProperty("whitelist") val whitelist: List?, ) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/spans/Export.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/spans/Export.kt deleted file mode 100644 index 204cec26e..000000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/spans/Export.kt +++ /dev/null @@ -1,37 +0,0 @@ -package io.emeraldpay.dshackle.config.spans - -import brave.handler.MutableSpan -import io.emeraldpay.dshackle.commons.SPAN_ERROR -import io.emeraldpay.dshackle.commons.SPAN_NO_RESPONSE_MESSAGE -import org.springframework.beans.factory.annotation.Value -import org.springframework.stereotype.Component -import java.util.concurrent.TimeUnit.MICROSECONDS -import java.util.concurrent.TimeUnit.MILLISECONDS - -interface SpanExportable { - fun isExportable(span: MutableSpan): Boolean -} - -@Component -class ErrorSpanExportable : SpanExportable { - override fun isExportable(span: MutableSpan): Boolean = span.tags().containsKey(SPAN_ERROR) -} - -@Component -class NoResponseSpanExportable : SpanExportable { - override fun isExportable(span: MutableSpan): Boolean = span.tags().containsKey(SPAN_NO_RESPONSE_MESSAGE) -} - -@Component -class LongResponseSpanExportable( - @Value("\${spans.collect.long-span-threshold}") - private val longSpanThreshold: Long? = null, -) : SpanExportable { - - override fun isExportable(span: MutableSpan): Boolean { - return MILLISECONDS.convert( - span.finishTimestamp() - span.startTimestamp(), - MICROSECONDS, - ) >= longSpanThreshold!! - } -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/spans/ProviderSpanHandler.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/spans/ProviderSpanHandler.kt deleted file mode 100644 index 2f0c3dd0a..000000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/spans/ProviderSpanHandler.kt +++ /dev/null @@ -1,84 +0,0 @@ -package io.emeraldpay.dshackle.config.spans - -import brave.handler.MutableSpan -import brave.handler.SpanHandler -import brave.propagation.TraceContext -import com.github.benmanes.caffeine.cache.Caffeine -import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty -import org.springframework.stereotype.Component -import java.time.Duration - -@Component -@ConditionalOnProperty(value = ["spring.zipkin.enabled"], havingValue = "true") -class ProviderSpanHandler( - private val spanExportableList: List, - private val zipkinSpanHandler: SpanHandler, -) : SpanHandler() { - private val log = LoggerFactory.getLogger(ProviderSpanHandler::class.java) - - private val spans = Caffeine - .newBuilder() - .expireAfterWrite(Duration.ofMinutes(2)) - .build>() - - override fun end(context: TraceContext, span: MutableSpan, cause: Cause): Boolean { - if (span.traceId().length > 20 && cause == Cause.FINISHED) { - val key = span.parentId() ?: span.id() - val spanList = spans.asMap().computeIfAbsent(key) { mutableListOf() } - spanList.add(span) - } - return false - } - - fun sendSpans(traceContext: TraceContext) { - try { - sendSpansInternal(traceContext) - } catch (e: Exception) { - log.warn("Error while handling and sending spans - ${e.message}") - } - } - - private fun sendSpansInternal(traceContext: TraceContext) { - val spansInfo = SpansInfo() - - processSpans(traceContext.spanIdString(), spansInfo) - - spansInfo.spans - .map { it.parentId() } - .forEach { - if (it != null) { - spans.invalidate(it) - } - } - - if (spansInfo.exportable) { - spansInfo.spans.forEach { - zipkinSpanHandler.end(traceContext, it, Cause.FINISHED) - } - } - } - - private fun processSpans(spanId: String, spansInfo: SpansInfo) { - val currentSpans: List? = spans.getIfPresent(spanId) - - currentSpans?.forEach { - processSpanInfo(it, spansInfo) - if (spanId != it.id()) { - processSpans(it.id(), spansInfo) - } - } - } - - private fun processSpanInfo(span: MutableSpan, spansInfo: SpansInfo) { - spansInfo.spans.add(span) - if (spanExportableList.any { it.isExportable(span) }) { - spansInfo.exportable = true - } - } - - private data class SpansInfo( - var exportable: Boolean = false, - val spans: MutableList = mutableListOf(), - ) -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/config/spans/ZipkinSSLCustomizer.kt b/src/main/kotlin/io/emeraldpay/dshackle/config/spans/ZipkinSSLCustomizer.kt deleted file mode 100644 index ff908665b..000000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/config/spans/ZipkinSSLCustomizer.kt +++ /dev/null @@ -1,90 +0,0 @@ -package io.emeraldpay.dshackle.config.spans - -import io.emeraldpay.dshackle.config.MainConfig -import org.apache.http.conn.ssl.SSLConnectionSocketFactory -import org.apache.http.impl.client.HttpClients -import org.bouncycastle.openssl.PEMParser -import org.springframework.cloud.sleuth.zipkin2.ZipkinRestTemplateCustomizer -import org.springframework.http.HttpRequest -import org.springframework.http.client.ClientHttpRequestExecution -import org.springframework.http.client.ClientHttpRequestInterceptor -import org.springframework.http.client.ClientHttpResponse -import org.springframework.http.client.HttpComponentsClientHttpRequestFactory -import org.springframework.stereotype.Component -import org.springframework.web.client.RestTemplate -import java.io.ByteArrayOutputStream -import java.io.StringReader -import java.nio.file.Files -import java.nio.file.Paths -import java.security.KeyFactory -import java.security.KeyStore -import java.security.SecureRandom -import java.security.cert.CertificateFactory -import java.security.cert.X509Certificate -import java.security.spec.PKCS8EncodedKeySpec -import java.util.zip.GZIPOutputStream -import javax.net.ssl.KeyManagerFactory -import javax.net.ssl.SSLContext - -@Component -class ZipkinSSLCustomizer(private val mainConfig: MainConfig) : ZipkinRestTemplateCustomizer { - override fun customizeTemplate(restTemplate: RestTemplate): RestTemplate { - return if (mainConfig.tls?.enabled == true) { - setupSSL(mainConfig.tls!!.certificate!!, mainConfig.tls!!.key!!) - } else { - restTemplate - }.apply { - interceptors.add(0, GZipInterceptor()) - } - } - - private fun setupSSL(certPath: String, privateKeyPath: String): RestTemplate { - // Load the certificate - val certificateReader = StringReader(Files.readString(Paths.get(certPath))) - val pemObject = PEMParser(certificateReader).readPemObject() - val certificate = CertificateFactory.getInstance("X.509").generateCertificate(pemObject.content.inputStream()) as X509Certificate - // Load the private key - val privateKeyReader = StringReader(Files.readString(Paths.get(privateKeyPath))) - val pemKeyPair = PEMParser(privateKeyReader).readPemObject() - val privKeySpec = PKCS8EncodedKeySpec(pemKeyPair.content) - val privateKey = KeyFactory.getInstance("RSA").generatePrivate(privKeySpec) - - // Create the key store - val keyStore = KeyStore.getInstance(KeyStore.getDefaultType()).apply { - load(null, null) - setCertificateEntry("certificate", certificate) - setKeyEntry("private-key", privateKey, null, arrayOf(certificate)) - } - - // Create the SSL context - val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()).apply { - init(keyStore, null) - } - val sslContext = SSLContext.getInstance("TLS").apply { - init(keyManagerFactory.keyManagers, null, SecureRandom()) - } - - // Create the HTTP client - val socketFactory = SSLConnectionSocketFactory(sslContext) - val httpClient = HttpClients.custom().setSSLSocketFactory(socketFactory).build() - - // Create the request factory - val requestFactory = HttpComponentsClientHttpRequestFactory(httpClient) - - // Create the RestTemplate - return RestTemplate(requestFactory) - } - - private class GZipInterceptor : ClientHttpRequestInterceptor { - override fun intercept( - request: HttpRequest, - body: ByteArray, - execution: ClientHttpRequestExecution, - ): ClientHttpResponse { - request.headers.add("Content-Encoding", "gzip") - val gzipped = ByteArrayOutputStream() - GZIPOutputStream(gzipped).use { compressor -> compressor.write(body) } - return execution.execute(request, gzipped.toByteArray()) - } - } -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt index 66632d3d7..59720c100 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/HealthCheckSetup.kt @@ -18,19 +18,18 @@ package io.emeraldpay.dshackle.monitoring import com.sun.net.httpserver.HttpServer import io.emeraldpay.dshackle.config.HealthConfig import io.emeraldpay.dshackle.upstream.MultistreamHolder +import jakarta.annotation.PostConstruct +import jakarta.annotation.PreDestroy +import org.apache.hc.core5.http.HttpStatus import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.http.HttpStatus import org.springframework.stereotype.Service import java.io.IOException import java.net.InetSocketAddress -import javax.annotation.PostConstruct -import javax.annotation.PreDestroy @Service class HealthCheckSetup( - @Autowired private val healthConfig: HealthConfig, - @Autowired private val multistreamHolder: MultistreamHolder, + private val healthConfig: HealthConfig, + private val multistreamHolder: MultistreamHolder, ) { companion object { @@ -63,9 +62,9 @@ class HealthCheckSetup( } val ok = response.ok val data = response.details.joinToString("\n") - val code = if (ok) HttpStatus.OK else HttpStatus.SERVICE_UNAVAILABLE - log.debug("Health check response: ${code.value()} ${code.reasonPhrase} $data") - httpExchange.sendResponseHeaders(code.value(), data.toByteArray().size.toLong()) + val code = if (ok) HttpStatus.SC_OK else HttpStatus.SC_SERVICE_UNAVAILABLE + log.debug("Health check response: $code $data") + httpExchange.sendResponseHeaders(code, data.toByteArray().size.toLong()) httpExchange.responseBody.use { os -> os.write(data.toByteArray()) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/MonitoringSetup.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/MonitoringSetup.kt index 878158e22..c7b578a62 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/MonitoringSetup.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/MonitoringSetup.kt @@ -15,7 +15,6 @@ */ package io.emeraldpay.dshackle.monitoring -import com.sun.net.httpserver.HttpServer import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.config.MonitoringConfig import io.micrometer.core.instrument.Meter @@ -26,46 +25,22 @@ import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics import io.micrometer.core.instrument.binder.system.ProcessorMetrics import io.micrometer.core.instrument.config.MeterFilter -import io.micrometer.prometheus.PrometheusConfig -import io.micrometer.prometheus.PrometheusMeterRegistry +import io.micrometer.prometheusmetrics.PrometheusConfig +import io.micrometer.prometheusmetrics.PrometheusMeterRegistry +import io.prometheus.metrics.exporter.httpserver.HTTPServer +import jakarta.annotation.PostConstruct import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import java.io.IOException -import java.net.InetAddress -import java.net.InetSocketAddress -import java.net.ServerSocket -import java.nio.charset.StandardCharsets -import java.util.concurrent.Executors -import java.util.concurrent.ThreadFactory -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicInteger -import java.util.zip.GZIPOutputStream -import javax.annotation.PostConstruct @Service class MonitoringSetup( - @Autowired private val monitoringConfig: MonitoringConfig, + private val monitoringConfig: MonitoringConfig, ) { companion object { private val log = LoggerFactory.getLogger(MonitoringSetup::class.java) } - private fun isTcpPortAvailable(host: String, port: Int): Boolean { - try { - ServerSocket().use { serverSocket -> - // setReuseAddress(false) is required only on macOS, - // otherwise the code will not work correctly on that platform - serverSocket.reuseAddress = false - serverSocket.bind(InetSocketAddress(InetAddress.getByName(host), port), 1) - return true - } - } catch (ex: java.lang.Exception) { - return false - } - } - @PostConstruct fun setup() { val prometheusRegistry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT) @@ -94,77 +69,13 @@ class MonitoringSetup( } if (monitoringConfig.prometheus.enabled) { - // use standard JVM server with a single thread blocking processing - // prometheus is a single thread periodic call, no reason to setup anything complex - Thread { - var started = false - val scrapeThreadCounter = AtomicInteger(0) - val scrapeExecutor = Executors.newFixedThreadPool( - 4, - ThreadFactory { runnable -> - Thread(runnable, "prometheus-scrape-${scrapeThreadCounter.incrementAndGet()}").apply { - isDaemon = true - } - }, - ) - while (true) { - if (isTcpPortAvailable(monitoringConfig.prometheus.host, monitoringConfig.prometheus.port)) { - started = true - try { - log.info("Run Prometheus metrics on ${monitoringConfig.prometheus.host}:${monitoringConfig.prometheus.port}${monitoringConfig.prometheus.path}") - val server = HttpServer.create( - InetSocketAddress( - monitoringConfig.prometheus.host, - monitoringConfig.prometheus.port, - ), - 0, - ) - server.executor = scrapeExecutor - server.createContext(monitoringConfig.prometheus.path) { httpExchange -> - val startTime = System.nanoTime() - try { - val response = prometheusRegistry.scrape() - val responseBytes = response.toByteArray(StandardCharsets.UTF_8) - httpExchange.responseHeaders.add("Content-Encoding", "gzip") - httpExchange.responseHeaders.add("Content-Type", "text/plain; version=0.0.4; charset=utf-8") - httpExchange.sendResponseHeaders(200, 0) - httpExchange.responseBody.use { os -> - GZIPOutputStream(os).use { gzos -> - gzos.write(responseBytes) - } - } - val durationMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime) - if (durationMs > 5000) { - log.warn("Prometheus scrape served in {} ms", durationMs) - } else { - log.debug("Prometheus scrape served in {} ms", durationMs) - } - } catch (e: Exception) { - log.error("Failed to serve Prometheus metrics", e) - val messageBytes = "internal error".toByteArray(StandardCharsets.UTF_8) - runCatching { - httpExchange.responseHeaders.add("Content-Type", "text/plain; charset=utf-8") - httpExchange.sendResponseHeaders(500, messageBytes.size.toLong()) - httpExchange.responseBody.use { os -> - os.write(messageBytes) - } - } - } finally { - httpExchange.close() - } - } - Thread(server::start).start() - } catch (e: IOException) { - log.error("Failed to start Prometheus Server", e) - } - } else { - if (!started) { - log.error("Can't start prometheus metrics on ${monitoringConfig.prometheus.host}:${monitoringConfig.prometheus.port}${monitoringConfig.prometheus.path}") - } - Thread.sleep(1000) - } - } - }.start() + HTTPServer + .builder() + .hostname(monitoringConfig.prometheus.host) + .port(monitoringConfig.prometheus.port) + .registry(prometheusRegistry.prometheusRegistry) + .metricsHandlerPath(monitoringConfig.prometheus.path) + .buildAndStart() } } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerGrpc.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerGrpc.kt index 3c6a75509..efb54799b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerGrpc.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerGrpc.kt @@ -23,13 +23,12 @@ import io.grpc.ServerCall import io.grpc.ServerCallHandler import io.grpc.ServerInterceptor import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import java.time.Instant @Service class AccessHandlerGrpc( - @Autowired private val accessLogWriter: AccessLogWriter, + private val accessLogWriter: AccessLogWriter, ) : ServerInterceptor { companion object { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerHttp.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerHttp.kt index eb805bfce..8294b569e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerHttp.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessHandlerHttp.kt @@ -5,7 +5,6 @@ import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.config.MainConfig import io.emeraldpay.dshackle.rpc.NativeCall import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import reactor.netty.http.server.HttpServerRequest import reactor.netty.http.websocket.WebsocketInbound @@ -20,8 +19,8 @@ import kotlin.concurrent.withLock */ @Service class AccessHandlerHttp( - @Autowired private val mainConfig: MainConfig, - @Autowired accessLogWriter: AccessLogWriter, + mainConfig: MainConfig, + accessLogWriter: AccessLogWriter, ) { companion object { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessLogWriter.kt b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessLogWriter.kt index bc33d3a68..798c6a164 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessLogWriter.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/monitoring/accesslog/AccessLogWriter.kt @@ -17,6 +17,7 @@ package io.emeraldpay.dshackle.monitoring.accesslog import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.config.MainConfig +import jakarta.annotation.PostConstruct import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Repository @@ -28,7 +29,6 @@ import java.time.Instant import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.Executors import java.util.concurrent.TimeUnit -import javax.annotation.PostConstruct @Repository class AccessLogWriter( diff --git a/src/main/kotlin/io/emeraldpay/dshackle/proxy/HttpHandler.kt b/src/main/kotlin/io/emeraldpay/dshackle/proxy/HttpHandler.kt index def791dd5..cd06499e6 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/proxy/HttpHandler.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/proxy/HttpHandler.kt @@ -24,9 +24,9 @@ import io.emeraldpay.dshackle.upstream.ChainResponse import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.netty.buffer.ByteBuf import io.netty.buffer.Unpooled +import org.apache.hc.core5.http.HttpHeaders import org.reactivestreams.Publisher import org.slf4j.LoggerFactory -import org.springframework.http.HttpHeaders import reactor.core.publisher.Flux import reactor.core.publisher.Mono import reactor.netty.http.server.HttpServerRequest diff --git a/src/main/kotlin/io/emeraldpay/dshackle/proxy/ProxyServer.kt b/src/main/kotlin/io/emeraldpay/dshackle/proxy/ProxyServer.kt index 180f69c4d..4e7fdfbd9 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/proxy/ProxyServer.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/proxy/ProxyServer.kt @@ -28,7 +28,8 @@ import io.micrometer.core.instrument.Metrics import io.micrometer.core.instrument.Timer import io.netty.channel.ChannelHandler import io.netty.channel.ChannelHandlerContext -import io.netty.channel.nio.NioEventLoopGroup +import io.netty.channel.MultiThreadIoEventLoopGroup +import io.netty.channel.nio.NioIoHandler import org.slf4j.LoggerFactory import reactor.netty.http.server.HttpServer import reactor.netty.http.server.HttpServerRoutes @@ -108,7 +109,7 @@ class ProxyServer( serverBuilder .route(this::setupRoutes) - .runOn(NioEventLoopGroup()) + .runOn(MultiThreadIoEventLoopGroup(NioIoHandler.newFactory())) .bindNow() } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/JsonBroadcastQuorum.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/JsonBroadcastQuorum.kt new file mode 100644 index 000000000..31a50fc1e --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/JsonBroadcastQuorum.kt @@ -0,0 +1,54 @@ +package io.emeraldpay.dshackle.quorum + +import com.fasterxml.jackson.databind.JsonNode +import io.emeraldpay.dshackle.upstream.ChainResponse +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.signature.ResponseSigner + +open class JsonBroadcastQuorum() : CallQuorum, ValueAwareQuorum(JsonNode::class.java) { + + private var result: ChainResponse? = null + private var txResponse: JsonNode? = null + private var sig: ResponseSigner.Signature? = null + + override fun isResolved(): Boolean { + return result != null + } + + override fun isFailed(): Boolean { + return result == null + } + + override fun getResponse(): ChainResponse? { + return result + } + + override fun getSignature(): ResponseSigner.Signature? { + return sig + } + + override fun recordValue( + response: ChainResponse, + responseValue: JsonNode?, + signature: ResponseSigner.Signature?, + upstream: Upstream, + ) { + if (txResponse == null && responseValue != null) { + txResponse = responseValue + sig = signature + result = response + } + } + + override fun recordError( + errorMessage: String?, + signature: ResponseSigner.Signature?, + upstream: Upstream, + ) { + resolvers.add(upstream) + } + + override fun toString(): String { + return "Quorum: Json Broadcast to upstreams" + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumRequestReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumRequestReader.kt index d64a9f1c8..54bca259c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumRequestReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/quorum/QuorumRequestReader.kt @@ -16,12 +16,9 @@ package io.emeraldpay.dshackle.quorum import io.emeraldpay.dshackle.Global -import io.emeraldpay.dshackle.commons.API_READER -import io.emeraldpay.dshackle.commons.SPAN_NO_RESPONSE_MESSAGE import io.emeraldpay.dshackle.commons.SPAN_REQUEST_API_TYPE import io.emeraldpay.dshackle.commons.SPAN_REQUEST_UPSTREAM_ID import io.emeraldpay.dshackle.reader.RequestReader -import io.emeraldpay.dshackle.reader.SpannedReader import io.emeraldpay.dshackle.upstream.ApiSource import io.emeraldpay.dshackle.upstream.ChainCallUpstreamException import io.emeraldpay.dshackle.upstream.ChainException @@ -30,9 +27,9 @@ import io.emeraldpay.dshackle.upstream.ChainResponse import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.error.UpstreamErrorHandler import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException +import io.emeraldpay.dshackle.upstream.signature.DisabledSigner import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import org.slf4j.LoggerFactory -import org.springframework.cloud.sleuth.Tracer import reactor.core.publisher.Flux import reactor.core.publisher.Mono import reactor.util.function.Tuple2 @@ -49,8 +46,7 @@ import java.util.function.Function class QuorumRequestReader( private val apiControl: ApiSource, private val quorum: CallQuorum, - signer: ResponseSigner?, - private val tracer: Tracer, + signer: ResponseSigner, ) : RequestReader(signer) { private val errorHandler = UpstreamErrorHandler @@ -58,7 +54,7 @@ class QuorumRequestReader( private val log = LoggerFactory.getLogger(QuorumRequestReader::class.java) } - constructor(apiControl: ApiSource, quorum: CallQuorum, tracer: Tracer) : this(apiControl, quorum, null, tracer) + constructor(apiControl: ApiSource, quorum: CallQuorum) : this(apiControl, quorum, DisabledSigner()) override fun attempts(): AtomicInteger = apiControl.attempts() @@ -135,7 +131,7 @@ class QuorumRequestReader( .map { quorum -> val response = quorum.getResponse()!! // TODO find actual quorum number - Result(response.getResult(), quorum.getSignature(), 1, resolvedBy(), response.stream) + Result(response.getResult(), quorum.getSignature(), 1, resolvedBy(), response.stream, response.responseHeaders) } .switchIfEmpty(defaultResult) } @@ -147,7 +143,7 @@ class QuorumRequestReader( SPAN_REQUEST_API_TYPE to apiReader.javaClass.name, SPAN_REQUEST_UPSTREAM_ID to api.getId(), ) - return SpannedReader(apiReader, tracer, API_READER, spanParams) + return apiReader .read(key) .flatMap { response -> log.trace("Received response from upstream ${api.getId()} for method ${key.method}") @@ -173,7 +169,7 @@ class QuorumRequestReader( } } - private fun withErrorResume(api: Upstream, key: ChainRequest): Function, Mono> { + private fun withErrorResume(api: Upstream, key: ChainRequest): Function, Mono> { return Function { src -> src.onErrorResume { err -> errorHandler.handle(api, key, err.message) @@ -232,11 +228,10 @@ class QuorumRequestReader( private fun noResponse(method: String, q: CallQuorum): Mono { return apiControl.upstreamsMatchesResponse()?.run { - tracer.currentSpan()?.tag(SPAN_NO_RESPONSE_MESSAGE, getFullCause()) val cause = getCause(method) ?: return Mono.error(RpcException(1, "No response for method $method", getFullCause())) if (cause.shouldReturnNull) { Mono.just( - Result(Global.nullValue, null, 1, emptyList(), null), + Result(Global.nullValue, null, 1, emptyList(), null, emptyMap()), ) } else { Mono.error(RpcException(1, "No response for method $method. Cause - ${cause.cause}")) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/reader/BroadcastReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/reader/BroadcastReader.kt index 490952d38..b4c221d13 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/reader/BroadcastReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/reader/BroadcastReader.kt @@ -1,7 +1,5 @@ package io.emeraldpay.dshackle.reader -import io.emeraldpay.dshackle.commons.BROADCAST_READER -import io.emeraldpay.dshackle.commons.SPAN_REQUEST_UPSTREAM_ID import io.emeraldpay.dshackle.quorum.CallQuorum import io.emeraldpay.dshackle.upstream.ChainException import io.emeraldpay.dshackle.upstream.ChainRequest @@ -11,7 +9,6 @@ import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.error.UpstreamErrorHandler import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import org.slf4j.LoggerFactory -import org.springframework.cloud.sleuth.Tracer import reactor.core.publisher.Flux import reactor.core.publisher.Mono import java.util.concurrent.atomic.AtomicInteger @@ -19,9 +16,8 @@ import java.util.concurrent.atomic.AtomicInteger class BroadcastReader( private val upstreams: List, matcher: Selector.Matcher, - signer: ResponseSigner?, + signer: ResponseSigner, private val quorum: CallQuorum, - private val tracer: Tracer, ) : RequestReader(signer) { private val errorHandler = UpstreamErrorHandler private val internalMatcher = Selector.MultiMatcher( @@ -64,6 +60,7 @@ class BroadcastReader( upstreams.size, upsData, null, + quorum.getResponse()!!.responseHeaders, ) Mono.just(res) } else { @@ -76,12 +73,7 @@ class BroadcastReader( key: ChainRequest, upstream: Upstream, ): Mono = - SpannedReader( - upstream.getIngressReader(), - tracer, - BROADCAST_READER, - mapOf(SPAN_REQUEST_UPSTREAM_ID to upstream.getId()), - ) + upstream.getIngressReader() .read(key) .map { BroadcastResponse(it, upstream) } .onErrorResume { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/reader/CompoundReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/reader/CompoundReader.kt index 722c7bfb3..8fc3ee947 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/reader/CompoundReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/reader/CompoundReader.kt @@ -25,7 +25,7 @@ import reactor.core.publisher.Mono * Composition of multiple readers. * Reader returns first value returned by any of the source readers by checking one by one until one of them returns a non-empty result. */ -class CompoundReader ( +class CompoundReader ( private vararg val readers: Reader, ) : Reader { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/reader/EmptyReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/reader/EmptyReader.kt index f25cc08ce..c81485f60 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/reader/EmptyReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/reader/EmptyReader.kt @@ -18,7 +18,7 @@ package io.emeraldpay.dshackle.reader import reactor.core.publisher.Mono -class EmptyReader : Reader { +class EmptyReader : Reader { override fun read(key: K): Mono { return Mono.empty() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/reader/Reader.kt b/src/main/kotlin/io/emeraldpay/dshackle/reader/Reader.kt index eec6fd9b0..3277c7cb2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/reader/Reader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/reader/Reader.kt @@ -18,7 +18,7 @@ package io.emeraldpay.dshackle.reader import reactor.core.publisher.Mono -interface Reader { +interface Reader { fun read(key: K): Mono } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/reader/RekeyingReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/reader/RekeyingReader.kt index 31c5244fc..37d078456 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/reader/RekeyingReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/reader/RekeyingReader.kt @@ -21,7 +21,7 @@ import java.util.function.Function /** * Reader wrapper that maps the input key from ne value to another (ex. convert from Long to String) */ -class RekeyingReader( +class RekeyingReader( /** * Mapping between original Key and Key supported by the reader */ diff --git a/src/main/kotlin/io/emeraldpay/dshackle/reader/RequestReaderFactory.kt b/src/main/kotlin/io/emeraldpay/dshackle/reader/RequestReaderFactory.kt index bdb11fc50..f58161624 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/reader/RequestReaderFactory.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/reader/RequestReaderFactory.kt @@ -15,12 +15,11 @@ import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.stream.Chunk -import org.springframework.cloud.sleuth.Tracer import reactor.core.publisher.Flux import java.util.concurrent.atomic.AtomicInteger abstract class RequestReader( - private val signer: ResponseSigner?, + private val signer: ResponseSigner, ) : Reader { abstract fun attempts(): AtomicInteger @@ -44,17 +43,18 @@ abstract class RequestReader( protected fun getSignature(key: ChainRequest, response: ChainResponse, upstreamId: String) = response.providedSignature ?: if (key.nonce != null) { - signer?.sign(key.nonce, response.getResult(), upstreamId) + signer.sign(key.nonce, response.getResult(), upstreamId) } else { null } - class Result( + class Result @JvmOverloads constructor( val value: ByteArray, val signature: ResponseSigner.Signature?, val quorum: Int, val resolvedUpstreamData: List, val stream: Flux?, + val responseHeaders: Map = emptyMap(), ) } @@ -71,10 +71,10 @@ interface RequestReaderFactory { class Default : RequestReaderFactory { override fun create(data: ReaderData): RequestReader { if (data.quorum is MaximumValueQuorum || data.quorum is BroadcastQuorum) { - return BroadcastReader(data.multistream.getAll(), data.upstreamFilter.matcher, data.signer, data.quorum, data.tracer) + return BroadcastReader(data.multistream.getAll(), data.upstreamFilter.matcher, data.signer, data.quorum) } val apis = data.multistream.getApiSource(data.upstreamFilter) - return QuorumRequestReader(apis, data.quorum, data.signer, data.tracer) + return QuorumRequestReader(apis, data.quorum, data.signer) } } @@ -82,7 +82,6 @@ interface RequestReaderFactory { val multistream: Multistream, val upstreamFilter: Selector.UpstreamFilter, val quorum: CallQuorum, - val signer: ResponseSigner?, - val tracer: Tracer, + val signer: ResponseSigner, ) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/reader/SpannedReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/reader/SpannedReader.kt deleted file mode 100644 index b33896b93..000000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/reader/SpannedReader.kt +++ /dev/null @@ -1,58 +0,0 @@ -package io.emeraldpay.dshackle.reader - -import io.emeraldpay.dshackle.commons.SPAN_ERROR -import io.emeraldpay.dshackle.commons.SPAN_READER_NAME -import io.emeraldpay.dshackle.commons.SPAN_READER_RESULT -import io.emeraldpay.dshackle.commons.SPAN_REQUEST_CANCELLED -import io.emeraldpay.dshackle.commons.SPAN_REQUEST_INFO -import io.emeraldpay.dshackle.commons.SPAN_STATUS_MESSAGE -import io.emeraldpay.dshackle.data.HashId -import io.emeraldpay.dshackle.upstream.ChainRequest -import org.springframework.cloud.sleuth.Tracer -import org.springframework.cloud.sleuth.instrument.reactor.ReactorSleuth -import reactor.core.publisher.Mono -import reactor.kotlin.core.publisher.switchIfEmpty - -class SpannedReader( - private val reader: Reader, - private val tracer: Tracer, - private val name: String, - private val additionalParams: Map = emptyMap(), -) : Reader { - - override fun read(key: K): Mono { - val newSpan = tracer.nextSpan(tracer.currentSpan()) - .name(reader.javaClass.name) - .tag(SPAN_READER_NAME, name) - .start() - - extractInfoFromKey(key)?.let { - newSpan.tag(SPAN_REQUEST_INFO, it) - } - additionalParams.forEach { newSpan.tag(it.key, it.value) } - - return reader.read(key) - .contextWrite { ReactorSleuth.putSpanInScope(tracer, it, newSpan) } - .doOnError { - newSpan.tag(SPAN_ERROR, "true") - .tag(SPAN_STATUS_MESSAGE, it.message) - .end() - } - .doOnNext { newSpan.end() } - .doOnCancel { - newSpan.tag(SPAN_STATUS_MESSAGE, SPAN_REQUEST_CANCELLED).end() - } - .switchIfEmpty { - newSpan.tag(SPAN_READER_RESULT, "empty result").end() - Mono.empty() - } - } - - private fun extractInfoFromKey(key: K): String? { - return when (key) { - is ChainRequest -> "method: ${key.method}" - is HashId, Long -> "params: $key" - else -> null - } - } -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/reader/TransformingReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/reader/TransformingReader.kt index 4c408b1cc..a43d469f3 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/reader/TransformingReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/reader/TransformingReader.kt @@ -21,7 +21,7 @@ import java.util.function.Function /** * Reader wrapper that transforms output of the reader to a different format */ -class TransformingReader( +class TransformingReader( /** * Actual reader */ diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt index 3e3100137..65a261921 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/BlockchainRpc.kt @@ -16,19 +16,16 @@ */ package io.emeraldpay.dshackle.rpc -import brave.Tracer import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.api.proto.Common import io.emeraldpay.api.proto.ReactorBlockchainGrpc import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.ChainValue -import io.emeraldpay.dshackle.config.spans.ProviderSpanHandler import io.micrometer.core.instrument.Counter import io.micrometer.core.instrument.Metrics import io.micrometer.core.instrument.Timer import org.apache.commons.lang3.RandomStringUtils import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Qualifier import org.springframework.context.annotation.DependsOn import org.springframework.stereotype.Service @@ -47,11 +44,8 @@ class BlockchainRpc( private val describe: Describe, private val subscribeStatus: SubscribeStatus, private val subscribeNodeStatus: SubscribeNodeStatus, - @Qualifier("rpcScheduler") + @param:Qualifier("rpcScheduler") private val scheduler: Scheduler, - @Autowired(required = false) - private val providerSpanHandler: ProviderSpanHandler?, - private val tracer: Tracer, private val subscribeChainStatus: SubscribeChainStatus, ) : ReactorBlockchainGrpc.BlockchainImplBase() { @@ -100,10 +94,6 @@ class BlockchainRpc( } }.doOnError { failMetric.increment() - }.doFinally { - tracer.currentSpan()?.run { - providerSpanHandler?.sendSpans(this.context()) - } } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/ChainEventMapper.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/ChainEventMapper.kt index d193a4ed3..a5d037935 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/ChainEventMapper.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/ChainEventMapper.kt @@ -32,11 +32,12 @@ class ChainEventMapper { } fun mapCapabilities(capabilities: Collection): BlockchainOuterClass.ChainEvent { - val caps = capabilities.map { + val caps = capabilities.filter { it != Capability.WS_PENDING_TX }.map { when (it) { Capability.RPC -> BlockchainOuterClass.Capabilities.CAP_CALLS Capability.BALANCE -> BlockchainOuterClass.Capabilities.CAP_BALANCE Capability.WS_HEAD -> BlockchainOuterClass.Capabilities.CAP_WS_HEAD + else -> null } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/Describe.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/Describe.kt index ea922ad5b..27b79b622 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/Describe.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/Describe.kt @@ -21,14 +21,13 @@ import io.emeraldpay.api.proto.Common import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.upstream.Capability import io.emeraldpay.dshackle.upstream.MultistreamHolder -import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import reactor.core.publisher.Mono @Service class Describe( - @Autowired private val multistreamHolder: MultistreamHolder, - @Autowired private val subscribeStatus: SubscribeStatus, + private val multistreamHolder: MultistreamHolder, + private val subscribeStatus: SubscribeStatus, ) { fun describe(requestMono: Mono): Mono { @@ -64,11 +63,12 @@ class Describe( } capabilities.addAll(chainUpstreams.getCapabilities()) chainDescription.addAllCapabilities( - capabilities.map { + capabilities.filter { it != Capability.WS_PENDING_TX }.map { when (it) { Capability.RPC -> BlockchainOuterClass.Capabilities.CAP_CALLS Capability.BALANCE -> BlockchainOuterClass.Capabilities.CAP_BALANCE Capability.WS_HEAD -> BlockchainOuterClass.Capabilities.CAP_WS_HEAD + else -> null } }, ) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt index fd9e35ca8..32169a30a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt @@ -25,19 +25,12 @@ import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.Global.Companion.nullValue import io.emeraldpay.dshackle.SilentException -import io.emeraldpay.dshackle.commons.LOCAL_READER -import io.emeraldpay.dshackle.commons.RPC_READER -import io.emeraldpay.dshackle.commons.SPAN_ERROR -import io.emeraldpay.dshackle.commons.SPAN_REQUEST_CANCELLED -import io.emeraldpay.dshackle.commons.SPAN_REQUEST_ID -import io.emeraldpay.dshackle.commons.SPAN_STATUS_MESSAGE import io.emeraldpay.dshackle.config.MainConfig import io.emeraldpay.dshackle.quorum.CallQuorum import io.emeraldpay.dshackle.quorum.NotLaggingQuorum import io.emeraldpay.dshackle.reader.RequestReader import io.emeraldpay.dshackle.reader.RequestReaderFactory import io.emeraldpay.dshackle.reader.RequestReaderFactory.ReaderData -import io.emeraldpay.dshackle.reader.SpannedReader import io.emeraldpay.dshackle.upstream.ApiSource import io.emeraldpay.dshackle.upstream.ChainCallError import io.emeraldpay.dshackle.upstream.ChainException @@ -61,21 +54,16 @@ import io.micrometer.core.instrument.Metrics import org.apache.commons.lang3.StringUtils import org.reactivestreams.Publisher import org.slf4j.LoggerFactory -import org.springframework.cloud.sleuth.Span -import org.springframework.cloud.sleuth.Tracer -import org.springframework.cloud.sleuth.instrument.reactor.ReactorSleuth import org.springframework.stereotype.Service import reactor.core.publisher.Flux import reactor.core.publisher.Mono import reactor.kotlin.core.publisher.toMono -import reactor.util.context.Context @Service open class NativeCall( private val multistreamHolder: MultistreamHolder, private val signer: ResponseSigner, config: MainConfig, - private val tracer: Tracer, ) { private val log = LoggerFactory.getLogger(NativeCall::class.java) @@ -89,16 +77,14 @@ open class NativeCall( return nativeCallResult(requestMono) .flatMapSequential(this::processCallResult) .onErrorResume(this::processException) + .contextCapture() } open fun nativeCallResult(requestMono: Mono): Flux { - val requestSpan = tracer.currentSpan() return requestMono.flatMapMany(this::prepareCall) .flatMap { - val requestId = it.requestId - val requestCount = it.requestCount val id = it.getContextId() - val result = processCallContext(it, requestSpan) + val result = processCallContext(it) return@flatMap result .onErrorResume { err -> @@ -122,12 +108,7 @@ open class NativeCall( } } } - completeSpan(callRes, requestCount) } - .doOnCancel { - tracer.currentSpan()?.tag(SPAN_STATUS_MESSAGE, SPAN_REQUEST_CANCELLED)?.end() - } - .contextWrite { ctx -> createTracingReactorContext(ctx, requestCount, requestId, requestSpan) } } } @@ -143,7 +124,7 @@ open class NativeCall( Flux.concat( Mono.just(firstChunk) .map { - val result = buildStreamResult(it, callResult.id) + val result = buildStreamResult(it, callResult.id, callResult.responseHeaders) if (callResult.upstreamSettingsData.isNotEmpty()) { getUpstreamIdsAndVersions(callResult.upstreamSettingsData) .let { idsAndVersions -> @@ -160,51 +141,26 @@ open class NativeCall( } } - private fun buildStreamResult(chunk: Chunk, id: Int): BlockchainOuterClass.NativeCallReplyItem.Builder { - return BlockchainOuterClass.NativeCallReplyItem.newBuilder() + private fun buildStreamResult(chunk: Chunk, id: Int, headers: Map = emptyMap()): BlockchainOuterClass.NativeCallReplyItem.Builder { + val builder = BlockchainOuterClass.NativeCallReplyItem.newBuilder() .setSucceed(true) .setFinalChunk(chunk.finalChunk) .setChunked(true) .setPayload(ByteString.copyFrom(chunk.chunkData)) .setId(id) - } - - private fun completeSpan(callResult: CallResult, requestCount: Int) { - val span = tracer.currentSpan() - if (callResult.isError()) { - errorSpan(span, callResult.error?.message ?: "Internal error") - } - if (requestCount > 1) { - span?.end() - } - } - - private fun errorSpan(span: Span?, message: String) { - span?.apply { - tag(SPAN_ERROR, "true") - tag(SPAN_STATUS_MESSAGE, message) - } - } - - private fun createTracingReactorContext( - ctx: Context, - requestCount: Int, - requestId: String, - requestSpan: Span?, - ): Context { - if (requestCount > 1) { - val span = tracer.nextSpan(requestSpan) - .name("emerald.blockchain/nativecall") - .tag(SPAN_REQUEST_ID, requestId) - .start() - return ReactorSleuth.putSpanInScope(tracer, ctx, span) + headers.forEach { (key, value) -> + builder.addResponseHeaders( + BlockchainOuterClass.KeyValue.newBuilder() + .setKey(key) + .setValue(value) + .build(), + ) } - return ctx + return builder } private fun processCallContext( callContext: CallContext, - requestSpan: Span?, ): Mono { return if (callContext.isValid()) { run { @@ -213,9 +169,6 @@ open class NativeCall( } catch (e: Exception) { return@run Mono.error(e) } - if (callContext.requestCount == 1 && callContext.requestId.isNotBlank()) { - requestSpan?.tag(SPAN_REQUEST_ID, callContext.requestId) - } this.fetch(parsed) .doOnError { e -> log.warn("Error during native call: ${e.message}") } } @@ -245,6 +198,9 @@ open class NativeCall( error.data?.let { data -> result.setErrorData(data) } + if (error.errorAsIs != null) { + result.errorAsIs = ByteString.copyFrom(error.errorAsIs) + } } } else { result.payload = ByteString.copyFrom(it.result) @@ -263,6 +219,14 @@ open class NativeCall( .setType(it.type.toProtoFinalizationType()) .build() } + it.responseHeaders.forEach { (key, value) -> + result.addResponseHeaders( + BlockchainOuterClass.KeyValue.newBuilder() + .setKey(key) + .setValue(value) + .build(), + ) + } return result.build() } @@ -286,7 +250,6 @@ open class NativeCall( 0 } val message = it?.message ?: "Internal error" - errorSpan(tracer.currentSpan(), message) return BlockchainOuterClass.NativeCallReplyItem.newBuilder() .setSucceed(false) .setErrorMessage(message) @@ -446,7 +409,7 @@ open class NativeCall( fun fetch(ctx: ValidCallContext): Mono { return ctx.upstream.getLocalReader() .flatMap { api -> - SpannedReader(api, tracer, LOCAL_READER) + api .read(ctx.payload.toChainRequest(ctx.nonce, ctx.forwardedSelector, false, ctx.upstreamFilter)) .map { val result = it.getResult() @@ -460,9 +423,9 @@ open class NativeCall( } else { ctx.upstream.getId() } - CallResult.ok(ctx.id, ctx.nonce, result, signer.sign(ctx.nonce, result, source), resolvedUpstreamData, ctx, it.finalization) + CallResult.ok(ctx.id, ctx.nonce, result, signer.sign(ctx.nonce, result, source), resolvedUpstreamData, ctx, it.finalization, it.responseHeaders) } else { - CallResult.ok(ctx.id, null, result, null, resolvedUpstreamData, ctx, it.finalization) + CallResult.ok(ctx.id, null, result, null, resolvedUpstreamData, ctx, it.finalization, it.responseHeaders) } } }.switchIfEmpty( @@ -483,7 +446,7 @@ open class NativeCall( return Mono.error(RpcException(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unsupported method")) } val reader = requestReaderFactory.create( - ReaderData(ctx.upstream, ctx.upstreamFilter, ctx.callQuorum, signer, tracer), + ReaderData(ctx.upstream, ctx.upstreamFilter, ctx.callQuorum, signer), ) val counter = reader.attempts() val isRipple = ctx.upstream.getChain() in listOf(Chain.RIPPLE__MAINNET, Chain.RIPPLE__TESTNET) @@ -492,7 +455,7 @@ open class NativeCall( streamRequest = false } - return SpannedReader(reader, tracer, RPC_READER) + return reader .read(ctx.payload.toChainRequest(ctx.nonce, ctx.forwardedSelector, streamRequest)) .map { val resolvedUpstreamData = it.resolvedUpstreamData.ifEmpty { @@ -505,7 +468,7 @@ open class NativeCall( callResult(ctx, it, resolvedUpstreamData) } } else { - CallResult.ok(ctx.id, ctx.nonce, ByteArray(0), it.signature, resolvedUpstreamData, ctx, it.stream) + CallResult.ok(ctx.id, ctx.nonce, ByteArray(0), it.signature, resolvedUpstreamData, ctx, it.stream, it.responseHeaders) } } .onErrorResume { t -> @@ -534,7 +497,7 @@ open class NativeCall( ): CallResult { val bytes = ctx.resultDecorator.processResult(it) validateResult(bytes, "remote", ctx) - return CallResult.ok(ctx.id, ctx.nonce, bytes, it.signature, resolvedUpstreamData, ctx) + return CallResult.ok(ctx.id, ctx.nonce, bytes, it.signature, resolvedUpstreamData, ctx, it.responseHeaders) } private fun callRippleResult( @@ -784,6 +747,7 @@ open class NativeCall( val upstreamError: ChainCallError?, val data: String?, val upstreamSettingsData: List = emptyList(), + val errorAsIs: ByteArray? = null, ) { companion object { @@ -801,7 +765,7 @@ open class NativeCall( } fun from(t: Throwable): CallError { return when (t) { - is ChainException -> CallError(t.error.code, t.error.message, t.error, getDataAsSting(t.error.details), t.upstreamSettingsData) + is ChainException -> CallError(t.error.code, t.error.message, t.error, getDataAsSting(t.error.details), t.upstreamSettingsData, t.error.errorAsIs) is RpcException -> CallError(t.code, t.rpcMessage, null, getDataAsSting(t.details)) is CallFailure -> CallError(t.id, t.reason.message ?: "Upstream Error", null, null) else -> { @@ -816,6 +780,28 @@ open class NativeCall( } } } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is CallError) return false + + if (id != other.id) return false + if (message != other.message) return false + if (upstreamError != other.upstreamError) return false + if (data != other.data) return false + if (upstreamSettingsData != other.upstreamSettingsData) return false + + return true + } + + override fun hashCode(): Int { + var result = id + result = 31 * result + message.hashCode() + result = 31 * result + (upstreamError?.hashCode() ?: 0) + result = 31 * result + (data?.hashCode() ?: 0) + result = 31 * result + upstreamSettingsData.hashCode() + return result + } } open class CallResult @JvmOverloads constructor( @@ -828,6 +814,7 @@ open class NativeCall( val ctx: ValidCallContext?, val stream: Flux? = null, val finalization: FinalizationData? = null, + val responseHeaders: Map = emptyMap(), ) { constructor( @@ -840,16 +827,16 @@ open class NativeCall( ) : this(id, nonce, result, callError, signature, callError?.upstreamSettingsData ?: emptyList(), ctx) companion object { - fun ok(id: Int, nonce: Long?, result: ByteArray, signature: ResponseSigner.Signature?, upstreamSettingsData: List, ctx: ValidCallContext?): CallResult { - return CallResult(id, nonce, result, null, signature, upstreamSettingsData, ctx) + fun ok(id: Int, nonce: Long?, result: ByteArray, signature: ResponseSigner.Signature?, upstreamSettingsData: List, ctx: ValidCallContext?, responseHeaders: Map = emptyMap()): CallResult { + return CallResult(id, nonce, result, null, signature, upstreamSettingsData, ctx, null, null, responseHeaders) } - fun ok(id: Int, nonce: Long?, result: ByteArray, signature: ResponseSigner.Signature?, upstreamSettingsData: List, ctx: ValidCallContext?, final: FinalizationData?): CallResult { - return CallResult(id, nonce, result, null, signature, upstreamSettingsData, ctx, null, final) + fun ok(id: Int, nonce: Long?, result: ByteArray, signature: ResponseSigner.Signature?, upstreamSettingsData: List, ctx: ValidCallContext?, final: FinalizationData?, responseHeaders: Map = emptyMap()): CallResult { + return CallResult(id, nonce, result, null, signature, upstreamSettingsData, ctx, null, final, responseHeaders) } - fun ok(id: Int, nonce: Long?, result: ByteArray, signature: ResponseSigner.Signature?, upstreamSettingsData: List, ctx: ValidCallContext?, stream: Flux?): CallResult { - return CallResult(id, nonce, result, null, signature, upstreamSettingsData, ctx, stream) + fun ok(id: Int, nonce: Long?, result: ByteArray, signature: ResponseSigner.Signature?, upstreamSettingsData: List, ctx: ValidCallContext?, stream: Flux?, responseHeaders: Map = emptyMap()): CallResult { + return CallResult(id, nonce, result, null, signature, upstreamSettingsData, ctx, stream, null, responseHeaders) } fun fail(id: Int, nonce: Long?, error: CallError, ctx: ValidCallContext?): CallResult { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt index 86135c8be..f33317564 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeSubscribe.kt @@ -31,7 +31,6 @@ import io.grpc.Status import io.grpc.StatusException import org.reactivestreams.Publisher import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import reactor.core.publisher.Flux import reactor.core.publisher.Mono @@ -39,8 +38,8 @@ import java.util.concurrent.atomic.AtomicLong @Service open class NativeSubscribe( - @Autowired private val multistreamHolder: MultistreamHolder, - @Autowired private val signer: ResponseSigner, + private val multistreamHolder: MultistreamHolder, + private val signer: ResponseSigner, ) { companion object { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/StreamHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/StreamHead.kt index f72d75d21..bfacb428e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/StreamHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/StreamHead.kt @@ -26,14 +26,13 @@ import io.emeraldpay.dshackle.upstream.MultistreamHolder import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import reactor.core.publisher.Flux import reactor.core.publisher.Mono @Service class StreamHead( - @Autowired private val multistreamHolder: MultistreamHolder, + private val multistreamHolder: MultistreamHolder, ) { private val log = LoggerFactory.getLogger(StreamHead::class.java) @@ -45,7 +44,7 @@ class StreamHead( ms.getHead() .getFlux() .map { asProto(ms, chain, it!!) } - .onErrorContinue { t, _ -> + .onErrorContinue { t, _: Any? -> log.warn("Head subscription error", t) } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeChainStatus.kt b/src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeChainStatus.kt index 319361fbe..1072e145a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeChainStatus.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeChainStatus.kt @@ -25,25 +25,25 @@ class SubscribeChainStatus( } fun chainStatuses(): Flux { - return Flux.merge( - // we need to track not only multistreams with upstreams but all of them - // because upstreams can be added in runtime with hot config reload - multistreamHolder.all() - .filter { Common.ChainRef.forNumber(it.getChain().id) != null } - .map { ms -> - Flux.concat( - // the first event must be filled with all fields - firstFullEvent(ms), - Flux.merge( - // head events are separated from others - headEvents(ms), - multistreamEvents(ms), - ), - ) - }, - ).doOnError { - log.error("Error during sending chain statuses", it) - } + // we need to track not only multistreams with upstreams but all of them + // because upstreams can be added in runtime with hot config reload + val sources = multistreamHolder.all() + .filter { Common.ChainRef.forNumber(it.getChain().id) != null } + .map { ms -> + Flux.concat( + // the first event must be filled with all fields + firstFullEvent(ms), + Flux.merge( + // head events are separated from others + headEvents(ms), + multistreamEvents(ms), + ), + ) + } + return Flux.merge(Flux.fromIterable(sources), sources.size) + .doOnError { + log.error("Error during sending chain statuses", it) + } } private fun multistreamEvents(ms: Multistream): Flux { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/BitcoinUpstreamCreator.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/BitcoinUpstreamCreator.kt index 9925aa615..e16a2e8e8 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/BitcoinUpstreamCreator.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/BitcoinUpstreamCreator.kt @@ -16,6 +16,7 @@ import io.emeraldpay.dshackle.upstream.bitcoin.EsploraClient import io.emeraldpay.dshackle.upstream.bitcoin.ExtractBlock import io.emeraldpay.dshackle.upstream.bitcoin.ZMQServer import io.emeraldpay.dshackle.upstream.forkchoice.MostWorkForkChoice +import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import org.springframework.stereotype.Component import reactor.core.scheduler.Scheduler import java.util.concurrent.atomic.AtomicInteger @@ -27,7 +28,8 @@ class BitcoinUpstreamCreator( private val genericConnectorFactoryCreator: ConnectorFactoryCreator, private val fileResolver: FileResolver, private val headScheduler: Scheduler, -) : UpstreamCreator(chainsConfig, callTargets) { + signer: ResponseSigner, +) : UpstreamCreator(chainsConfig, callTargets, signer) { private var seq = AtomicInteger(0) override fun createUpstream( @@ -38,7 +40,7 @@ class BitcoinUpstreamCreator( ): UpstreamCreationData { val config = upstreamsConfig.cast(UpstreamsConfig.BitcoinConnection::class.java) val conn = config.connection!! - val httpFactory = genericConnectorFactoryCreator.buildHttpFactory(conn.rpc) + val httpFactory = genericConnectorFactoryCreator.buildHttpFactory(conn.rpc, customHeaders = config.customHeaders) if (httpFactory == null) { log.warn("Upstream doesn't have API configuration") return UpstreamCreationData.default() @@ -50,7 +52,7 @@ class BitcoinUpstreamCreator( fileResolver.resolve(ca).readBytes() } } - EsploraClient(endpoint.url, endpoint.basicAuth, tls) + EsploraClient(endpoint.url, endpoint.basicAuth, tls, endpoint.bearerAuth) } val extractBlock = ExtractBlock() @@ -67,7 +69,7 @@ class BitcoinUpstreamCreator( ?: "bitcoin-${seq.getAndIncrement()}", chain, directApi, head, options, config.role, - QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(config.labels)), + QuorumForLabels.QuorumItem(1, buildUpstreamLabels(config.labels)), methods, esplora, chainConf, ) upstream.start() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/ConnectorFactoryCreator.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/ConnectorFactoryCreator.kt index d71cc2976..570a46348 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/ConnectorFactoryCreator.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/ConnectorFactoryCreator.kt @@ -19,9 +19,14 @@ interface ConnectorFactoryCreator { forkChoice: ForkChoice, blockValidator: BlockValidator, chainsConf: ChainsConfig.ChainConfig, + customHeaders: Map = emptyMap(), ): ConnectorFactory? - fun buildHttpFactory(conn: UpstreamsConfig.HttpEndpoint?, urls: ArrayList? = null): HttpFactory? + fun buildHttpFactory( + conn: UpstreamsConfig.HttpEndpoint?, + urls: ArrayList? = null, + customHeaders: Map = emptyMap(), + ): HttpFactory? } @Component diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/EthereumUpstreamCreator.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/EthereumUpstreamCreator.kt index 2b9243fbe..154508fa3 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/EthereumUpstreamCreator.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/EthereumUpstreamCreator.kt @@ -6,6 +6,7 @@ import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.config.hot.CompatibleVersionsRules import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.upstream.CallTargetsHolder +import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import org.springframework.stereotype.Component import java.util.function.Supplier @@ -15,7 +16,8 @@ class EthereumUpstreamCreator( callTargets: CallTargetsHolder, connectorFactoryCreatorResolver: ConnectorFactoryCreatorResolver, versionRules: Supplier, -) : GenericUpstreamCreator(chainsConfig, callTargets, connectorFactoryCreatorResolver, versionRules) { + signer: ResponseSigner, +) : GenericUpstreamCreator(chainsConfig, callTargets, connectorFactoryCreatorResolver, versionRules, signer) { override fun createUpstream( upstreamsConfig: UpstreamsConfig.Upstream<*>, diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/GenericConnectorFactoryCreator.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/GenericConnectorFactoryCreator.kt index 8dc32db52..fa2a650b2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/GenericConnectorFactoryCreator.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/GenericConnectorFactoryCreator.kt @@ -26,6 +26,8 @@ open class GenericConnectorFactoryCreator( private val wsScheduler: Scheduler, private val headLivenessScheduler: Scheduler, private val monitoringCfg: MonitoringConfig, + private val httpScheduler: Scheduler, + private val eventsScheduler: Scheduler, ) : ConnectorFactoryCreator { protected val log = LoggerFactory.getLogger(this::class.java) @@ -36,10 +38,11 @@ open class GenericConnectorFactoryCreator( forkChoice: ForkChoice, blockValidator: BlockValidator, chainsConf: ChainsConfig.ChainConfig, + customHeaders: Map, ): ConnectorFactory? { val urls = ArrayList() - val wsFactoryApi = buildWsFactory(id, chain, conn, urls) - val httpFactory = buildHttpFactory(conn.rpc, urls) + val wsFactoryApi = buildWsFactory(id, chain, conn, urls, customHeaders) + val httpFactory = buildHttpFactory(conn.rpc, urls, customHeaders) log.info("Using ${chain.chainName} upstream, at ${urls.joinToString()}") val connectorFactory = GenericConnectorFactory( @@ -60,7 +63,11 @@ open class GenericConnectorFactoryCreator( return connectorFactory } - override fun buildHttpFactory(conn: UpstreamsConfig.HttpEndpoint?, urls: ArrayList?): HttpFactory? { + override fun buildHttpFactory( + conn: UpstreamsConfig.HttpEndpoint?, + urls: ArrayList?, + customHeaders: Map, + ): HttpFactory? { return conn?.let { endpoint -> val tls = conn.tls?.let { tls -> tls.ca?.let { ca -> @@ -75,6 +82,9 @@ open class GenericConnectorFactoryCreator( conn.basicAuth, tls, monitoringCfg.nettyMetricsConfig.enabled, + httpScheduler, + customHeaders, + conn.bearerAuth, ) } } @@ -84,6 +94,7 @@ open class GenericConnectorFactoryCreator( chain: Chain, conn: UpstreamsConfig.RpcConnection, urls: ArrayList? = null, + customHeaders: Map = emptyMap(), ): WsConnectionPoolFactory? { return conn.ws?.let { endpoint -> val wsConnectionFactory = WsConnectionFactory( @@ -92,9 +103,12 @@ open class GenericConnectorFactoryCreator( endpoint.url, endpoint.origin ?: URI("http://localhost"), wsScheduler, + eventsScheduler, ).apply { config = endpoint basicAuth = endpoint.basicAuth + bearerAuth = endpoint.bearerAuth + this.customHeaders = customHeaders } val wsApi = WsConnectionPoolFactory( id, diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/GenericUpstreamCreator.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/GenericUpstreamCreator.kt index a813ea678..c4d0c7b08 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/GenericUpstreamCreator.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/GenericUpstreamCreator.kt @@ -12,6 +12,7 @@ import io.emeraldpay.dshackle.upstream.forkchoice.NoChoiceWithPriorityForkChoice import io.emeraldpay.dshackle.upstream.generic.ChainSpecificRegistry import io.emeraldpay.dshackle.upstream.generic.GenericUpstream import io.emeraldpay.dshackle.upstream.generic.connectors.GenericConnectorFactory +import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import org.springframework.stereotype.Component import java.util.function.Supplier @@ -21,7 +22,8 @@ open class GenericUpstreamCreator( callTargets: CallTargetsHolder, private val connectorFactoryCreatorResolver: ConnectorFactoryCreatorResolver, private val versionRules: Supplier, -) : UpstreamCreator(chainsConfig, callTargets) { + signer: ResponseSigner, +) : UpstreamCreator(chainsConfig, callTargets, signer) { private val hashes = HashSet() override fun createUpstream( @@ -64,6 +66,7 @@ open class GenericUpstreamCreator( NoChoiceWithPriorityForkChoice(nodeRating, config.id!!), BlockValidator.ALWAYS_VALID, chainConfig, + config.customHeaders, ) ?: return UpstreamCreationData.default() val hashUrl = connection.let { @@ -77,7 +80,7 @@ open class GenericUpstreamCreator( chain, hash, options, - QuorumForLabels.QuorumItem(1, UpstreamsConfig.Labels.fromMap(config.labels)), + QuorumForLabels.QuorumItem(1, buildUpstreamLabels(config.labels)), chainConfig, connectorFactory, cs::validator, diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/RestConnectorFactoryCreator.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/RestConnectorFactoryCreator.kt index 321c8eeb9..19cfc7691 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/RestConnectorFactoryCreator.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/RestConnectorFactoryCreator.kt @@ -22,6 +22,8 @@ class RestConnectorFactoryCreator( private val headScheduler: Scheduler, private val headLivenessScheduler: Scheduler, monitoringCfg: MonitoringConfig, + httpScheduler: Scheduler, + eventsScheduler: Scheduler, ) : GenericConnectorFactoryCreator( fileResolver, Schedulers.single(), @@ -29,6 +31,8 @@ class RestConnectorFactoryCreator( Schedulers.single(), headLivenessScheduler, monitoringCfg, + httpScheduler, + eventsScheduler, ) { override fun createConnectorFactory( id: String, @@ -37,10 +41,11 @@ class RestConnectorFactoryCreator( forkChoice: ForkChoice, blockValidator: BlockValidator, chainsConf: ChainsConfig.ChainConfig, + customHeaders: Map, ): ConnectorFactory? { val urls = ArrayList() - val httpFactory = buildHttpFactory(conn.rpc, urls) - val tonV3HttpFactory = buildHttpFactory(conn.getEndpointByTag("ton_v3")?.rpc, urls) + val httpFactory = buildHttpFactory(conn.rpc, urls, customHeaders) + val tonV3HttpFactory = buildHttpFactory(conn.getEndpointByTag("ton_v3")?.rpc, urls, customHeaders) val upstreamHttpFactory = if (httpFactory != null && chain.type == BlockchainType.TON) { TonCompoundHttpFactory(httpFactory, tonV3HttpFactory) } else { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/UpstreamCreator.kt b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/UpstreamCreator.kt index 5a7e7f276..1ef9a8f46 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/UpstreamCreator.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/startup/configure/UpstreamCreator.kt @@ -9,6 +9,9 @@ import io.emeraldpay.dshackle.foundation.ChainOptions.Options import io.emeraldpay.dshackle.upstream.CallTargetsHolder import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.ManagedCallMethods +import io.emeraldpay.dshackle.upstream.lowerbound.GoldLowerBounds +import io.emeraldpay.dshackle.upstream.signature.ResponseSigner +import jakarta.annotation.PostConstruct import org.slf4j.Logger import org.slf4j.LoggerFactory import java.util.function.Function @@ -17,10 +20,32 @@ import kotlin.math.abs abstract class UpstreamCreator( private val chainsConfig: ChainsConfig, private val callTargets: CallTargetsHolder, + private val signer: ResponseSigner, ) { protected val log: Logger = LoggerFactory.getLogger(this::class.java) + /** + * Builds the label map for an upstream starting from the user-provided labels. + * When response signing is actually enabled on this dshackle instance, the + * `secure-signed=true` label is injected automatically (unless the user has + * explicitly overridden it). + */ + protected fun buildUpstreamLabels(userLabels: Map): UpstreamsConfig.Labels { + val labels = UpstreamsConfig.Labels.fromMap(userLabels) + if (signer.enabled && !labels.containsKey(SECURE_SIGNED_LABEL)) { + labels[SECURE_SIGNED_LABEL] = "true" + } + return labels + } + + @PostConstruct + fun init() { + GoldLowerBounds.init(chainsConfig.getChainConfigs()) + } + companion object { + const val SECURE_SIGNED_LABEL = "secure-signed" + fun getHash(nodeId: Int?, obj: Any, hashes: MutableSet): Short { val hash = nodeId?.toShort() ?: run { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/BasicHttpFactory.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/BasicHttpFactory.kt index 7db34b1ee..259e83605 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/BasicHttpFactory.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/BasicHttpFactory.kt @@ -10,6 +10,7 @@ import io.micrometer.core.instrument.Metrics import io.micrometer.core.instrument.Tag import io.micrometer.core.instrument.Timer import org.slf4j.LoggerFactory +import reactor.core.scheduler.Scheduler class BasicHttpFactory( private val url: String, @@ -18,6 +19,9 @@ class BasicHttpFactory( private val basicAuth: AuthConfig.ClientBasicAuth?, private val tls: ByteArray?, private val nettyMetricsEnabled: Boolean, + private val httpScheduler: Scheduler, + private val customHeaders: Map = emptyMap(), + private val bearerAuth: AuthConfig.ClientBearerAuth? = null, ) : HttpFactory { private val log = LoggerFactory.getLogger(this::class.java) @@ -31,11 +35,14 @@ class BasicHttpFactory( Tag.of("chain", chain.chainCode), ) val metrics = RequestMetrics( - Timer.builder("upstream.rpc.conn") - .description("Request time through a HTTP JSON RPC connection") - .tags(metricsTags) - .publishPercentileHistogram() - .register(Metrics.globalRegistry), + { method -> + Timer.builder("upstream.rpc.conn") + .description("Request time through a HTTP JSON RPC connection") + .tags(metricsTags) + .tag("method", method ?: "unknown") + .publishPercentileHistogram() + .register(Metrics.globalRegistry) + }, Counter.builder("upstream.rpc.fail") .description("Number of failures of HTTP JSON RPC requests") .tags(metricsTags) @@ -44,8 +51,8 @@ class BasicHttpFactory( ) if (chain.type.apiType == ApiType.REST) { - return RestHttpReader(url, maxConnections, queueSize, metrics, basicAuth, tls) + return RestHttpReader(url, maxConnections, queueSize, metrics, httpScheduler, chain, basicAuth, tls, customHeaders, bearerAuth) } - return JsonRpcHttpReader(url, maxConnections, queueSize, metrics, basicAuth, tls) + return JsonRpcHttpReader(url, maxConnections, queueSize, metrics, httpScheduler, basicAuth, tls, customHeaders, bearerAuth) } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CallTargetsHolder.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CallTargetsHolder.kt index 6e5e05c4d..b877d56f4 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CallTargetsHolder.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CallTargetsHolder.kt @@ -1,5 +1,7 @@ package io.emeraldpay.dshackle.upstream +import io.emeraldpay.dshackle.BlockchainType.AVM +import io.emeraldpay.dshackle.BlockchainType.AZTEC import io.emeraldpay.dshackle.BlockchainType.BITCOIN import io.emeraldpay.dshackle.BlockchainType.COSMOS import io.emeraldpay.dshackle.BlockchainType.ETHEREUM @@ -16,6 +18,8 @@ import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.foundation.ChainOptions import io.emeraldpay.dshackle.upstream.calls.CallMethods +import io.emeraldpay.dshackle.upstream.calls.DefaultAvmMethods +import io.emeraldpay.dshackle.upstream.calls.DefaultAztecMethods import io.emeraldpay.dshackle.upstream.calls.DefaultBeaconChainMethods import io.emeraldpay.dshackle.upstream.calls.DefaultBitcoinMethods import io.emeraldpay.dshackle.upstream.calls.DefaultCosmosMethods @@ -47,6 +51,8 @@ class CallTargetsHolder { ): CallMethods { val created = when (chain.type) { BITCOIN -> DefaultBitcoinMethods(options.providesBalance == true) + AVM -> DefaultAvmMethods() + AZTEC -> DefaultAztecMethods() ETHEREUM -> DefaultEthereumMethods(chain) STARKNET -> DefaultStarknetMethods(chain) POLKADOT -> DefaultPolkadotMethods(chain) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Capability.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Capability.kt index 5e2189e59..949bb1d73 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Capability.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Capability.kt @@ -4,4 +4,5 @@ enum class Capability { RPC, BALANCE, WS_HEAD, + WS_PENDING_TX, } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainCallError.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainCallError.kt index bcca80bed..1c7f180f7 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainCallError.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainCallError.kt @@ -17,7 +17,12 @@ package io.emeraldpay.dshackle.upstream import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException -data class ChainCallError(val code: Int, val message: String, val details: Any?) { +data class ChainCallError( + val code: Int, + val message: String, + val details: Any?, + val errorAsIs: ByteArray? = null, +) { constructor(code: Int, message: String) : this(code, message, null) @@ -39,4 +44,22 @@ data class ChainCallError(val code: Int, val message: String, val details: Any?) fun asException(id: ChainResponse.Id?, upstreamSettingsData: List): ChainException { return ChainException(id ?: ChainResponse.NumberId(-1), this, upstreamSettingsData, false) } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is ChainCallError) return false + + if (code != other.code) return false + if (message != other.message) return false + if (details != other.details) return false + + return true + } + + override fun hashCode(): Int { + var result = code + result = 31 * result + message.hashCode() + result = 31 * result + (details?.hashCode() ?: 0) + return result + } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainResponse.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainResponse.kt index d73bc8652..6e49836b8 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainResponse.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ChainResponse.kt @@ -35,17 +35,25 @@ class ChainResponse @JvmOverloads constructor( val providedSignature: ResponseSigner.Signature? = null, val resolvedUpstreamData: List = emptyList(), val finalization: FinalizationData? = null, + val responseHeaders: Map = emptyMap(), ) { constructor(stream: Flux, id: Int) : - this(null, null, NumberId(id.toLong()), stream, null, emptyList(), null) + this(null, null, NumberId(id.toLong()), stream, null, emptyList(), null, emptyMap()) + + constructor(stream: Flux, id: Int, responseHeaders: Map) : + this(null, null, NumberId(id.toLong()), stream, null, emptyList(), null, responseHeaders) constructor(result: ByteArray?, error: ChainCallError?) : this(result, error, NumberId(0), null, null) + constructor(result: ByteArray?, error: ChainCallError?, responseHeaders: Map) : + this(result, error, NumberId(0), null, null, emptyList(), null, responseHeaders) + constructor(result: ByteArray?, error: ChainCallError?, resolvedUpstreamData: List) : - this(result, error, NumberId(0), null, null, resolvedUpstreamData, null) + this(result, error, NumberId(0), null, null, resolvedUpstreamData, null, emptyMap()) + constructor(result: ByteArray?, resolvedUpstreamData: List, finalization: FinalizationData) : - this(result, null, NumberId(0), null, null, resolvedUpstreamData, finalization) + this(result, null, NumberId(0), null, null, resolvedUpstreamData, finalization, emptyMap()) companion object { private val NULL_VALUE = "null".toByteArray() @@ -139,7 +147,7 @@ class ChainResponse @JvmOverloads constructor( } fun copyWithId(id: Id): ChainResponse { - return ChainResponse(result, error, id, stream, providedSignature, resolvedUpstreamData) + return ChainResponse(result, error, id, stream, providedSignature, resolvedUpstreamData, finalization, responseHeaders) } override fun equals(other: Any?): Boolean { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentMultistreamHolder.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentMultistreamHolder.kt index c3d340564..df499dd5f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentMultistreamHolder.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentMultistreamHolder.kt @@ -17,9 +17,9 @@ package io.emeraldpay.dshackle.upstream import io.emeraldpay.dshackle.Chain +import jakarta.annotation.PreDestroy import org.slf4j.LoggerFactory import org.springframework.stereotype.Component -import javax.annotation.PreDestroy @Component open class CurrentMultistreamHolder( diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt index 3ea8649b6..5fd5d758b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt @@ -172,6 +172,10 @@ abstract class DefaultUpstream( return 0 } + override fun getAdditionalSettings(): UpstreamsConfig.AdditionalSettings? { + return null + } + protected fun sendUpstreamStateEvent(eventType: UpstreamChangeEvent.ChangeType) { stateEventStream.emitNext( UpstreamChangeEvent(chain, this, eventType), diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/HeadLagObserver.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/HeadLagObserver.kt index 76cb719c0..5c116266a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/HeadLagObserver.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/HeadLagObserver.kt @@ -75,7 +75,7 @@ class HeadLagObserver( .parallel(followers.size) .flatMap { up -> mapLagging(top, up, getCurrentBlocks(up)).subscribeOn(lagObserverScheduler) } .sequential() - .onErrorContinue { t, _ -> log.warn("Failed to update lagging distance", t) } + .onErrorContinue { t, _: Any? -> log.warn("Failed to update lagging distance", t) } } open fun getCurrentBlocks(up: Upstream): Flux { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/HttpReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/HttpReader.kt index c0ec9d0f4..818a96d2a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/HttpReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/HttpReader.kt @@ -27,6 +27,8 @@ abstract class HttpReader( protected val metrics: RequestMetrics?, basicAuth: AuthConfig.ClientBasicAuth? = null, tlsCAAuth: ByteArray? = null, + customHeaders: Map = emptyMap(), + bearerAuth: AuthConfig.ClientBearerAuth? = null, ) : ChainReader { constructor() : this("", 1500, 1000, null) @@ -65,6 +67,22 @@ abstract class HttpReader( build = build.headers(headers) } + if (basicAuth == null) { + bearerAuth?.let { auth -> + val headers = Consumer { h: HttpHeaders -> h.add(HttpHeaderNames.AUTHORIZATION, "Bearer ${auth.token}") } + build = build.headers(headers) + } + } + + if (customHeaders.isNotEmpty()) { + val headers = Consumer { h: HttpHeaders -> + customHeaders.forEach { (key, value) -> + h.add(key, value) + } + } + build = build.headers(headers) + } + tlsCAAuth?.let { auth -> val cf = CertificateFactory.getInstance("X.509") val cert = cf.generateCertificate(ByteArrayInputStream(auth)) as X509Certificate @@ -91,7 +109,7 @@ abstract class HttpReader( open fun onStop() { if (metrics != null) { - Metrics.globalRegistry.remove(metrics.timer) + metrics.registeredTimers().forEach { Metrics.globalRegistry.remove(it) } Metrics.globalRegistry.remove(metrics.fails) } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/IngressSubscription.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/IngressSubscription.kt index af1205c9b..014f0ebfd 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/IngressSubscription.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/IngressSubscription.kt @@ -21,5 +21,5 @@ package io.emeraldpay.dshackle.upstream interface IngressSubscription { fun getAvailableTopics(): List - fun get(topic: String, params: Any?, unsubscribeMethod: String): SubscriptionConnect? + fun get(topic: String, params: Any?, unsubscribeMethod: String): SubscriptionConnect? } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/MatchesResponse.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/MatchesResponse.kt index f8026ce08..471fb5561 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/MatchesResponse.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/MatchesResponse.kt @@ -25,6 +25,7 @@ sealed class MatchesResponse { this.allResponses .filter { it !is Success } .joinToString("; ") { it.getCause()!! } + is NotMatchedResponse -> "Not matched - ${response.getCause()}" is SameNodeResponse -> "Upstream does not have hash ${this.upstreamHash}" is LowerHeightResponse -> { @@ -34,6 +35,8 @@ sealed class MatchesResponse { "Upstream lower height ${this.predictedHeight} of type ${this.boundType} is greater than ${this.lowerHeight}" } } + + is RangeVersionResponse -> "Upstream version is not within the range ${this.minVersion}-${this.maxVersion}" else -> null } @@ -100,5 +103,10 @@ sealed class MatchesResponse { val upstreamHash: Short, ) : MatchesResponse() + data class RangeVersionResponse( + val minVersion: String, + val maxVersion: String, + ) : MatchesResponse() + object AvailabilityResponse : MatchesResponse() } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt index 6f04efcc7..7c7bb332a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Multistream.kt @@ -108,6 +108,7 @@ abstract class Multistream( private fun monitorUpstream(upstream: Upstream) { val upstreamId = upstream.getId() + val clientType = upstream.getLabels().firstOrNull()?.get("client_type") ?: "" // otherwise metric will stuck with prev upstream instance removeUpstreamMeters(upstreamId) @@ -118,11 +119,13 @@ abstract class Multistream( } .tag("chain", chain.chainCode) .tag("upstream", upstreamId) + .tag("client_type", clientType) .register(Metrics.globalRegistry) .id, Gauge.builder("$metrics.availability.status", upstream) { it.getStatus().grpcId.toDouble() } .tag("chain", chain.chainCode) .tag("upstream", upstreamId) + .tag("client_type", clientType) .register(Metrics.globalRegistry) .id, ) @@ -261,6 +264,10 @@ abstract class Multistream( return 0 } + override fun getAdditionalSettings(): UpstreamsConfig.AdditionalSettings? { + return null + } + override fun getStatus(): UpstreamAvailability { return state.getStatus() } @@ -445,6 +452,7 @@ abstract class Multistream( UpstreamChangeEvent.ChangeType.REVALIDATED -> {} UpstreamChangeEvent.ChangeType.UPDATED -> { onUpstreamsUpdated() + monitorUpstream(event.upstream) updateUpstreams.emitNext(event.upstream) { _, res -> res == Sinks.EmitResult.FAIL_NON_SERIALIZED } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/NoIngressSubscription.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/NoIngressSubscription.kt index 32b1e5d02..b429ed910 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/NoIngressSubscription.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/NoIngressSubscription.kt @@ -25,7 +25,7 @@ open class NoIngressSubscription : IngressSubscription { return listOf() } - override fun get(topic: String, params: Any?, unsubscribeMethod: String): SubscriptionConnect? { + override fun get(topic: String, params: Any?, unsubscribeMethod: String): SubscriptionConnect? { return null } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/RequestMetrics.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/RequestMetrics.kt index 75ef2f1f3..7caca387a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/RequestMetrics.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/RequestMetrics.kt @@ -17,9 +17,20 @@ package io.emeraldpay.dshackle.upstream import io.micrometer.core.instrument.Counter import io.micrometer.core.instrument.Timer +import java.util.concurrent.ConcurrentHashMap class RequestMetrics( - val timer: Timer, + private val timerFactory: (String?) -> Timer, val fails: Counter, val nettyMetricsEnabled: Boolean, -) +) { + private val timerCache = ConcurrentHashMap() + + constructor(timer: Timer, fails: Counter, nettyMetricsEnabled: Boolean) : + this({ _ -> timer }, fails, nettyMetricsEnabled) + + fun timer(method: String? = null): Timer = + timerCache.computeIfAbsent(method ?: "") { timerFactory(method) } + + fun registeredTimers(): Collection = timerCache.values +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Selector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Selector.kt index f79bda828..b42dbf490 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Selector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Selector.kt @@ -16,6 +16,7 @@ */ package io.emeraldpay.dshackle.upstream +import com.vdurmont.semver4j.Semver import io.emeraldpay.api.proto.BlockchainOuterClass import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.upstream.MatchesResponse.AvailabilityResponse @@ -25,6 +26,7 @@ import io.emeraldpay.dshackle.upstream.MatchesResponse.GrpcResponse import io.emeraldpay.dshackle.upstream.MatchesResponse.HeightResponse import io.emeraldpay.dshackle.upstream.MatchesResponse.LowerHeightResponse import io.emeraldpay.dshackle.upstream.MatchesResponse.NotMatchedResponse +import io.emeraldpay.dshackle.upstream.MatchesResponse.RangeVersionResponse import io.emeraldpay.dshackle.upstream.MatchesResponse.SameNodeResponse import io.emeraldpay.dshackle.upstream.MatchesResponse.SlotHeightResponse import io.emeraldpay.dshackle.upstream.MatchesResponse.Success @@ -35,7 +37,6 @@ import org.apache.commons.lang3.StringUtils import java.util.Collections class Selector { - companion object { @JvmStatic @@ -55,6 +56,7 @@ class Selector { } else { Number(selector.height) } + BlockchainOuterClass.HeightSelector.HeightOrNumberCase.NUMBER -> Number(selector.number) BlockchainOuterClass.HeightSelector.HeightOrNumberCase.TAG -> when (selector.tag) { BlockchainOuterClass.BlockTag.SAFE -> Safe @@ -63,10 +65,12 @@ class Selector { BlockchainOuterClass.BlockTag.FINALIZED -> Finalized else -> null } + else -> null } } } + class Number(val num: Long) : HeightNumberOrTag() object Pending : HeightNumberOrTag() object Latest : HeightNumberOrTag() @@ -97,12 +101,14 @@ class Selector { it.hasSlotHeightSelector() -> { SlotMatcher(it.slotHeightSelector.slotHeight) } + it.hasHeightSelector() -> { when (val selector = HeightNumberOrTag.fromHeightSelector(it.heightSelector)) { is HeightNumberOrTag.Number -> HeightMatcher(selector.num) else -> empty } } + it.hasLowerHeightSelector() -> { if (it.lowerHeightSelector.height > 0) { LowerHeightMatcher( @@ -115,6 +121,7 @@ class Selector { empty } } + else -> empty } }.run { @@ -126,7 +133,8 @@ class Selector { private fun getSort(selectors: List): Sort { selectors.forEach { selector -> if (selector.hasHeightSelector()) { - val heightSort = HeightNumberOrTag.fromHeightSelector(selector.heightSelector)?.getSort() ?: Sort.default + val heightSort = + HeightNumberOrTag.fromHeightSelector(selector.heightSelector)?.getSort() ?: Sort.default if (heightSort != Sort.default) { return heightSort } @@ -159,6 +167,7 @@ class Selector { anyLabel } } + req.hasAndSelector() -> AndMatcher( Collections.unmodifiableCollection( req.andSelector.selectorsList.map { @@ -168,6 +177,7 @@ class Selector { }, ), ) + req.hasOrSelector() -> OrMatcher( Collections.unmodifiableCollection( req.orSelector.selectorsList.map { @@ -177,8 +187,29 @@ class Selector { }, ), ) + req.hasNotSelector() -> NotMatcher(convertToMatcher(req.notSelector.selector)) req.hasExistsSelector() -> ExistsMatcher(req.existsSelector.name) + + req.hasMinVersionSelector() || req.hasMaxVersionSelector() -> { + val min = if (req.hasMinVersionSelector()) { + req.minVersionSelector.version + } else { + "" + } + val max = if (req.hasMaxVersionSelector()) { + req.maxVersionSelector.version + } else { + "" + } + + if (min.isEmpty() && max.isEmpty()) { + anyLabel + } else { + RangeVersionMatcher(min, max) + } + } + else -> anyLabel } } @@ -693,4 +724,80 @@ class Selector { override fun toString(): String = "Matcher: ${describeInternal()}" } + + class RangeVersionMatcher( + private val minRawVersion: String, + private val maxRawVersion: String, + ) : LabelSelectorMatcher() { + + override fun matchesWithCause(labels: UpstreamsConfig.Labels): MatchesResponse { + val actualRawVersion = labels["client_version"] + ?: return RangeVersionResponse(minRawVersion, maxRawVersion) + + val actualSemver = runCatching { + Semver(actualRawVersion.removePrefix("v"), Semver.SemverType.STRICT) + }.getOrElse { + return RangeVersionResponse(minRawVersion, maxRawVersion) + } + + val greaterOk = + if (minRawVersion.isEmpty()) { + true + } else { + runCatching { + val min = Semver(minRawVersion.removePrefix("v"), Semver.SemverType.STRICT) + actualSemver.isGreaterThanOrEqualTo(min) + }.getOrElse { + false + } + } + val lessOk = + if (maxRawVersion.isEmpty()) { + true + } else { + runCatching { + val max = Semver(maxRawVersion.removePrefix("v"), Semver.SemverType.STRICT) + actualSemver.isLowerThanOrEqualTo(max) + }.getOrElse { + false + } + } + + return if (greaterOk && lessOk) { + Success + } else { + RangeVersionResponse(minRawVersion, maxRawVersion) + } + } + + override fun asProto(): BlockchainOuterClass.Selector = + BlockchainOuterClass.Selector.newBuilder().setAndSelector( + BlockchainOuterClass.AndSelector.newBuilder().apply { + if (minRawVersion.isNotEmpty()) { + addSelectors( + BlockchainOuterClass.Selector.newBuilder().setMinVersionSelector( + BlockchainOuterClass.MinVersionSelector.newBuilder() + .setVersion(minRawVersion) + .build(), + ), + ) + } + if (maxRawVersion.isNotEmpty()) { + addSelectors( + BlockchainOuterClass.Selector.newBuilder().setMaxVersionSelector( + BlockchainOuterClass.MaxVersionSelector.newBuilder() + .setVersion(maxRawVersion) + .build(), + ), + ) + } + }, + ).build() + + override fun describeInternal(): String = + "version range from '$minRawVersion' to '$maxRawVersion'" + + override fun toString(): String = + "Matcher: ${describeInternal()}" + } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/SubscriptionConnect.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/SubscriptionConnect.kt index 34ec08acb..4ab001ec1 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/SubscriptionConnect.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/SubscriptionConnect.kt @@ -20,7 +20,7 @@ import reactor.core.publisher.Flux /** * Note that T is supposed to be serializable as JSON */ -interface SubscriptionConnect { +interface SubscriptionConnect { fun connect(matcher: Selector.Matcher): Flux } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstream.kt index 2a091cb69..b1fdac7e0 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/Upstream.kt @@ -56,6 +56,7 @@ interface Upstream : Lifecycle { fun getUpstreamSettingsData(): UpstreamSettingsData? fun updateLowerBound(lowerBound: Long, type: LowerBoundType) fun predictLowerBound(type: LowerBoundType, timeOffsetSeconds: Long): Long + fun getAdditionalSettings(): UpstreamsConfig.AdditionalSettings? fun getChain(): Chain diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamSettingsDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamSettingsDetector.kt index 1ddd05af4..01ff6bd6c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamSettingsDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamSettingsDetector.kt @@ -1,7 +1,7 @@ package io.emeraldpay.dshackle.upstream +import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.module.kotlin.readValue import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Defaults.Companion.internalCallsTimeout import io.emeraldpay.dshackle.Global @@ -47,6 +47,21 @@ abstract class UpstreamSettingsDetector( protected abstract fun parseClientVersion(data: ByteArray): String } +/** + * Parse the response of `web3_clientVersion`-like calls leniently. Some nodes + * (e.g. Moca's Tendermint EVM) return a JSON string that contains raw, unescaped + * control characters such as line feeds. Jackson rejects those by default with + * "Illegal unquoted character", so we enable ALLOW_UNQUOTED_CONTROL_CHARS for + * this single call site. + */ +internal fun parseLenientJson(data: ByteArray): JsonNode { + val factory = Global.objectMapper.factory + factory.createParser(data).use { parser -> + parser.enable(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS) + return Global.objectMapper.readTree(parser) + } +} + abstract class BasicUpstreamSettingsDetector( private val upstream: Upstream, ) : UpstreamSettingsDetector(upstream) { @@ -54,13 +69,13 @@ abstract class BasicUpstreamSettingsDetector( protected abstract fun clientVersion(node: JsonNode): String? protected abstract fun clientType(node: JsonNode): String? - protected fun detectNodeType(): Flux?> { + protected fun detectNodeType(): Flux> { val nodeTypeRequest = nodeTypeRequest() return upstream .getIngressReader() .read(nodeTypeRequest.request) .flatMap(ChainResponse::requireResult) - .map { Global.objectMapper.readValue(it) } + .map { parseLenientJson(it) } .flatMapMany { node -> val labels = mutableListOf>() clientType(node)?.let { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamValidator.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamValidator.kt index 69d043eaa..863ec57aa 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamValidator.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/UpstreamValidator.kt @@ -71,7 +71,7 @@ enum class ValidateUpstreamSettingsResult(val priority: Int) { UPSTREAM_FATAL_SETTINGS_ERROR(2), } -interface SingleValidator { +interface SingleValidator { fun validate(onError: T): Mono } @@ -115,17 +115,21 @@ class VersionValidator( } val type = upstream.getLabels().first().getOrDefault("client_type", "unknown") val version = upstream.getLabels().first().getOrDefault("client_version", "unknown") - val rule = versionsConfig.get()!!.rules.find { it.client == type } + val chain = upstream.getChain() + val rule = versionsConfig.get()!!.rules.find { rule -> + rule.client == type && + (rule.networks.isNullOrEmpty() || rule.networks.any { chain.shortNames.contains(it) }) + } if (rule == null) { - log.info("No rules for client type $type, skipping validation for upstream ${upstream.getId()}") + log.info("No rules for client type $type on chain ${chain.chainCode}, skipping validation for upstream ${upstream.getId()}") return Mono.just(OK) } if (!rule.whitelist.isNullOrEmpty() && !rule.whitelist.contains(version)) { - log.warn("Version $version is in not in defined whitelist for $type, please change client version for upstream ${upstream.getId()}") + log.warn("Version $version is in not in defined whitelist for $type on chain ${chain.chainCode}, please change client version for upstream ${upstream.getId()}") return Mono.just(UNAVAILABLE) } if (!rule.blacklist.isNullOrEmpty() && rule.blacklist.contains(version)) { - log.warn("Version $version is in defined blacklist for $type, please change client version for upstream ${upstream.getId()}") + log.warn("Version $version is in defined blacklist for $type on chain ${chain.chainCode}, please change client version for upstream ${upstream.getId()}") return Mono.just(UNAVAILABLE) } return Mono.just(OK) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/avm/AvmChainSpecific.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/avm/AvmChainSpecific.kt new file mode 100644 index 000000000..fa49bf701 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/avm/AvmChainSpecific.kt @@ -0,0 +1,228 @@ +package io.emeraldpay.dshackle.upstream.avm + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty +import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId +import io.emeraldpay.dshackle.foundation.ChainOptions.Options +import io.emeraldpay.dshackle.reader.ChainReader +import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.GenericSingleCallValidator +import io.emeraldpay.dshackle.upstream.SingleValidator +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.UpstreamAvailability +import io.emeraldpay.dshackle.upstream.UpstreamSettingsDetector +import io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult +import io.emeraldpay.dshackle.upstream.generic.AbstractPollChainSpecific +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundService +import io.emeraldpay.dshackle.upstream.rpcclient.RestParams +import org.slf4j.LoggerFactory +import reactor.core.publisher.Mono +import java.math.BigInteger +import java.time.Instant + +object AvmChainSpecific : AbstractPollChainSpecific() { + + private val log = LoggerFactory.getLogger(AvmChainSpecific::class.java) + + override fun latestBlockRequest(): ChainRequest = + ChainRequest("GET#/v2/status", RestParams.emptyParams()) + + override fun parseBlock(data: ByteArray, upstreamId: String, api: ChainReader): Mono { + val status = Global.objectMapper.readValue(data, AvmStatus::class.java) + val round = status.lastRound + val blockRequest = ChainRequest( + "GET#/v2/blocks/$round", + RestParams( + headers = emptyList(), + queryParams = listOf("format" to "json", "header-only" to "true"), + pathParams = emptyList(), + payload = ByteArray(0), + ), + ) + return api.read(blockRequest) + .map { resp -> + val blockData = resp.getResult() + val block = Global.objectMapper.readValue(blockData, AvmBlockResult::class.java).block + BlockContainer( + height = block.round, + hash = BlockId.from(toHashBytes(block.seed ?: block.txnRoot, block.round)), + difficulty = BigInteger.ZERO, + timestamp = Instant.ofEpochSecond(block.timestamp), + full = false, + json = blockData, + parsed = block, + transactions = emptyList(), + upstreamId = upstreamId, + parentHash = BlockId.from(toHashBytes(block.previousBlockHash, block.round - 1)), + ) + } + } + + override fun getFromHeader(data: ByteArray, upstreamId: String, api: ChainReader): Mono { + throw NotImplementedError() + } + + override fun listenNewHeadsRequest(): ChainRequest { + throw NotImplementedError() + } + + override fun unsubscribeNewHeadsRequest(subId: Any): ChainRequest { + throw NotImplementedError() + } + + override fun upstreamValidators( + chain: Chain, + upstream: Upstream, + options: Options, + config: ChainConfig, + ): List> { + return listOf( + GenericSingleCallValidator( + ChainRequest("GET#/v2/status", RestParams.emptyParams()), + upstream, + ) { data -> + validate(data, upstream.getId()) + }, + ) + } + + override fun upstreamSettingsValidators( + chain: Chain, + upstream: Upstream, + options: Options, + config: ChainConfig, + ): List> { + if (chain.chainId.isBlank()) { + return emptyList() + } + return listOf( + GenericSingleCallValidator( + ChainRequest("GET#/genesis", RestParams.emptyParams()), + upstream, + ) { data -> + validateGenesis(data, chain, upstream.getId()) + }, + ) + } + + override fun upstreamSettingsDetector( + chain: Chain, + upstream: Upstream, + ): UpstreamSettingsDetector { + return AvmUpstreamSettingsDetector(upstream) + } + + override fun lowerBoundService(chain: Chain, upstream: Upstream): LowerBoundService { + return AvmLowerBoundService(chain, upstream) + } + + fun validateGenesis(data: ByteArray, chain: Chain, upstreamId: String): ValidateUpstreamSettingsResult { + val expected = chain.chainId.trim() + if (expected.isBlank()) { + return ValidateUpstreamSettingsResult.UPSTREAM_VALID + } + if (data.isEmpty()) { + log.warn("AVM node {} returned empty genesis response", upstreamId) + return ValidateUpstreamSettingsResult.UPSTREAM_SETTINGS_ERROR + } + val genesis = try { + Global.objectMapper.readValue(data, AvmGenesis::class.java) + } catch (e: Exception) { + log.warn("AVM node {} returned unparseable genesis payload: {}", upstreamId, e.message) + return ValidateUpstreamSettingsResult.UPSTREAM_SETTINGS_ERROR + } + val expectedNetwork = chain.chainName.substringAfterLast(" ").lowercase() + if (genesis.network.equals(expectedNetwork, ignoreCase = true)) { + return ValidateUpstreamSettingsResult.UPSTREAM_VALID + } + log.warn( + "AVM node {} chain mismatch: chain={} (expected network={}) but node reports network={} id={}", + upstreamId, + chain.chainName, + expectedNetwork, + genesis.network, + genesis.id, + ) + return ValidateUpstreamSettingsResult.UPSTREAM_FATAL_SETTINGS_ERROR + } + + fun validate(data: ByteArray, upstreamId: String): UpstreamAvailability { + val status = Global.objectMapper.readValue(data, AvmStatus::class.java) + if (status.lastRound == 0L) { + log.warn("AVM node {} reports no last-round", upstreamId) + return UpstreamAvailability.UNAVAILABLE + } + if (status.stoppedAtUnsupportedRound) { + log.warn("AVM node {} halted on an unsupported consensus round", upstreamId) + return UpstreamAvailability.UNAVAILABLE + } + if (status.catchupTime > 0L) { + log.warn("AVM node {} is catching up: catchupTime={}ns", upstreamId, status.catchupTime) + return UpstreamAvailability.SYNCING + } + return UpstreamAvailability.OK + } + + private fun toHashBytes(raw: String?, round: Long): ByteArray { + if (raw.isNullOrBlank()) { + return roundToBytes(round) + } + val stripped = raw.removePrefix("blk-") + return try { + java.util.Base64.getDecoder().decode(stripped) + } catch (_: IllegalArgumentException) { + try { + java.util.Base64.getUrlDecoder().decode(stripped) + } catch (_: IllegalArgumentException) { + roundToBytes(round) + } + } + } + + private fun roundToBytes(round: Long): ByteArray { + val bytes = ByteArray(32) + var value = if (round < 0) 0L else round + for (i in 0 until 8) { + bytes[31 - i] = (value and 0xff).toByte() + value = value ushr 8 + } + return bytes + } +} + +@JsonIgnoreProperties(ignoreUnknown = true) +data class AvmStatus( + @param:JsonProperty("last-round") var lastRound: Long = 0, + @param:JsonProperty("catchup-time") var catchupTime: Long = 0, + @param:JsonProperty("time-since-last-round") var timeSinceLastRound: Long = 0, + @param:JsonProperty("last-version") var lastVersion: String? = null, + @param:JsonProperty("next-version") var nextVersion: String? = null, + @param:JsonProperty("stopped-at-unsupported-round") var stoppedAtUnsupportedRound: Boolean = false, +) + +@JsonIgnoreProperties(ignoreUnknown = true) +data class AvmBlockResult( + @param:JsonProperty("block") var block: AvmBlock, +) + +@JsonIgnoreProperties(ignoreUnknown = true) +data class AvmBlock( + @param:JsonProperty("rnd") var round: Long, + @param:JsonProperty("ts") var timestamp: Long, + @param:JsonProperty("prev") var previousBlockHash: String? = null, + @param:JsonProperty("seed") var seed: String? = null, + @param:JsonProperty("txn") var txnRoot: String? = null, + @param:JsonProperty("gh") var genesisHash: String? = null, + @param:JsonProperty("gen") var genesisId: String? = null, +) + +@JsonIgnoreProperties(ignoreUnknown = true) +data class AvmGenesis( + @param:JsonProperty("network") var network: String = "", + @param:JsonProperty("id") var id: String = "", + @param:JsonProperty("proto") var proto: String = "", +) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/avm/AvmLowerBoundService.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/avm/AvmLowerBoundService.kt new file mode 100644 index 000000000..d264e55e0 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/avm/AvmLowerBoundService.kt @@ -0,0 +1,15 @@ +package io.emeraldpay.dshackle.upstream.avm + +import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundDetector +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundService + +class AvmLowerBoundService( + chain: Chain, + private val upstream: Upstream, +) : LowerBoundService(chain, upstream) { + override fun detectors(): List { + return listOf(AvmLowerBoundStateDetector(upstream)) + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/avm/AvmLowerBoundStateDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/avm/AvmLowerBoundStateDetector.kt new file mode 100644 index 000000000..6e2d1eacc --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/avm/AvmLowerBoundStateDetector.kt @@ -0,0 +1,108 @@ +package io.emeraldpay.dshackle.upstream.avm + +import com.fasterxml.jackson.databind.JsonNode +import io.emeraldpay.dshackle.Defaults +import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.upstream.ChainCallError +import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.ChainResponse +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundDetector +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType +import io.emeraldpay.dshackle.upstream.lowerbound.detector.RecursiveLowerBound +import io.emeraldpay.dshackle.upstream.rpcclient.RestParams +import org.slf4j.LoggerFactory +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono +import reactor.kotlin.core.publisher.toFlux + +class AvmLowerBoundStateDetector( + private val upstream: Upstream, +) : LowerBoundDetector(upstream.getChain()) { + + private val recursiveLowerBound = RecursiveLowerBound(upstream, LowerBoundType.STATE, notFoundErrors, lowerBounds) + + companion object { + private val log = LoggerFactory.getLogger(AvmLowerBoundStateDetector::class.java) + + val notFoundErrors = setOf( + "block not found", + "not available", + "does not have entry", + "failed to retrieve information", + "no information found", + ) + } + + override fun period(): Long = 60 + + override fun internalDetectLowerBound(): Flux { + return recursiveLowerBound.recursiveDetectLowerBound { block -> + val round = if (block <= 0L) 1L else block + val params = RestParams( + headers = emptyList(), + queryParams = emptyList(), + pathParams = listOf(round.toString()), + payload = ByteArray(0), + ) + upstream.getIngressReader() + .read(ChainRequest("GET#/v2/blocks/*/hash", params)) + .timeout(Defaults.internalCallsTimeout) + .map { response -> interpretHashResponse(round, response) } + } + .switchIfEmpty(Mono.fromSupplier { cachedOrUnknown("recursive search returned no bound") }) + .onErrorResume { err -> Mono.just(cachedOrUnknown(err.message ?: "unknown error")) } + .toFlux() + } + + override fun types(): Set = setOf(LowerBoundType.STATE, LowerBoundType.UNKNOWN) + + private fun cachedOrUnknown(reason: String): LowerBoundData { + val cached = lowerBounds.getLastBound(LowerBoundType.STATE) + if (cached != null) { + log.debug( + "AVM upstream {} lower-bound search failed ({}); retaining cached STATE={}", + upstream.getId(), + reason, + cached.lowerBound, + ) + return cached + } + log.warn( + "AVM upstream {} lower-bound search failed ({}) and no cache is available; emitting UNKNOWN", + upstream.getId(), + reason, + ) + return LowerBoundData(0, LowerBoundType.UNKNOWN) + } + + private fun interpretHashResponse(round: Long, response: ChainResponse): ChainResponse { + if (response.hasError()) { + return response + } + val raw = response.getResult() + if (raw.isEmpty()) { + return ChainResponse(null, ChainCallError(404, "empty body for round $round")) + } + val node = runCatching { Global.objectMapper.readTree(raw) }.getOrNull() ?: return response + val message = node.get("message")?.asText().orEmpty() + if (message.isNotBlank() && looksLikeNotFound(message)) { + return ChainResponse(null, ChainCallError(404, message)) + } + if (!hasHashPayload(node)) { + return ChainResponse(null, ChainCallError(404, "round $round not available")) + } + return response + } + + private fun looksLikeNotFound(message: String): Boolean { + val lower = message.lowercase() + return notFoundErrors.any { lower.contains(it) } + } + + private fun hasHashPayload(node: JsonNode): Boolean { + val hash = node.get("blockHash")?.asText().orEmpty() + return hash.isNotBlank() + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/avm/AvmUpstreamSettingsDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/avm/AvmUpstreamSettingsDetector.kt new file mode 100644 index 000000000..4ef66a5cf --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/avm/AvmUpstreamSettingsDetector.kt @@ -0,0 +1,46 @@ +package io.emeraldpay.dshackle.upstream.avm + +import com.fasterxml.jackson.databind.JsonNode +import io.emeraldpay.dshackle.upstream.BasicUpstreamSettingsDetector +import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.NodeTypeRequest +import io.emeraldpay.dshackle.upstream.UNKNOWN_CLIENT_VERSION +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.rpcclient.RestParams +import reactor.core.publisher.Flux + +class AvmUpstreamSettingsDetector( + upstream: Upstream, +) : BasicUpstreamSettingsDetector(upstream) { + + override fun internalDetectLabels(): Flux> { + return Flux.merge( + detectNodeType(), + ) + } + + override fun clientVersionRequest(): ChainRequest = + ChainRequest("GET#/versions", RestParams.emptyParams()) + + override fun parseClientVersion(data: ByteArray): String { + if (data.isEmpty()) return UNKNOWN_CLIENT_VERSION + val node = runCatching { io.emeraldpay.dshackle.Global.objectMapper.readTree(data) }.getOrNull() + ?: return UNKNOWN_CLIENT_VERSION + return clientVersion(node) ?: UNKNOWN_CLIENT_VERSION + } + + override fun nodeTypeRequest(): NodeTypeRequest = NodeTypeRequest(clientVersionRequest()) + + override fun clientType(node: JsonNode): String = "algod" + + override fun clientVersion(node: JsonNode): String { + val build = node.get("build") ?: return UNKNOWN_CLIENT_VERSION + val major = build.get("major")?.asInt(-1) ?: -1 + val minor = build.get("minor")?.asInt(-1) ?: -1 + val patch = build.get("build_number")?.asInt(-1) ?: -1 + if (major < 0 || minor < 0 || patch < 0) { + return UNKNOWN_CLIENT_VERSION + } + return "$major.$minor.$patch" + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/aztec/AztecChainSpecific.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/aztec/AztecChainSpecific.kt new file mode 100644 index 000000000..c4bf14b96 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/aztec/AztecChainSpecific.kt @@ -0,0 +1,291 @@ +package io.emeraldpay.dshackle.upstream.aztec + +import com.fasterxml.jackson.databind.JsonNode +import com.github.benmanes.caffeine.cache.Caffeine +import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig +import io.emeraldpay.dshackle.data.BlockContainer +import io.emeraldpay.dshackle.data.BlockId +import io.emeraldpay.dshackle.foundation.ChainOptions.Options +import io.emeraldpay.dshackle.reader.ChainReader +import io.emeraldpay.dshackle.upstream.ChainException +import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.GenericSingleCallValidator +import io.emeraldpay.dshackle.upstream.SingleValidator +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.UpstreamAvailability +import io.emeraldpay.dshackle.upstream.UpstreamSettingsDetector +import io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult +import io.emeraldpay.dshackle.upstream.generic.AbstractPollChainSpecific +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundService +import io.emeraldpay.dshackle.upstream.rpcclient.ListParams +import org.slf4j.LoggerFactory +import reactor.core.publisher.Mono +import java.math.BigInteger +import java.time.Duration +import java.time.Instant + +object AztecChainSpecific : AbstractPollChainSpecific() { + private val log = LoggerFactory.getLogger(AztecChainSpecific::class.java) + + private const val METHOD_NOT_FOUND = -32601 + + // Aztec v5 (v5.0.0-rc.1) renamed the tips RPC: node_getL2Tips -> node_getChainTips. + // Older nodes (incl. current mainnet) expose only the legacy name; v5+ nodes expose + // only the new one. We probe the legacy method first (most upstreams are still on it) + // and fall back to the new one on "method not found", remembering the working method + // per upstream so we stop probing the dead one on every poll. + private val LEGACY_TIPS_REQUEST = ChainRequest("node_getL2Tips", ListParams()) + private val CHAIN_TIPS_REQUEST = ChainRequest("node_getChainTips", ListParams()) + + // Bounded so per-upstream entries can't accumulate without limit (e.g. across config + // reloads): a hard size cap plus idle expiry evict stale ids, and an evicted entry just + // costs one re-probe. Only upstreams that actually fall back take a slot — legacy-only + // ones keep using the default and never populate it. + private val workingTipsRequest = Caffeine.newBuilder() + .maximumSize(1024) + .expireAfterAccess(Duration.ofHours(1)) + .build() + + // The tips response reshaped between Aztec versions: + // v3 (and earlier): {proposed: {number, hash}, proven: {number, hash}, checkpointed: {number, hash}} + // v4/v5: proven/finalized/checkpointed each became {block: {number, hash}, checkpoint: {number, hash}} + // proposed stayed flat across all versions. We look at the flat path first and fall back + // to the nested path so an upstream on any version is parsed correctly. + private val PROPOSED_NUMBER = arrayOf("proposed.number", "proposed.block.number") + private val PROPOSED_HASH = arrayOf("proposed.hash", "proposed.block.hash") + + override fun parseBlock(data: ByteArray, upstreamId: String, api: ChainReader): Mono { + val root = Global.objectMapper.readTree(data) + val height = parseLong(findNode(root, *PROPOSED_NUMBER)) ?: 0L + val hashValue = parseText(findNode(root, *PROPOSED_HASH)) + + return Mono.just( + BlockContainer( + height = height, + hash = BlockId.from(hashValue ?: "0x0"), + difficulty = BigInteger.ZERO, + timestamp = Instant.EPOCH, + full = false, + json = data, + parsed = root, + transactions = emptyList(), + upstreamId = upstreamId, + parentHash = null, + ), + ) + } + + // Aztec is HTTP-poll only; getFromHeader / listenNewHeadsRequest / + // unsubscribeNewHeadsRequest are reachable only from GenericWsHead, which is + // never wired for a polling chain. Fail fast so a misconfigured WS connector + // surfaces immediately instead of silently producing height=0 blocks from a + // header-shaped event being parsed as the L2Tips response. + override fun getFromHeader(data: ByteArray, upstreamId: String, api: ChainReader): Mono { + throw UnsupportedOperationException("Aztec does not support websocket subscriptions") + } + + override fun listenNewHeadsRequest(): ChainRequest { + throw UnsupportedOperationException("Aztec does not support websocket subscriptions") + } + + override fun unsubscribeNewHeadsRequest(subId: Any): ChainRequest { + throw UnsupportedOperationException("Aztec does not support websocket subscriptions") + } + + override fun upstreamValidators( + chain: Chain, + upstream: Upstream, + options: Options, + config: ChainConfig, + ): List> { + return listOf( + GenericSingleCallValidator( + ChainRequest("node_isReady", ListParams()), + upstream, + ) { data -> + val raw = Global.objectMapper.readTree(data) + val ready = when { + raw.isBoolean -> raw.asBoolean() + raw.isTextual -> raw.asText().equals("true", ignoreCase = true) + else -> raw.asBoolean(false) + } + if (ready) { + UpstreamAvailability.OK + } else { + log.warn("Aztec node {} reports not ready", upstream.getId()) + UpstreamAvailability.SYNCING + } + }, + ) + } + + override fun upstreamSettingsValidators( + chain: Chain, + upstream: Upstream, + options: Options, + config: ChainConfig, + ): List> { + if (chain.chainId.isBlank()) { + return emptyList() + } + return listOf( + GenericSingleCallValidator( + ChainRequest("node_getChainId", ListParams()), + upstream, + ) { data -> + validateChainId(data, chain, upstream.getId()) + }, + ) + } + + override fun lowerBoundService(chain: Chain, upstream: Upstream): LowerBoundService { + return AztecLowerBoundService(chain, upstream) + } + + override fun latestBlockRequest(): ChainRequest = LEGACY_TIPS_REQUEST + + // Try the per-upstream remembered method (legacy by default); on "method not found" + // fall back to the other one and remember whichever succeeds, so subsequent polls go + // straight to the working method. Any other error propagates as before. + override fun getLatestBlock(api: ChainReader, upstreamId: String): Mono { + val preferred = workingTipsRequest.getIfPresent(upstreamId) ?: LEGACY_TIPS_REQUEST + val fallback = if (preferred === LEGACY_TIPS_REQUEST) CHAIN_TIPS_REQUEST else LEGACY_TIPS_REQUEST + return fetchTips(api, upstreamId, preferred) + .onErrorResume { err -> + if (isMethodNotFound(err)) { + log.info( + "Aztec upstream {} does not support {}, falling back to {}", + upstreamId, + preferred.method, + fallback.method, + ) + fetchTips(api, upstreamId, fallback) + .doOnNext { workingTipsRequest.put(upstreamId, fallback) } + } else { + Mono.error(err) + } + } + } + + private fun fetchTips(api: ChainReader, upstreamId: String, request: ChainRequest): Mono { + return api.read(request).flatMap { + parseBlock(it.getResult(), upstreamId, api) + } + } + + private fun isMethodNotFound(err: Throwable): Boolean { + if (err is ChainException && err.error.code == METHOD_NOT_FOUND) { + return true + } + return err.message?.contains("method not found", ignoreCase = true) ?: false + } + + override fun upstreamSettingsDetector( + chain: Chain, + upstream: Upstream, + ): UpstreamSettingsDetector { + return AztecUpstreamSettingsDetector(upstream) + } + + fun validateChainId(data: ByteArray, chain: Chain, upstreamId: String): ValidateUpstreamSettingsResult { + if (data.isEmpty() || String(data).isBlank()) { + log.warn("Aztec node {} returned empty chain id response", upstreamId) + return ValidateUpstreamSettingsResult.UPSTREAM_SETTINGS_ERROR + } + val raw = try { + Global.objectMapper.readTree(data) + } catch (e: Exception) { + log.warn("Aztec node {} returned unparseable chain id payload: {}", upstreamId, e.message) + return ValidateUpstreamSettingsResult.UPSTREAM_SETTINGS_ERROR + } + if (raw == null || raw.isNull) { + log.warn("Aztec node {} returned null chain id", upstreamId) + return ValidateUpstreamSettingsResult.UPSTREAM_SETTINGS_ERROR + } + val reported = parseChainId(raw) + if (reported.isNullOrBlank()) { + log.warn("Aztec node {} returned no chain id ({})", upstreamId, raw) + return ValidateUpstreamSettingsResult.UPSTREAM_SETTINGS_ERROR + } + val expected = chain.chainId + return if (chainIdMatches(reported, expected)) { + ValidateUpstreamSettingsResult.UPSTREAM_VALID + } else { + log.warn( + "Aztec node {} chain id mismatch: reported={} expected={}", + upstreamId, + reported, + expected, + ) + ValidateUpstreamSettingsResult.UPSTREAM_FATAL_SETTINGS_ERROR + } + } + + private fun parseChainId(node: JsonNode): String? { + return when { + node.isNumber -> node.asLong().toString() + node.isTextual -> node.asText().trim().ifBlank { null } + else -> null + } + } + + fun chainIdMatches(reported: String, expected: String): Boolean { + val normalize: (String) -> String = { value -> + val trimmed = value.trim().lowercase() + val withoutPrefix = if (trimmed.startsWith("0x")) trimmed.substring(2) else trimmed + // Aztec returns chain id as a decimal number; configured chainId may be hex. + // Compare numerically when both sides parse, fall back to literal match. + withoutPrefix.trimStart('0').ifEmpty { "0" } + } + val a = normalize(reported) + val b = normalize(expected) + if (a == b) return true + val aNum = runCatching { BigInteger(a, if (reported.lowercase().startsWith("0x")) 16 else 10) }.getOrNull() + val bNum = runCatching { BigInteger(b, if (expected.lowercase().startsWith("0x")) 16 else 10) }.getOrNull() + return aNum != null && bNum != null && aNum == bNum + } + + private fun findNode(root: JsonNode, vararg paths: String): JsonNode? { + for (path in paths) { + var current: JsonNode? = root + for (part in path.split(".")) { + current = current?.get(part) + if (current == null || current.isMissingNode) { + break + } + } + if (current != null && !current.isMissingNode && !current.isNull) { + return current + } + } + return null + } + + private fun parseText(node: JsonNode?): String? { + if (node == null || node.isNull || node.isMissingNode) { + return null + } + return node.asText().ifBlank { null } + } + + private fun parseLong(node: JsonNode?): Long? { + if (node == null || node.isNull || node.isMissingNode) { + return null + } + return when { + node.isNumber -> node.asLong() + node.isTextual -> parseNumericString(node.asText()) + else -> null + } + } + + private fun parseNumericString(value: String): Long? { + val trimmed = value.trim() + if (trimmed.isEmpty()) return null + val isHex = trimmed.startsWith("0x") || trimmed.startsWith("0X") + val raw = if (isHex) trimmed.substring(2) else trimmed + return runCatching { BigInteger(raw, if (isHex) 16 else 10).toLong() }.getOrNull() + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/aztec/AztecLowerBoundService.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/aztec/AztecLowerBoundService.kt new file mode 100644 index 000000000..874ec8821 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/aztec/AztecLowerBoundService.kt @@ -0,0 +1,15 @@ +package io.emeraldpay.dshackle.upstream.aztec + +import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundDetector +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundService + +class AztecLowerBoundService( + chain: Chain, + private val upstream: Upstream, +) : LowerBoundService(chain, upstream) { + override fun detectors(): List { + return listOf(AztecLowerBoundStateDetector(upstream)) + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/aztec/AztecLowerBoundStateDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/aztec/AztecLowerBoundStateDetector.kt new file mode 100644 index 000000000..8f6bbe6d1 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/aztec/AztecLowerBoundStateDetector.kt @@ -0,0 +1,91 @@ +package io.emeraldpay.dshackle.upstream.aztec + +import io.emeraldpay.dshackle.Defaults +import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.ChainResponse +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundDetector +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType +import io.emeraldpay.dshackle.upstream.rpcclient.ListParams +import org.slf4j.LoggerFactory +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono + +/** + * Detects the lowest L2 block for which the upstream still has state available. + * + * Aztec exposes `node_getWorldStateSyncStatus`, whose response contains + * `oldestHistoricBlockNumber` - the prune boundary kept by the world-state + * synchronizer. One RPC call per refresh, no binary search needed. + * + * The bound is a sliding window: it monotonically increases as the node prunes + * older blocks (configured by `historyToKeep`). The base detector + LowerBounds + * already model this correctly via linear regression over the most recent + * three samples, so we only need to feed it real readings. + * + * Failure handling: + * - On RPC error / unparseable response we re-emit the cached LowerBoundData + * unchanged (same instance / same timestamp) so `updateBound`'s + * `newBound.timestamp != lastBound.timestamp` guard skips the regression + * update and the cached bound stays put. + * - If there is no cached value yet, we emit nothing. We do **not** synthesize + * `STATE=1`: that would falsely advertise full archive history to the + * router. The next refresh tick will retry. + */ +class AztecLowerBoundStateDetector( + private val upstream: Upstream, +) : LowerBoundDetector(upstream.getChain()) { + + companion object { + private val log = LoggerFactory.getLogger(AztecLowerBoundStateDetector::class.java) + } + + override fun period(): Long = 5 + + override fun types(): Set = setOf(LowerBoundType.STATE) + + override fun internalDetectLowerBound(): Flux { + return upstream.getIngressReader() + .read(ChainRequest("node_getWorldStateSyncStatus", ListParams())) + .timeout(Defaults.internalCallsTimeout) + .flatMap(ChainResponse::requireResult) + .flatMap { data -> parseOldestHistoric(data) } + .onErrorResume { err -> retainCachedOrSkip(err.message) } + .flux() + } + + private fun parseOldestHistoric(data: ByteArray): Mono { + val raw = Global.objectMapper.readTree(data) + val node = raw.get("oldestHistoricBlockNumber") + if (node != null && !node.isNull && node.isNumber) { + return Mono.just(LowerBoundData(node.asLong().coerceAtLeast(1L), LowerBoundType.STATE)) + } + return retainCachedOrSkip("missing oldestHistoricBlockNumber") + } + + private fun retainCachedOrSkip(reason: String?): Mono { + val cached = lowerBounds.getLastBound(LowerBoundType.STATE) + if (cached != null) { + log.debug( + "Aztec upstream {} world state sync status unavailable; retaining cached STATE={}: {}", + upstream.getId(), + cached.lowerBound, + reason, + ) + // Same instance (same timestamp) so updateBound becomes a no-op + // and the linear-regression coefficients are preserved. + return Mono.just(cached) + } + // No cache and a malformed first response: best we can do is emit a + // synthetic archive bound. This is the only place STATE=1 is invented; + // see the trade-off in the class KDoc. + log.warn( + "Aztec upstream {} returned no oldestHistoricBlockNumber and we have no cached STATE: {}", + upstream.getId(), + reason, + ) + return Mono.just(LowerBoundData(0, LowerBoundType.UNKNOWN)) + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/aztec/AztecUpstreamSettingsDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/aztec/AztecUpstreamSettingsDetector.kt new file mode 100644 index 000000000..bb279f622 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/aztec/AztecUpstreamSettingsDetector.kt @@ -0,0 +1,75 @@ +package io.emeraldpay.dshackle.upstream.aztec + +import com.fasterxml.jackson.databind.JsonNode +import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.upstream.BasicUpstreamSettingsDetector +import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.NodeTypeRequest +import io.emeraldpay.dshackle.upstream.UNKNOWN_CLIENT_VERSION +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.rpcclient.ListParams +import reactor.core.publisher.Flux + +class AztecUpstreamSettingsDetector( + upstream: Upstream, +) : BasicUpstreamSettingsDetector(upstream) { + + override fun internalDetectLabels(): Flux> { + return Flux.merge( + detectNodeType(), + ) + } + + override fun clientVersionRequest(): ChainRequest { + return ChainRequest("node_getNodeVersion", ListParams()) + } + + /** + * node_getNodeVersion typically returns a JSON string ("v1.2.3"), but a few + * builds wrap it in an object like {"nodeVersion": "v1.2.3", "l1ChainId": ...} + * - the same payload node_getNodeInfo returns. Try parsing the payload as JSON + * first and reuse [clientVersion] so detectClientVersion() and detectLabels() + * agree on the version they extract; fall back to a literal trim/quote-strip + * for raw non-JSON answers, and treat anything that boils down to JSON null / + * the literal string "null" as UNKNOWN_CLIENT_VERSION. + */ + override fun parseClientVersion(data: ByteArray): String { + val parsed = runCatching { Global.objectMapper.readTree(data) }.getOrNull() + if (parsed != null && !parsed.isNull && !parsed.isMissingNode) { + // JSON parsed successfully; trust the structured extractor. If it + // can't find a usable version we report UNKNOWN rather than falling + // through to literal-strip (which would happily return the whole + // JSON-encoded object string). + return clientVersion(parsed) + } + + var version = String(data).trim() + if (version.startsWith("\"") && version.endsWith("\"") && version.length >= 2) { + version = version.substring(1, version.length - 1) + } + if (version.isBlank() || version.equals("null", ignoreCase = true)) { + return UNKNOWN_CLIENT_VERSION + } + if (version.startsWith("v") || version.startsWith("V")) { + version = version.substring(1) + } + return version.ifBlank { UNKNOWN_CLIENT_VERSION } + } + + override fun nodeTypeRequest(): NodeTypeRequest = NodeTypeRequest(clientVersionRequest()) + + override fun clientType(node: JsonNode): String = "aztec" + + override fun clientVersion(node: JsonNode): String { + val raw = when { + node.isTextual -> node.asText() + node.isObject -> node.get("nodeVersion")?.asText().orEmpty() + else -> "" + }.trim() + if (raw.isEmpty() || raw.equals("null", ignoreCase = true)) { + return UNKNOWN_CLIENT_VERSION + } + val stripped = if (raw.startsWith("v") || raw.startsWith("V")) raw.substring(1) else raw + return stripped.ifBlank { UNKNOWN_CLIENT_VERSION } + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainLowerBoundBlobDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainLowerBoundBlobDetector.kt index 6e83959d6..96b502c8d 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainLowerBoundBlobDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainLowerBoundBlobDetector.kt @@ -55,12 +55,16 @@ class BeaconChainLowerBoundBlobDetector( private fun parseHeadersResponse(data: ByteArray): ChainResponse { val node = Global.objectMapper.readValue(data) - if (node.get("code") != null && node.get("message") != null && node.get("code").textValue() == "404") { - return ChainResponse(null, ChainCallError(node.get("code").asInt(), node.get("message").asText(), node.get("message").asText())) + val codeNode = node.get("code") + val messageNode = node.get("message") + val is404 = codeNode != null && (codeNode.asInt() == 404 || codeNode.asText() == "404") + if (is404 && messageNode != null) { + return ChainResponse(null, ChainCallError(404, messageNode.asText(), messageNode.asText())) } - if (node.get("data").toString() == "[]") { + val dataNode = node.get("data") + if (dataNode == null || dataNode.isNull || (dataNode.isArray && dataNode.size() == 0)) { return ChainResponse(null, ChainCallError(404, notFoundError)) } - return ChainResponse(node.get("data").toString().toByteArray(), null) + return ChainResponse(dataNode.toString().toByteArray(), null) } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainLowerBoundBlockDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainLowerBoundBlockDetector.kt index d44f2c5c0..d38407739 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainLowerBoundBlockDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainLowerBoundBlockDetector.kt @@ -57,8 +57,11 @@ class BeaconChainLowerBoundBlockDetector( private fun parseHeadersResponse(data: ByteArray): ChainResponse { val node = Global.objectMapper.readValue(data) - if (node.get("code") != null && node.get("message") != null && node.get("code").textValue() == "404") { - return ChainResponse(null, ChainCallError(node.get("code").asInt(), node.get("message").asText(), node.get("message").asText())) + val codeNode = node.get("code") + val messageNode = node.get("message") + val is404 = codeNode != null && (codeNode.asInt() == 404 || codeNode.asText() == "404") + if (is404 && messageNode != null) { + return ChainResponse(null, ChainCallError(404, messageNode.asText(), messageNode.asText())) } if (node.get("data")?.get("message")?.get("slot") != null) { return ChainResponse(node.get("data").toString().toByteArray(), null) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainLowerBoundEpochDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainLowerBoundEpochDetector.kt index 4be970207..bd2fa2f13 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainLowerBoundEpochDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainLowerBoundEpochDetector.kt @@ -80,10 +80,12 @@ class BeaconChainLowerBoundEpochDetector( private fun parseHeadersResponse(data: ByteArray): ChainResponse { val node = Global.objectMapper.readValue(data) - if (node.get("code") != null && node.get("message") != null && node.get("code").textValue() == "404") { - return ChainResponse(null, ChainCallError(node.get("code").asInt(), node.get("message").asText(), node.get("message").asText())) + val codeNode = node.get("code") + val messageNode = node.get("message") + val is404 = codeNode != null && (codeNode.asInt() == 404 || codeNode.asText() == "404") + if (is404 && messageNode != null) { + return ChainResponse(null, ChainCallError(404, messageNode.asText(), messageNode.asText())) } - val jsonData = node.get("data") if (jsonData != null) { val str = jsonData.toString() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainLowerBoundStateDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainLowerBoundStateDetector.kt index e84fdca50..d68b1c1fa 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainLowerBoundStateDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainLowerBoundStateDetector.kt @@ -54,10 +54,12 @@ class BeaconChainLowerBoundStateDetector( private fun parseHeadersResponse(data: ByteArray): ChainResponse { val node = Global.objectMapper.readValue(data) - if (node.get("code") != null && node.get("message") != null && node.get("code").textValue() == "404") { - return ChainResponse(null, ChainCallError(node.get("code").asInt(), node.get("message").asText(), node.get("message").asText())) + val codeNode = node.get("code") + val messageNode = node.get("message") + val is404 = codeNode != null && (codeNode.asInt() == 404 || codeNode.asText() == "404") + if (is404 && messageNode != null) { + return ChainResponse(null, ChainCallError(404, messageNode.asText(), messageNode.asText())) } - val jsonData = node.get("data") if (jsonData != null) { val str = jsonData.toString() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainSpecific.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainSpecific.kt index b7f335242..c756cc29f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainSpecific.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/beaconchain/BeaconChainSpecific.kt @@ -121,6 +121,16 @@ object BeaconChainSpecific : AbstractPollChainSpecific() { override fun lowerBoundService(chain: Chain, upstream: Upstream): LowerBoundService { return BeaconChainLowerBoundService(chain, upstream) } + + override fun getResponseHeadersToForward(): List = listOf( + "Content-Type", + "Eth-Consensus-Version", + "Eth-Consensus-Finalized", + "Eth-Execution-Optimistic", + "Eth-Execution-Payload-Blinded", + "Eth-Execution-Payload-Value", + "Eth-Consensus-Block-Value", + ) } data class BeaconChainBlockHeader( @@ -144,23 +154,23 @@ class BeaconChainBlockHeaderDeserializer : JsonDeserializer castedRead(req: ChainRequest, clazz: Class): Mono { + fun castedRead(req: ChainRequest, clazz: Class): Mono { return upstreams.getDirectApi(Selector.empty).flatMap { api -> api.read(req) .flatMap(ChainResponse::requireResult) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcHead.kt index af7624552..c5ad9c634 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/BitcoinRpcHead.kt @@ -71,8 +71,8 @@ class BitcoinRpcHead( .map(extractBlock::extract) .timeout(Defaults.timeout, Mono.error(Exception("Block data is not received"))) } - .onErrorContinue { err, _ -> - log.debug("RPC error ${err.message}") + .onErrorContinue { err, _: Any? -> + log.warn("RPC error: ${err.message}") } refreshSubscription = super.follow(base) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClient.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClient.kt index f9364dbae..7c5ad1a61 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClient.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClient.kt @@ -22,7 +22,7 @@ import io.netty.handler.codec.http.HttpHeaderNames import io.netty.handler.codec.http.HttpHeaders import io.netty.handler.ssl.SslContextBuilder import io.netty.resolver.DefaultAddressResolverGroup -import org.bitcoinj.core.Address +import org.bitcoinj.base.Address import org.slf4j.LoggerFactory import reactor.core.publisher.Mono import reactor.netty.http.client.HttpClient @@ -34,10 +34,11 @@ import java.security.cert.X509Certificate import java.util.Base64 import java.util.function.Consumer -class EsploraClient( +class EsploraClient @JvmOverloads constructor( private val url: URI, basicAuth: AuthConfig.ClientBasicAuth? = null, tlsCAAuth: ByteArray? = null, + bearerAuth: AuthConfig.ClientBearerAuth? = null, ) { companion object { @@ -62,6 +63,13 @@ class EsploraClient( build = build.headers(headers) } + if (basicAuth == null) { + bearerAuth?.let { auth -> + val headers = Consumer { h: HttpHeaders -> h.add(HttpHeaderNames.AUTHORIZATION, "Bearer ${auth.token}") } + build = build.headers(headers) + } + } + tlsCAAuth?.let { auth -> val cf = CertificateFactory.getInstance("X.509") val cert = cf.generateCertificate(ByteArrayInputStream(auth)) as X509Certificate diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraUnspentReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraUnspentReader.kt index 58d39d673..4575c2d6e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraUnspentReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/EsploraUnspentReader.kt @@ -18,7 +18,7 @@ package io.emeraldpay.dshackle.upstream.bitcoin import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.bitcoin.data.EsploraUnspent import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent -import org.bitcoinj.core.Address +import org.bitcoinj.base.Address import org.slf4j.LoggerFactory import reactor.core.publisher.Mono import java.util.function.Function diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/LocalCallRouter.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/LocalCallRouter.kt index 5617eaccd..77c58c116 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/LocalCallRouter.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/LocalCallRouter.kt @@ -26,7 +26,8 @@ import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError import io.emeraldpay.dshackle.upstream.rpcclient.ListParams -import org.bitcoinj.core.Address +import org.bitcoinj.base.Address +import org.bitcoinj.base.AddressParser import org.slf4j.LoggerFactory import reactor.core.publisher.Mono @@ -70,7 +71,7 @@ class LocalCallRouter( } val addresses = key.params.list[2] if (addresses is List<*> && addresses.size > 0) { - val address = addresses[0].toString().let { Address.fromString(null, it) } + val address = addresses[0].toString().let { AddressParser.getDefault().parseAddress(it) } return reader.listUnspent(address).map { val rpc = it.map(convertUnspent(address)) val json = Global.objectMapper.writeValueAsBytes(rpc) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RemoteUnspentReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RemoteUnspentReader.kt index 28e836b83..3331a03c1 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RemoteUnspentReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RemoteUnspentReader.kt @@ -3,7 +3,7 @@ package io.emeraldpay.dshackle.upstream.bitcoin import io.emeraldpay.dshackle.upstream.Capability import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent -import org.bitcoinj.core.Address +import org.bitcoinj.base.Address import reactor.core.publisher.Mono class RemoteUnspentReader( diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReader.kt index ecc53cd8b..3dca232bf 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReader.kt @@ -24,7 +24,7 @@ import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.bitcoin.data.RpcUnspent import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent import io.emeraldpay.dshackle.upstream.rpcclient.ListParams -import org.bitcoinj.core.Address +import org.bitcoinj.base.Address import org.slf4j.LoggerFactory import reactor.core.publisher.Mono diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/UnspentReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/UnspentReader.kt index d922c5877..cb1c88cbd 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/UnspentReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/UnspentReader.kt @@ -17,6 +17,6 @@ package io.emeraldpay.dshackle.upstream.bitcoin import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.bitcoin.data.SimpleUnspent -import org.bitcoinj.core.Address +import org.bitcoinj.base.Address interface UnspentReader : Reader> diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddresses.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddresses.kt index 4f55f8767..64d5b3af1 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddresses.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddresses.kt @@ -15,15 +15,16 @@ */ package io.emeraldpay.dshackle.upstream.bitcoin -import org.bitcoinj.core.Address -import org.bitcoinj.core.ECKey +import org.bitcoinj.base.Address +import org.bitcoinj.base.BitcoinNetwork +import org.bitcoinj.base.LegacyAddress +import org.bitcoinj.base.ScriptType +import org.bitcoinj.base.SegwitAddress import org.bitcoinj.core.NetworkParameters import org.bitcoinj.crypto.ChildNumber import org.bitcoinj.crypto.DeterministicKey +import org.bitcoinj.crypto.ECKey import org.bitcoinj.crypto.HDKeyDerivation -import org.bitcoinj.params.MainNetParams -import org.bitcoinj.params.TestNet3Params -import org.bitcoinj.script.Script import org.slf4j.LoggerFactory import reactor.core.publisher.Flux import reactor.util.function.Tuples @@ -35,8 +36,6 @@ open class XpubAddresses( companion object { private val log = LoggerFactory.getLogger(XpubAddresses::class.java) - private val MAINNET = MainNetParams() - private val TESTNET = TestNet3Params() private val INACTIVE_LIMIT = 20 } @@ -45,39 +44,45 @@ open class XpubAddresses( // https://electrum.readthedocs.io/en/latest/xpub_version_bytes.html // TODO doesn't support SH keys right now. should? val prefix = xpub.substring(0, 4) - val type: Script.ScriptType - val network: NetworkParameters + val type: ScriptType + val params: NetworkParameters when (prefix) { "xpub" -> { - type = Script.ScriptType.P2PKH - network = MAINNET + type = ScriptType.P2PKH + params = NetworkParameters.of(BitcoinNetwork.MAINNET) } "zpub" -> { - type = Script.ScriptType.P2WPKH - network = MAINNET + type = ScriptType.P2WPKH + params = NetworkParameters.of(BitcoinNetwork.MAINNET) } "tpub" -> { - type = Script.ScriptType.P2PKH - network = TESTNET + type = ScriptType.P2PKH + params = NetworkParameters.of(BitcoinNetwork.TESTNET) } "vpub" -> { - type = Script.ScriptType.P2WPKH - network = TESTNET + type = ScriptType.P2WPKH + params = NetworkParameters.of(BitcoinNetwork.TESTNET) } else -> return Flux.error(IllegalArgumentException("Unsupported type: $prefix")) } val key: DeterministicKey try { - key = DeterministicKey.deserializeB58(xpub, network) + key = DeterministicKey.deserializeB58(xpub, params) } catch (t: Throwable) { return Flux.error(t) } return Flux.range(start, limit) .map { HDKeyDerivation.deriveChildKey(key, ChildNumber(it, false)) } - .map { Address.fromKey(network, ECKey.fromPublicOnly(it.pubKeyPoint), type) } + .map { addressFromKey(params, ECKey.fromPublicOnly(it.pubKey), type) } + } + + private fun addressFromKey(params: NetworkParameters, key: ECKey, type: ScriptType): Address = when (type) { + ScriptType.P2PKH -> LegacyAddress.fromKey(params, key) + ScriptType.P2WPKH -> SegwitAddress.fromKey(params, key) + else -> throw IllegalArgumentException("Unsupported script type: $type") } open fun activeAddresses(xpub: String, start: Int, limit: Int): Flux
{ diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultAvmMethods.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultAvmMethods.kt new file mode 100644 index 000000000..f75455a70 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultAvmMethods.kt @@ -0,0 +1,150 @@ +/** + * Copyright (c) 2020 EmeraldPay, Inc + * Copyright (c) 2019 ETCDEV GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.emeraldpay.dshackle.upstream.calls + +import io.emeraldpay.dshackle.quorum.AlwaysQuorum +import io.emeraldpay.dshackle.quorum.BroadcastQuorum +import io.emeraldpay.dshackle.quorum.CallQuorum +import io.emeraldpay.dshackle.quorum.NotNullQuorum +import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException + +/** + * Default configuration for AVM (Algorand Virtual Machine) REST API, matching + * the algod OpenAPI spec. Method identifiers use the `VERB#/path` convention + * consumed by dshackle's REST HTTP reader. + */ +class DefaultAvmMethods : CallMethods { + + // Root-level common endpoints (not under /v2/*) + private val commonMethods = setOf( + getMethod("/genesis"), + getMethod("/health"), + getMethod("/ready"), + getMethod("/metrics"), + getMethod("/versions"), + getMethod("/swagger.json"), + ) + + // Node / ledger / blocks read endpoints under /v2/* + private val nodeMethods = setOf( + getMethod("/v2/status"), + getMethod("/v2/status/wait-for-block-after/*"), + getMethod("/v2/ledger/supply"), + getMethod("/v2/ledger/sync"), + getMethod("/v2/blocks/*"), + getMethod("/v2/blocks/*/hash"), + getMethod("/v2/blocks/*/txids"), + getMethod("/v2/blocks/*/logs"), + getMethod("/v2/blocks/*/lightheader/proof"), + getMethod("/v2/blocks/*/transactions/*/proof"), + getMethod("/v2/stateproofs/*"), + getMethod("/v2/deltas/*"), + getMethod("/v2/deltas/*/txn/group"), + getMethod("/v2/deltas/txn/group/*"), + ) + + private val accountMethods = setOf( + getMethod("/v2/accounts/*"), + getMethod("/v2/accounts/*/assets"), + getMethod("/v2/accounts/*/assets/*"), + getMethod("/v2/accounts/*/applications/*"), + getMethod("/v2/accounts/*/transactions/pending"), + getMethod("/v2/applications/*"), + getMethod("/v2/applications/*/box"), + getMethod("/v2/applications/*/boxes"), + getMethod("/v2/assets/*"), + ) + + private val transactionReadMethods = setOf( + getMethod("/v2/transactions/params"), + getMethod("/v2/transactions/pending"), + getMethod("/v2/transactions/pending/*"), + ) + + private val sendMethods = setOf( + postMethod("/v2/transactions"), + postMethod("/v2/transactions/async"), + ) + + private val computeMethods = setOf( + postMethod("/v2/transactions/simulate"), + postMethod("/v2/teal/compile"), + postMethod("/v2/teal/disassemble"), + postMethod("/v2/teal/dryrun"), + ) + + private val allowedMethods: Set = + commonMethods + nodeMethods + accountMethods + transactionReadMethods + sendMethods + computeMethods + + // Paths that look up a specific resource by id/round and should reject + // empty/404 answers via NotNullQuorum, so a single missing-replica + // response doesn't silently beat valid ones from other upstreams. + private val notNullReadMethods: Set = setOf( + getMethod("/v2/blocks/*"), + getMethod("/v2/blocks/*/hash"), + getMethod("/v2/blocks/*/txids"), + getMethod("/v2/blocks/*/logs"), + getMethod("/v2/blocks/*/lightheader/proof"), + getMethod("/v2/blocks/*/transactions/*/proof"), + getMethod("/v2/accounts/*"), + getMethod("/v2/accounts/*/assets/*"), + getMethod("/v2/accounts/*/applications/*"), + getMethod("/v2/applications/*"), + getMethod("/v2/applications/*/box"), + getMethod("/v2/applications/*/boxes"), + getMethod("/v2/assets/*"), + getMethod("/v2/transactions/pending/*"), + getMethod("/v2/stateproofs/*"), + getMethod("/v2/deltas/*"), + getMethod("/v2/deltas/*/txn/group"), + getMethod("/v2/deltas/txn/group/*"), + ) + + override fun createQuorumFor(method: String): CallQuorum { + return when { + sendMethods.contains(method) -> BroadcastQuorum() + notNullReadMethods.contains(method) -> NotNullQuorum() + else -> AlwaysQuorum() + } + } + + override fun isCallable(method: String): Boolean { + return allowedMethods.contains(method) + } + + override fun isHardcoded(method: String): Boolean { + return false + } + + override fun executeHardcoded(method: String): ByteArray { + throw RpcException(-32601, "Method not found") + } + + override fun getGroupMethods(groupName: String): Set = + when (groupName) { + "default" -> getSupportedMethods() + else -> emptySet() + } + + override fun getSupportedMethods(): Set { + return allowedMethods.toSortedSet() + } + + private fun getMethod(path: String) = "GET#$path" + + private fun postMethod(path: String) = "POST#$path" +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultAztecMethods.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultAztecMethods.kt new file mode 100644 index 000000000..d2d89a081 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultAztecMethods.kt @@ -0,0 +1,148 @@ +package io.emeraldpay.dshackle.upstream.calls + +import io.emeraldpay.dshackle.quorum.AlwaysQuorum +import io.emeraldpay.dshackle.quorum.BroadcastQuorum +import io.emeraldpay.dshackle.quorum.CallQuorum +import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException + +class DefaultAztecMethods : CallMethods { + + private val broadcast = setOf( + "node_sendTx", + ) + + private val allowedMethods: Set = setOf( + // Block / tip + "node_getBlockNumber", + "node_getProvenBlockNumber", + "node_getL2Tips", + "node_getChainTips", + "node_getBlock", + "node_getBlocks", + "node_getBlockData", + "node_getBlockHeader", + "node_getBlockByArchive", + "node_getBlockByHash", + "node_getBlockHeaderByArchive", + + // Checkpoints / consensus + "node_getCheckpointNumber", + "node_getCheckpoint", + "node_getCheckpointedBlockNumber", + "node_getCheckpointedBlocks", + "node_getCheckpoints", + "node_getCheckpointsData", + "node_getCheckpointAttestationsForSlot", + "node_getProposalsForSlot", + + // Transactions + "node_sendTx", + "node_getTxReceipt", + "node_getTxEffect", + "node_getTxByHash", + "node_getTxsByHash", + "node_getPendingTxs", + "node_getPendingTxCount", + "node_isValidTx", + "node_simulatePublicCalls", + + // State / storage + "node_getPublicStorageAt", + "node_getWorldStateSyncStatus", + "node_findLeavesIndexes", + + // Sync status + "node_getSyncedL1Timestamp", + "node_getSyncedL2EpochNumber", + "node_getSyncedL2SlotNumber", + + // Sibling paths + "node_getNullifierSiblingPath", + "node_getNoteHashSiblingPath", + "node_getArchiveSiblingPath", + "node_getPublicDataSiblingPath", + + // Membership witnesses + "node_getNullifierMembershipWitness", + "node_getLowNullifierMembershipWitness", + "node_getPublicDataWitness", + "node_getArchiveMembershipWitness", + "node_getNoteHashMembershipWitness", + "node_getBlockHashMembershipWitness", + "node_getL1ToL2MessageMembershipWitness", + + // L1 <-> L2 messages + "node_getL1ToL2MessageBlock", + "node_getL1ToL2MessageCheckpoint", + "node_isL1ToL2MessageSynced", + "node_getL2ToL1Messages", + "node_getL2ToL1MembershipWitness", + + // Logs + "node_getPrivateLogs", + "node_getPrivateLogsByTags", + "node_getPublicLogs", + "node_getPublicLogsByTagsFromContract", + "node_getPublicLogsByTags", + "node_getContractClassLogs", + "node_getLogsByTags", + + // Contracts + "node_getContractClass", + "node_getContract", + "node_registerContractFunctionSignatures", + + // Node info + "node_isReady", + "node_getNodeInfo", + "node_getNodeVersion", + "node_getVersion", + "node_getChainId", + "node_getL1ContractAddresses", + "node_getL1Constants", + "node_getProtocolContractAddresses", + "node_getEncodedEnr", + + // Fees + "node_getCurrentBaseFees", + "node_getCurrentMinFees", + "node_getPredictedMinFees", + "node_getMaxPriorityFees", + + // Validators + "node_getValidatorsStats", + "node_getValidatorStats", + + // Misc + "node_getAllowedPublicSetup", + ) + + override fun createQuorumFor(method: String): CallQuorum { + return when { + broadcast.contains(method) -> BroadcastQuorum() + else -> AlwaysQuorum() + } + } + + override fun isCallable(method: String): Boolean { + return allowedMethods.contains(method) + } + + override fun isHardcoded(method: String): Boolean { + return false + } + + override fun executeHardcoded(method: String): ByteArray { + throw RpcException(-32601, "Method not found") + } + + override fun getGroupMethods(groupName: String): Set = + when (groupName) { + "default" -> getSupportedMethods() + else -> emptySet() + } + + override fun getSupportedMethods(): Set { + return allowedMethods.toSortedSet() + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultBeaconChainMethods.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultBeaconChainMethods.kt index 1fa3ff225..3e0d137b5 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultBeaconChainMethods.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultBeaconChainMethods.kt @@ -29,6 +29,7 @@ class DefaultBeaconChainMethods : CallMethods { getMethod("/eth/v1/beacon/blocks/*/root"), getMethod("/eth/v1/beacon/blocks/*/attestations"), getMethod("/eth/v1/beacon/blob_sidecars/*"), + getMethod("/eth/v1/beacon/blobs/*"), postMethod("/eth/v1/beacon/rewards/sync_committee/*"), getMethod("/eth/v1/beacon/deposit_snapshot"), getMethod("/eth/v1/beacon/rewards/blocks/*"), @@ -116,7 +117,13 @@ class DefaultBeaconChainMethods : CallMethods { } override fun isCallable(method: String): Boolean { - return allowedMethods.contains(method) + if (allowedMethods.contains(method)) { + return true + } + // Check wildcard patterns (e.g., GET#/eth/v1/beacon/headers/* matches GET#/eth/v1/beacon/headers/head) + return allowedMethods.any { pattern -> + pattern.contains("*") && method.matches(pattern.replace("*", "[^/]+").toRegex()) + } } override fun getSupportedMethods(): Set { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultBitcoinMethods.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultBitcoinMethods.kt index 6a1691b89..69da49083 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultBitcoinMethods.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultBitcoinMethods.kt @@ -37,6 +37,9 @@ class DefaultBitcoinMethods(balances: Boolean) : CallMethods { "gettransaction", "gettxout", "getmemorypool", + "getrawmempool", + "getmempoolinfo", + "getblockheader", ).sorted() private val anyResponseMethods = listOf( diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultEthereumMethods.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultEthereumMethods.kt index ed1a3c98c..3c3616bc0 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultEthereumMethods.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultEthereumMethods.kt @@ -179,6 +179,7 @@ class DefaultEthereumMethods( private val firstValueMethods = listOf( "eth_call", "eth_getStorageAt", + "eth_getStorageValues", "eth_getCode", "eth_getLogs", "eth_maxPriorityFeePerGas", @@ -234,6 +235,10 @@ class DefaultEthereumMethods( "eth_chainId", ) + private val avalancheMethods = listOf( + "eth_baseFee", + ) + private val filecoinMethods = listOf( "Filecoin.ChainBlockstoreInfo", "Filecoin.ChainExport", @@ -611,6 +616,18 @@ class DefaultEthereumMethods( "zks_sendRawTransactionWithDetailedOutput", ) + private val borMethods = listOf( + "bor_getAuthor", + "bor_getCurrentValidators", + "bor_getCurrentProposer", + "bor_getRootHash", + "bor_getSigners", + "bor_getSignersAtHash", + "bor_getSnapshot", + "bor_getSnapshotAtHash", + "eth_getRootHash", + ) + private val allowedMethods: List init { @@ -664,14 +681,8 @@ class DefaultEthereumMethods( "rollup_gasPrices", "eth_getBlockRange", ) - Chain.POLYGON__MAINNET -> listOf( - "bor_getAuthor", - "bor_getCurrentValidators", - "bor_getCurrentProposer", - "bor_getRootHash", - "bor_getSignersAtHash", - "eth_getRootHash", - ) + Chain.POLYGON__MAINNET, Chain.POLYGON__AMOY -> borMethods + Chain.SHIBARIUM__MAINNET -> borMethods + listOf("eth_getTransactionReceiptsByBlock") Chain.POLYGON_ZKEVM__MAINNET, Chain.POLYGON_ZKEVM__CARDONA -> listOf( "zkevm_consolidatedBlockNumber", @@ -722,8 +733,14 @@ class DefaultEthereumMethods( "buildTransaction", ) + Chain.MONAD__MAINNET, Chain.MONAD__TESTNET -> listOf( + "eth_sendRawTransactionSync", + ) + Chain.FILECOIN__MAINNET, Chain.FILECOIN__CALIBRATION_TESTNET -> filecoinMethods + Chain.AVALANCHE__MAINNET, Chain.AVALANCHE__FUJI -> avalancheMethods + Chain.SEI__MAINNET, Chain.SEI__TESTNET, Chain.SEI__DEVNET -> seiMethods Chain.CRONOS_ZKEVM__MAINNET, Chain.CRONOS_ZKEVM__TESTNET -> zxkSyncMethods + "zk_estimateFee" diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultNearMethods.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultNearMethods.kt index bf39da1f3..89a279ed8 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultNearMethods.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultNearMethods.kt @@ -20,6 +20,7 @@ class DefaultNearMethods : CallMethods { "tx", "EXPERIMENTAL_tx_status", "EXPERIMENTAL_receipt", + "EXPERIMENTAL_protocol_config", ) private val add = setOf( diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultRippleMethods.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultRippleMethods.kt index abbb57470..ed2947707 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultRippleMethods.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultRippleMethods.kt @@ -1,8 +1,8 @@ package io.emeraldpay.dshackle.upstream.calls import io.emeraldpay.dshackle.quorum.AlwaysQuorum -import io.emeraldpay.dshackle.quorum.BroadcastQuorum import io.emeraldpay.dshackle.quorum.CallQuorum +import io.emeraldpay.dshackle.quorum.JsonBroadcastQuorum import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException class DefaultRippleMethods : CallMethods { @@ -66,7 +66,7 @@ class DefaultRippleMethods : CallMethods { override fun createQuorumFor(method: String): CallQuorum { if (add.contains(method)) { - return BroadcastQuorum() + return JsonBroadcastQuorum() } return AlwaysQuorum() } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/cosmos/CosmosChainSpecific.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/cosmos/CosmosChainSpecific.kt index ae6a8c8dd..064ff5f31 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/cosmos/CosmosChainSpecific.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/cosmos/CosmosChainSpecific.kt @@ -121,49 +121,49 @@ object CosmosChainSpecific : AbstractPollChainSpecific() { @JsonIgnoreProperties(ignoreUnknown = true) data class CosmosBlockResult( - @JsonProperty("block_id") var blockId: CosmosBlockId, - @JsonProperty("block") var block: CosmosBlockData, + @param:JsonProperty("block_id") var blockId: CosmosBlockId, + @param:JsonProperty("block") var block: CosmosBlockData, ) @JsonIgnoreProperties(ignoreUnknown = true) data class CosmosBlockId( - @JsonProperty("hash") var hash: String, + @param:JsonProperty("hash") var hash: String, ) @JsonIgnoreProperties(ignoreUnknown = true) data class CosmosHeader( - @JsonProperty("last_block_id") var lastBlockId: CosmosBlockId, - @JsonProperty("height") var height: String, - @JsonProperty("time") var time: Instant, + @param:JsonProperty("last_block_id") var lastBlockId: CosmosBlockId, + @param:JsonProperty("height") var height: String, + @param:JsonProperty("time") var time: Instant, ) @JsonIgnoreProperties(ignoreUnknown = true) data class CosmosStatus( - @JsonProperty("node_info") var nodeInfo: CosmosNodeInfo, - @JsonProperty("sync_info") var syncInfo: CosmosSyncInfo, + @param:JsonProperty("node_info") var nodeInfo: CosmosNodeInfo, + @param:JsonProperty("sync_info") var syncInfo: CosmosSyncInfo, ) @JsonIgnoreProperties(ignoreUnknown = true) data class CosmosNodeInfo( - @JsonProperty("version") var version: String, - @JsonProperty("network") var network: String, + @param:JsonProperty("version") var version: String, + @param:JsonProperty("network") var network: String, ) @JsonIgnoreProperties(ignoreUnknown = true) data class CosmosSyncInfo( - @JsonProperty("earliest_block_height") var earliestBlockHeight: String, + @param:JsonProperty("earliest_block_height") var earliestBlockHeight: String, ) @JsonIgnoreProperties(ignoreUnknown = true) data class CosmosBlockEvent( - @JsonProperty("data") var data: CosmosBlockEventData, + @param:JsonProperty("data") var data: CosmosBlockEventData, ) @JsonIgnoreProperties(ignoreUnknown = true) data class CosmosBlockEventData( - @JsonProperty("value") var value: CosmosBlockData, + @param:JsonProperty("value") var value: CosmosBlockData, ) data class CosmosBlockData( - @JsonProperty("header") var header: CosmosHeader, + @param:JsonProperty("header") var header: CosmosHeader, ) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/error/EthereumLowerBoundErrorHandler.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/error/EthereumLowerBoundErrorHandler.kt index 587d02d4e..e13409b96 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/error/EthereumLowerBoundErrorHandler.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/error/EthereumLowerBoundErrorHandler.kt @@ -12,8 +12,28 @@ abstract class EthereumLowerBoundErrorHandler : ErrorHandler { override fun handle(upstream: Upstream, request: ChainRequest, errorMessage: String?) { try { if (canHandle(request, errorMessage)) { - parseTagParam(request, tagIndex(request.method))?.let { - upstream.updateLowerBound(it, type()) + parseTagParam(request, tagIndex(request.method))?.let { parsed -> + val currentHeight = upstream.getHead().getCurrentHeight() + if (currentHeight == null) { + log.warn( + "Skip {} lower bound update for {}: head height unknown (parsed={})", + type(), + upstream.getId(), + parsed, + ) + return@let + } + if (parsed > currentHeight) { + log.warn( + "Skip {} lower bound update for {}: parsed tag {} exceeds head {}", + type(), + upstream.getId(), + parsed, + currentHeight, + ) + return@let + } + upstream.updateLowerBound(parsed, type()) } } } catch (e: RuntimeException) { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/error/EthereumStateLowerBoundErrorHandler.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/error/EthereumStateLowerBoundErrorHandler.kt index 1916b788c..ea0565e25 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/error/EthereumStateLowerBoundErrorHandler.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/error/EthereumStateLowerBoundErrorHandler.kt @@ -21,7 +21,9 @@ object EthereumStateLowerBoundErrorHandler : EthereumLowerBoundErrorHandler() { private val applicableMethods = firstTagIndexMethods + secondTagIndexMethods override fun canHandle(request: ChainRequest, errorMessage: String?): Boolean { - return stateErrors.any { errorMessage?.contains(it) ?: false } && applicableMethods.contains(request.method) + return !(errorMessage?.contains("execution reverted") ?: false) && + stateErrors.any { errorMessage?.contains(it) ?: false } && + applicableMethods.contains(request.method) } override fun tagIndex(method: String): Int { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/BasicEthUpstreamRpcMethodsDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/BasicEthUpstreamRpcMethodsDetector.kt index 5e9c01134..25e8223c3 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/BasicEthUpstreamRpcMethodsDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/BasicEthUpstreamRpcMethodsDetector.kt @@ -34,7 +34,9 @@ class BasicEthUpstreamRpcMethodsDetector( setOf( "eth_getBlockReceipts" to ListParams("latest"), "trace_callMany" to ListParams(listOf(listOf())), + "trace_rawTransaction" to ListParams(listOf()), "eth_simulateV1" to ListParams(listOf()), + "eth_getStorageValues" to ListParams(listOf()), "debug_storageRangeAt" to ListParams(listOf()), "eth_getTdByNumber" to ListParams(listOf()), "eth_callBundle" to ListParams(listOf()), diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumCachingReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumCachingReader.kt index dbb16b8c7..89d1136ce 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumCachingReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumCachingReader.kt @@ -17,11 +17,6 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.cache.Caches import io.emeraldpay.dshackle.cache.CurrentBlockCache -import io.emeraldpay.dshackle.commons.CACHE_BLOCK_BY_HASH_READER -import io.emeraldpay.dshackle.commons.CACHE_BLOCK_BY_HEIGHT_READER -import io.emeraldpay.dshackle.commons.CACHE_RECEIPTS_READER -import io.emeraldpay.dshackle.commons.CACHE_TX_BY_HASH_READER -import io.emeraldpay.dshackle.commons.DIRECT_QUORUM_RPC_READER import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.data.TxContainer @@ -29,7 +24,6 @@ import io.emeraldpay.dshackle.data.TxId import io.emeraldpay.dshackle.reader.CompoundReader import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.reader.RekeyingReader -import io.emeraldpay.dshackle.reader.SpannedReader import io.emeraldpay.dshackle.upstream.CachingReader import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.Selector @@ -43,7 +37,6 @@ import io.emeraldpay.dshackle.upstream.ethereum.domain.Wei import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionLogJson import io.emeraldpay.dshackle.upstream.finalization.FinalizationType import org.apache.commons.collections4.Factory -import org.springframework.cloud.sleuth.Tracer import reactor.core.publisher.Mono import java.util.function.Function @@ -54,29 +47,28 @@ open class EthereumCachingReader( private val up: Multistream, private val caches: Caches, callMethodsFactory: Factory, - private val tracer: Tracer, ) : CachingReader { private val balanceCache = CurrentBlockCache() - private val directReader = EthereumDirectReader(up, caches, balanceCache, callMethodsFactory, tracer) + private val directReader = EthereumDirectReader(up, caches, balanceCache, callMethodsFactory) open fun blockByFinalization(): Reader> { - return SpannedReader(directReader.blockByFinalizationReader, tracer, DIRECT_QUORUM_RPC_READER) + return directReader.blockByFinalizationReader } open fun blocksByIdAsCont(upstreamFilter: Selector.UpstreamFilter): Reader> { val idToBlockHash = Function> { id -> Request(BlockHash.from(id.value), upstreamFilter) } return CompoundReader( - SpannedReader(CacheWithUpstreamIdReader(caches.getBlocksByHash()), tracer, CACHE_BLOCK_BY_HASH_READER), - SpannedReader(RekeyingReader(idToBlockHash, directReader.blockReader), tracer, DIRECT_QUORUM_RPC_READER), + CacheWithUpstreamIdReader(caches.getBlocksByHash()), + RekeyingReader(idToBlockHash, directReader.blockReader), ) } open fun blocksByHeightAsCont(upstreamFilter: Selector.UpstreamFilter): Reader> { val numToRequest = Function> { num -> Request(num, upstreamFilter) } return CompoundReader( - SpannedReader(CacheWithUpstreamIdReader(caches.getBlocksByHeight()), tracer, CACHE_BLOCK_BY_HEIGHT_READER), - SpannedReader(RekeyingReader(numToRequest, directReader.blockByHeightReader), tracer, DIRECT_QUORUM_RPC_READER), + CacheWithUpstreamIdReader(caches.getBlocksByHeight()), + RekeyingReader(numToRequest, directReader.blockByHeightReader), ) } @@ -87,8 +79,8 @@ open class EthereumCachingReader( open fun txByHashAsCont(upstreamFilter: Selector.UpstreamFilter): Reader> { val idToTxHash = Function> { id -> Request(TransactionId.from(id.value), upstreamFilter) } return CompoundReader( - CacheWithUpstreamIdReader(SpannedReader(caches.getTxByHash(), tracer, CACHE_TX_BY_HASH_READER)), - SpannedReader(RekeyingReader(idToTxHash, directReader.txReader), tracer, DIRECT_QUORUM_RPC_READER), + CacheWithUpstreamIdReader(caches.getTxByHash()), + RekeyingReader(idToTxHash, directReader.txReader), ) } @@ -106,8 +98,8 @@ open class EthereumCachingReader( directReader.receiptReader, ) return CompoundReader( - CacheWithUpstreamIdReader(SpannedReader(caches.getReceipts(), tracer, CACHE_RECEIPTS_READER)), - SpannedReader(requested, tracer, DIRECT_QUORUM_RPC_READER), + CacheWithUpstreamIdReader(caches.getReceipts()), + requested, ) } @@ -126,7 +118,7 @@ open class EthereumCachingReader( override fun stop() { } - private class CacheWithUpstreamIdReader( + private class CacheWithUpstreamIdReader( private val reader: Reader, ) : Reader> { override fun read(key: K): Mono> { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumChainSpecific.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumChainSpecific.kt index bc57c9dc4..37d326833 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumChainSpecific.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumChainSpecific.kt @@ -36,7 +36,6 @@ import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundService import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import org.slf4j.Logger import org.slf4j.LoggerFactory -import org.springframework.cloud.sleuth.Tracer import reactor.core.publisher.Mono import reactor.core.scheduler.Scheduler import java.math.BigInteger @@ -89,8 +88,8 @@ object EthereumChainSpecific : AbstractPollChainSpecific() { } } - override fun makeCachingReaderBuilder(tracer: Tracer): CachingReaderBuilder { - return { ms, caches, methodsFactory -> EthereumCachingReader(ms, caches, methodsFactory, tracer) } + override fun makeCachingReaderBuilder(): CachingReaderBuilder { + return { ms, caches, methodsFactory -> EthereumCachingReader(ms, caches, methodsFactory) } } override fun upstreamValidators( diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumDirectReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumDirectReader.kt index a2e00178c..324acb515 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumDirectReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumDirectReader.kt @@ -34,10 +34,10 @@ import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError import io.emeraldpay.dshackle.upstream.finalization.FinalizationType import io.emeraldpay.dshackle.upstream.rpcclient.ListParams +import io.emeraldpay.dshackle.upstream.signature.DisabledSigner import org.apache.commons.collections4.Factory import org.apache.commons.lang3.exception.ExceptionUtils import org.slf4j.LoggerFactory -import org.springframework.cloud.sleuth.Tracer import reactor.core.publisher.Mono import reactor.util.retry.Retry import java.time.Duration @@ -51,7 +51,6 @@ class EthereumDirectReader( private val caches: Caches, private val balanceCache: CurrentBlockCache, private val callMethodsFactory: Factory, - private val tracer: Tracer, ) { companion object { @@ -256,8 +255,7 @@ class EthereumDirectReader( up, Selector.UpstreamFilter(sort, matcher), callMethodsFactory.create().createQuorumFor(request.method), - null, - tracer, + DisabledSigner(), ), ) }.flatMap { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumEgressSubscription.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumEgressSubscription.kt index 3c2a4bc8b..6cd99d93c 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumEgressSubscription.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumEgressSubscription.kt @@ -11,6 +11,7 @@ import io.emeraldpay.dshackle.upstream.ethereum.domain.Address import io.emeraldpay.dshackle.upstream.ethereum.hex.Hex32 import io.emeraldpay.dshackle.upstream.ethereum.subscribe.ConnectLogs import io.emeraldpay.dshackle.upstream.ethereum.subscribe.ConnectNewHeads +import io.emeraldpay.dshackle.upstream.ethereum.subscribe.NoPendingTxes import io.emeraldpay.dshackle.upstream.ethereum.subscribe.PendingTxesSource import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import org.slf4j.LoggerFactory @@ -72,7 +73,7 @@ open class EthereumEgressSubscription( } else { listOf() } - return if (pendingTxesSource != null) { + return if (pendingTxesSource != null && pendingTxesSource !is NoPendingTxes && upstream.getCapabilities().contains(Capability.WS_PENDING_TX)) { subs.plus(listOf(METHOD_PENDING_TXES, METHOD_DRPC_PENDING_TXES)) } else { subs diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLocalReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLocalReader.kt index 925451a5c..5983dbfed 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLocalReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLocalReader.kt @@ -48,10 +48,6 @@ class EthereumLocalReader( if (!methods.isCallable(key.method)) { return Mono.error(RpcException(RpcResponseError.CODE_METHOD_NOT_EXIST, "Unsupported method")) } - if (key.nonce != null) { - // we do not want to serve any requests (except hardcoded) that have nonces from cache - return Mono.empty() - } return commonRequests(key)?.switchIfEmpty { // we need to explicitly return null to prevent executeOnRemote // for example diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundBlockDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundBlockDetector.kt index 86487f869..c3523e726 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundBlockDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundBlockDetector.kt @@ -28,6 +28,10 @@ class EthereumLowerBoundBlockDetector( "method handler crashed", // sei "Unexpected error", // hyperliquid "invalid block height", // hyperliquid + "pruned history unavailable", // xlayer + "no transactions snapshot file for", + "has been pruned; earliest available is", + "could not find block for height", ) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundProofDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundProofDetector.kt index 7f4f175a5..dad06e075 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundProofDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundProofDetector.kt @@ -29,10 +29,25 @@ class EthereumLowerBoundProofDetector( "no historical RPC is available", "Method not found", // Monad error bc they don't have eth_getProofs "invalid block height", // hyperliquid + "not supported", + "evm module does not exist on height", + "state is not available", // opbnb / bsc — eth_getProof on pruned state ) } - private val recursiveLowerBound = RecursiveLowerBound(upstream, LowerBoundType.PROOF, NO_PROOF_ERRORS, lowerBounds, commonErrorPatterns) + private val recursiveLowerBound = RecursiveLowerBound( + upstream, + LowerBoundType.PROOF, + NO_PROOF_ERRORS, + lowerBounds, + commonErrorPatterns.plus( + setOf( + Regex("block #\\d not found"), + Regex("state at block #\\d is pruned"), + Regex("historical state .+ is not available"), + ), + ), + ) override fun period(): Long { return 3 diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundReceiptsDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundReceiptsDetector.kt index dd04709dc..b71a7aaaf 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundReceiptsDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundReceiptsDetector.kt @@ -1,15 +1,19 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.Defaults +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.ChainResponse import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.lowerbound.GoldLowerBounds import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType import io.emeraldpay.dshackle.upstream.lowerbound.detector.RecursiveLowerBound import io.emeraldpay.dshackle.upstream.lowerbound.toHex import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import reactor.core.publisher.Flux +import reactor.core.publisher.Mono class EthereumLowerBoundReceiptsDetector( private val upstream: Upstream, @@ -28,6 +32,9 @@ class EthereumLowerBoundReceiptsDetector( "Unexpected error", // hyperliquid "invalid block height", // hyperliquid "header not found", + "pruned history unavailable", // xlayer + "transaction indexing is in progress", + "has been pruned; earliest available is", ).plus(EthereumLowerBoundBlockDetector.NO_BLOCK_ERRORS) } @@ -38,6 +45,34 @@ class EthereumLowerBoundReceiptsDetector( } override fun internalDetectLowerBound(): Flux { + val receiptsGoldBound = GoldLowerBounds.getBound(upstream.getChain(), LowerBoundType.RECEIPTS) + if (receiptsGoldBound == null || receiptsGoldBound !is ChainsConfig.GoldLowerBoundWithHash) { + return recursiveDetectReceiptsLowerBound() + } + return Mono.just(receiptsGoldBound) + .flatMapMany { bound -> + upstream.getIngressReader() + .read(ChainRequest("eth_getTransactionReceipt", ListParams(bound.hash))) + .timeout(Defaults.internalCallsTimeout) + .flatMap(ChainResponse::requireResult) + .flatMapMany { + if (it.contentEquals("null".toByteArray())) { + throw IllegalStateException("no gold bound") + } else { + Flux.just(LowerBoundData(1, LowerBoundType.RECEIPTS)) + } + } + .onErrorResume { + recursiveDetectReceiptsLowerBound() + } + } + } + + override fun types(): Set { + return setOf(LowerBoundType.RECEIPTS) + } + + private fun recursiveDetectReceiptsLowerBound(): Flux { return recursiveLowerBound.recursiveDetectLowerBoundWithOffset(MAX_OFFSET) { block -> upstream.getIngressReader() .read( @@ -71,8 +106,4 @@ class EthereumLowerBoundReceiptsDetector( } } } - - override fun types(): Set { - return setOf(LowerBoundType.RECEIPTS) - } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundStateDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundStateDetector.kt index 9cfb5c50c..2e207110a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundStateDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundStateDetector.kt @@ -25,6 +25,8 @@ class EthereumLowerBoundStateDetector( setOf( Regex("block #\\d not found"), Regex("state at block #\\d is pruned"), + Regex("historical state .+ is not available"), + Regex("state .+ is not available"), ), ), ) @@ -83,6 +85,7 @@ class EthereumLowerBoundStateDetector( "state histories haven't been fully indexed yet", "but it out-of-bounds", "not supported", + "has been pruned; earliest available is", ) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundTxDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundTxDetector.kt index e984d068e..7bc2800c5 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundTxDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundTxDetector.kt @@ -1,15 +1,19 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.Defaults +import io.emeraldpay.dshackle.config.ChainsConfig import io.emeraldpay.dshackle.data.BlockContainer import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.ChainResponse import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.lowerbound.GoldLowerBounds import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType import io.emeraldpay.dshackle.upstream.lowerbound.detector.RecursiveLowerBound import io.emeraldpay.dshackle.upstream.lowerbound.toHex import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import reactor.core.publisher.Flux +import reactor.core.publisher.Mono class EthereumLowerBoundTxDetector( private val upstream: Upstream, @@ -23,6 +27,9 @@ class EthereumLowerBoundTxDetector( NO_TX_DATA, "Unexpected error", // hyperliquid "invalid block height", // hyperliquids + "pruned history unavailable", // xlayer blocks + "transaction indexing is in progress", + "has been pruned; earliest available is", ).plus(EthereumLowerBoundBlockDetector.NO_BLOCK_ERRORS) } @@ -33,6 +40,34 @@ class EthereumLowerBoundTxDetector( } override fun internalDetectLowerBound(): Flux { + val txGoldBound = GoldLowerBounds.getBound(upstream.getChain(), LowerBoundType.TX) + if (txGoldBound == null || txGoldBound !is ChainsConfig.GoldLowerBoundWithHash) { + return recursiveDetectTxLowerBound() + } + return Mono.just(txGoldBound) + .flatMapMany { bound -> + upstream.getIngressReader() + .read(ChainRequest("eth_getTransactionByHash", ListParams(bound.hash))) + .timeout(Defaults.internalCallsTimeout) + .flatMap(ChainResponse::requireResult) + .flatMapMany { + if (it.contentEquals("null".toByteArray())) { + throw IllegalStateException("no gold bound") + } else { + Flux.just(LowerBoundData(1, LowerBoundType.TX)) + } + } + .onErrorResume { + recursiveDetectTxLowerBound() + } + } + } + + override fun types(): Set { + return setOf(LowerBoundType.TX) + } + + private fun recursiveDetectTxLowerBound(): Flux { return recursiveLowerBound.recursiveDetectLowerBoundWithOffset(MAX_OFFSET) { block -> upstream.getIngressReader() .read( @@ -66,8 +101,4 @@ class EthereumLowerBoundTxDetector( } } } - - override fun types(): Set { - return setOf(LowerBoundType.TX) - } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamSettingsDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamSettingsDetector.kt index 08e7aa503..31caa0f5b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamSettingsDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamSettingsDetector.kt @@ -8,12 +8,14 @@ import io.emeraldpay.dshackle.upstream.ChainRequest import io.emeraldpay.dshackle.upstream.ChainResponse import io.emeraldpay.dshackle.upstream.NodeTypeRequest import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.generic.GenericUpstream import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import reactor.core.publisher.Flux import reactor.core.publisher.Mono import java.util.concurrent.atomic.AtomicInteger const val ZERO_ADDRESS = "0x0000000000000000000000000000000000000000" +const val HL_NATIVE_TX_FROM_MAINNET = "0x2222222222222222222222222222222222222222" class EthereumUpstreamSettingsDetector( private val _upstream: Upstream, @@ -35,7 +37,7 @@ class EthereumUpstreamSettingsDetector( ) } - private fun detectFlashBlocks(): Mono>? { + private fun detectFlashBlocks(): Mono> { return upstream.getIngressReader().read( ChainRequest( "eth_getBlockByNumber", @@ -89,6 +91,22 @@ class EthereumUpstreamSettingsDetector( * */ private fun detectGasLabels(): Flux> { + return if (chain == Chain.MONAD__MAINNET || chain == Chain.MONAD__TESTNET) { + basicGasLabels() + .collectList() + .flatMapMany { labels -> + if (labels.find { it.first == "extra_gas_limit" }?.second == "600000000") { + Flux.fromIterable(labels) + } else { + monadGasLabels() + } + } + } else { + basicGasLabels() + } + } + + private fun basicGasLabels(): Flux> { return upstream.getIngressReader().read( ChainRequest( "eth_call", @@ -123,14 +141,55 @@ class EthereumUpstreamSettingsDetector( } } + // the basic gas check doesn't work on monad nodes, let's check then by an error message + private fun monadGasLabels(): Flux> { + return upstream.getIngressReader().read( + ChainRequest( + "eth_call", + ListParams( + mapOf( + "to" to "0x53Daa71B04d589429f6d3DF52db123913B818F22", + "data" to "0x51be4eaa", + "gas" to "0x232AAF80", + ), + "latest", + mapOf( + "0x53Daa71B04d589429f6d3DF52db123913B818F22" to mapOf( + "code" to "0x6080604052348015600f57600080fd5b506004361060285760003560e01c806351be4eaa14602d575b600080fd5b60336047565b604051603e91906066565b60405180910390f35b60005a905090565b6000819050919050565b606081604f565b82525050565b6000602082019050607960008301846059565b9291505056fea26469706673582212201c0202887c1afe66974b06ee355dee07542bbc424cf4d1659c91f56c08c3dcc064736f6c63430008130033", + ), + ), + ), + ), + ).flatMapMany { + Flux.fromIterable(emptyList>()) + }.onErrorResume { + if (it.message?.contains("gas limit too high") ?: false) { + val labels = mutableListOf( + Pair("gas-limit", 600_000_000.toString()), + Pair("extra_gas_limit", 600_000_000.toString()), + ) + Flux.fromIterable(labels) + } else { + Flux.empty() + } + } + } + /* Some clients on hyperliquid don't include system topup transactions, set either one of labels */ private fun detectHlNativeTx(): Flux> { - // Only run HL native tx detection on Hyperliquid chains if (chain != Chain.HYPERLIQUID__MAINNET && chain != Chain.HYPERLIQUID__TESTNET) { return Flux.empty() } + // prefer the explicit ?hl= flag from the upstream URL + hlNativeTxLabelsFromUrl()?.let { return Flux.fromIterable(it) } + + // no ?hl= flag: block-scan fallback (reliable only on mainnet) + if (chain != Chain.HYPERLIQUID__MAINNET) { + return Flux.empty() + } + val hlNativeTxFrom = HL_NATIVE_TX_FROM_MAINNET if (detectCounter.get() % 5 != 1) { return Flux.empty() // reduce frequency of detection } @@ -161,7 +220,7 @@ class EthereumUpstreamSettingsDetector( if (receiptsJson.isArray) { receiptsJson.forEach { receipt -> val from = receipt.get("from")?.asText() - if (from == "0x2222222222222222222222222222222222222222") { // system topup transaction + if (from == hlNativeTxFrom) { // system topup transaction foundHlNativeTx = true } } @@ -198,6 +257,19 @@ class EthereumUpstreamSettingsDetector( } } + // maps the upstream URL's ?hl= flag to routing labels, or null if absent + private fun hlNativeTxLabelsFromUrl(): List>? { + val url = (upstream as? GenericUpstream)?.getRpcConnectionUrl()?.toString() ?: return null + // ?hl=false => serves native txs (include); ?hl=true => hl-node compliant (exclude) + return when { + url.contains(Regex("[?&]hl=false\\b")) -> + listOf("include_hl_native_tx" to "true", "exclude_hl_native_tx" to "false") + url.contains(Regex("[?&]hl=true\\b")) -> + listOf("exclude_hl_native_tx" to "true", "include_hl_native_tx" to "false") + else -> null + } + } + private fun detectArchiveNode(notArchived: Boolean): Mono> { if (notArchived) { return Mono.empty() diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidator.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidator.kt index 4921d2367..a2ecb61e4 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidator.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamValidator.kt @@ -34,7 +34,7 @@ import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import org.slf4j.Logger import org.slf4j.LoggerFactory import reactor.core.publisher.Mono -import reactor.kotlin.extra.retry.retryRandomBackoff +import reactor.util.retry.Retry import java.math.BigInteger import java.time.Duration import java.util.concurrent.TimeoutException @@ -75,12 +75,17 @@ abstract class AbstractCallLimitValidator( Mono.fromCallable { log.error("No response for eth_call limit check from ${upstream.getId()}") } .then(Mono.error(TimeoutException("Validation timeout for call limit"))), ) - .retryRandomBackoff(3, Duration.ofMillis(100), Duration.ofMillis(500)) { ctx -> - log.warn( - "error during validateCallLimit for ${upstream.getId()}, iteration ${ctx.iteration()}, " + - "message ${ctx.exception().message}", - ) - } + .retryWhen( + Retry.backoff(3, Duration.ofMillis(100)) + .maxBackoff(Duration.ofMillis(500)) + .jitter(1.0) + .doBeforeRetry { signal -> + log.warn( + "error during validateCallLimit for ${upstream.getId()}, iteration ${signal.totalRetries() + 1}, " + + "message ${signal.failure().message}", + ) + }, + ) .onErrorReturn(ValidateUpstreamSettingsResult.UPSTREAM_SETTINGS_ERROR) } @@ -164,14 +169,19 @@ class ChainIdValidator( private fun chainId(): Mono { return validatorReader.get() .read(ChainRequest("eth_chainId", ListParams())) - .retryRandomBackoff(3, Duration.ofMillis(100), Duration.ofMillis(500)) { ctx -> - log.warn( - "error during chainId retrieving for {}, iteration {}, reason - {}", - upstream.getId(), - ctx.iteration(), - ctx.exception().message, - ) - } + .retryWhen( + Retry.backoff(3, Duration.ofMillis(100)) + .maxBackoff(Duration.ofMillis(500)) + .jitter(1.0) + .doBeforeRetry { signal -> + log.warn( + "error during chainId retrieving for {}, iteration {}, reason - {}", + upstream.getId(), + signal.totalRetries() + 1, + signal.failure().message, + ) + }, + ) .doOnError { log.error("Error during execution 'eth_chainId' - {} for {}", it.message, upstream.getId()) } .flatMap(ChainResponse::requireStringResult) } @@ -179,14 +189,19 @@ class ChainIdValidator( private fun netVersion(): Mono { return validatorReader.get() .read(ChainRequest("net_version", ListParams())) - .retryRandomBackoff(3, Duration.ofMillis(100), Duration.ofMillis(500)) { ctx -> - log.warn( - "error during netVersion retrieving for {}, iteration {}, reason - {}", - upstream.getId(), - ctx.iteration(), - ctx.exception().message, - ) - } + .retryWhen( + Retry.backoff(3, Duration.ofMillis(100)) + .maxBackoff(Duration.ofMillis(500)) + .jitter(1.0) + .doBeforeRetry { signal -> + log.warn( + "error during netVersion retrieving for {}, iteration {}, reason - {}", + upstream.getId(), + signal.totalRetries() + 1, + signal.failure().message, + ) + }, + ) .doOnError { log.error("Error during execution 'net_version' - {} for {}", it.message, upstream.getId()) } .flatMap(ChainResponse::requireStringResult) } @@ -264,14 +279,19 @@ class ErigonBuggedValidator( private fun isErigon(): Mono = upstream.getIngressReader() .read(ChainRequest("web3_clientVersion", ListParams())) - .retryRandomBackoff(3, Duration.ofMillis(100), Duration.ofMillis(500)) { ctx -> - log.warn( - "error during clientVersion retrieving for {}, iteration {}, reason - {}", - upstream.getId(), - ctx.iteration(), - ctx.exception().message, - ) - } + .retryWhen( + Retry.backoff(3, Duration.ofMillis(100)) + .maxBackoff(Duration.ofMillis(500)) + .jitter(1.0) + .doBeforeRetry { signal -> + log.warn( + "error during clientVersion retrieving for {}, iteration {}, reason - {}", + upstream.getId(), + signal.totalRetries() + 1, + signal.failure().message, + ) + }, + ) .flatMap(ChainResponse::requireStringResult) .map { it.lowercase().contains("erigon") } .doOnError { log.error("Error during execution 'web3_clientVersion' - {} for {}", it.message, upstream.getId()) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/LogIndexValidator.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/LogIndexValidator.kt index d3c1bb338..bea004d3a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/LogIndexValidator.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/LogIndexValidator.kt @@ -26,7 +26,7 @@ import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import org.slf4j.Logger import org.slf4j.LoggerFactory import reactor.core.publisher.Mono -import reactor.kotlin.extra.retry.retryRandomBackoff +import reactor.util.retry.Retry import java.time.Duration import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicReference @@ -84,12 +84,17 @@ class LogIndexValidator( } } .timeout(Duration.ofSeconds(15)) - .retryRandomBackoff(2, Duration.ofMillis(200), Duration.ofMillis(1000)) { ctx -> - log.debug( - "Retry logIndex validation for ${upstream.getId()}, iteration ${ctx.iteration()}, " + - "error: ${ctx.exception().message}", - ) - } + .retryWhen( + Retry.backoff(2, Duration.ofMillis(200)) + .maxBackoff(Duration.ofMillis(1000)) + .jitter(1.0) + .doBeforeRetry { signal -> + log.debug( + "Retry logIndex validation for ${upstream.getId()}, iteration ${signal.totalRetries() + 1}, " + + "error: ${signal.failure().message}", + ) + }, + ) .onErrorResume { err -> log.warn("Error during logIndex validation for ${upstream.getId()}: ${err.message}") // In case of error, return last known state to avoid false positives diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/PendingTransactionValidator.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/PendingTransactionValidator.kt new file mode 100644 index 000000000..259ed0dbe --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/PendingTransactionValidator.kt @@ -0,0 +1,66 @@ +package io.emeraldpay.dshackle.upstream.ethereum + +import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.reader.ChainReader +import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.ChainResponse +import io.emeraldpay.dshackle.upstream.rpcclient.ListParams +import org.slf4j.LoggerFactory +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono +import java.time.Duration + +const val BASE_TX_LIMIT = 1000L + +interface PendingTransactionValidator { + fun pendingTxExists(): Flux +} + +class NoopPendingTransactionValidator : PendingTransactionValidator { + override fun pendingTxExists(): Flux { + return Flux.just(true) + } +} + +class PendingTransactionValidatorImpl( + private val upstreamId: String, + private val directReader: ChainReader, + private val interval: Duration, + private val txLimit: Long, +) : PendingTransactionValidator { + private val log = LoggerFactory.getLogger(this::class.java) + + override fun pendingTxExists(): Flux { + return Flux.interval( + Duration.ofSeconds(15), + interval, + ) + .flatMap { + directReader.read(ChainRequest("txpool_content", ListParams())) + .flatMap(ChainResponse::requireResult) + .map { + val node = Global.objectMapper.readTree(it) + val pendingTxsNode = node.get("pending") + val queuedTxsNode = node.get("queued") + + val pendingTxsCount = if (pendingTxsNode != null) { + pendingTxsNode.fieldNames().asSequence().toList().size + } else { + 0 + } + val queuedTxsCount = if (queuedTxsNode != null) { + queuedTxsNode.fieldNames().asSequence().toList().size + } else { + 0 + } + + ((pendingTxsCount + queuedTxsCount) >= txLimit) + } + .timeout(Duration.ofSeconds(30)) + .onErrorResume { + log.error("unable to read txs from txpool_content of upstream {}", upstreamId, it) + Mono.just(false) + } + } + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionFactory.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionFactory.kt index 4ef0914c2..87bc84650 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionFactory.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionFactory.kt @@ -17,10 +17,13 @@ open class WsConnectionFactory( private val uri: URI, private val origin: URI, private val scheduler: Scheduler, + private val eventsScheduler: Scheduler, ) { var basicAuth: AuthConfig.ClientBasicAuth? = null + var bearerAuth: AuthConfig.ClientBearerAuth? = null var config: UpstreamsConfig.WsEndpoint? = null + var customHeaders: Map = emptyMap() private fun metrics(connIndex: Int): RequestMetrics { val metricsTags = listOf( @@ -45,7 +48,7 @@ open class WsConnectionFactory( } open fun createWsConnection(connIndex: Int = 0): WsConnection = - WsConnectionImpl(uri, origin, basicAuth, metrics(connIndex), scheduler).also { ws -> + WsConnectionImpl(uri, origin, basicAuth, metrics(connIndex), scheduler, eventsScheduler, customHeaders, bearerAuth).also { ws -> config?.frameSize?.let { ws.frameSize = it } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImpl.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImpl.kt index d9c354332..238896566 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImpl.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImpl.kt @@ -29,7 +29,6 @@ import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError import io.emeraldpay.dshackle.upstream.rpcclient.JsonRpcWsMessage import io.emeraldpay.dshackle.upstream.rpcclient.ResponseWSParser import io.micrometer.core.instrument.Metrics -import io.netty.buffer.ByteBufInputStream import io.netty.handler.codec.http.HttpHeaderNames import io.netty.resolver.DefaultAddressResolverGroup import org.reactivestreams.Publisher @@ -66,6 +65,9 @@ open class WsConnectionImpl( private val basicAuth: AuthConfig.ClientBasicAuth?, private val requestMetrics: RequestMetrics?, private val scheduler: Scheduler, + private val eventsScheduler: Scheduler, + private val customHeaders: Map = emptyMap(), + private val bearerAuth: AuthConfig.ClientBearerAuth? = null, ) : AutoCloseable, WsConnection, Cloneable { companion object { @@ -193,6 +195,7 @@ open class WsConnectionImpl( log.info("Connecting to WebSocket: $uri") connection?.dispose() connection = HttpClient.create() + .runOn(Global.wsLoops) .resolver(DefaultAddressResolverGroup.INSTANCE) .doOnDisconnected { disconnects.tryEmitNext(Instant.now()) @@ -225,6 +228,14 @@ open class WsConnectionImpl( val base64password = Base64.getEncoder().encodeToString(tmp.toByteArray()) headers.add(HttpHeaderNames.AUTHORIZATION, "Basic $base64password") } + if (basicAuth == null) { + bearerAuth?.let { auth -> + headers.add(HttpHeaderNames.AUTHORIZATION, "Bearer ${auth.token}") + } + } + customHeaders.forEach { (key, value) -> + headers.add(key, value) + } } .let { if (uri.scheme == "wss") it.secure() else it @@ -251,8 +262,9 @@ open class WsConnectionImpl( var read = false val consumer = inbound .aggregateFrames(msgSizeLimit) - .receiveFrames() - .map { ByteBufInputStream(it.content()).readAllBytes() } + .receive() + .asByteArray() + .publishOn(eventsScheduler) .filter { it.isNotEmpty() } .flatMap { try { @@ -398,7 +410,7 @@ open class WsConnectionImpl( return Mono.from(onResponse.asMono()).or(failOnDisconnect) .doOnSubscribe { sendRpc(request) } .take(Defaults.timeout) - .doOnNext { requestMetrics?.timer?.record(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) } + .doOnNext { requestMetrics?.timer()?.record(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) } .doOnError { requestMetrics?.fails?.increment() } .map { it.copyWithId(ChainResponse.Id.from(originalId)) } .switchIfEmpty( @@ -418,7 +430,7 @@ open class WsConnectionImpl( it.close() Metrics.globalRegistry.remove(it) } - requestMetrics?.timer?.let { + requestMetrics?.registeredTimers()?.forEach { it.close() Metrics.globalRegistry.remove(it) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsSubscriptionsImpl.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsSubscriptionsImpl.kt index 6b7be8c27..089aa291e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsSubscriptionsImpl.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/WsSubscriptionsImpl.kt @@ -19,6 +19,8 @@ import io.emeraldpay.dshackle.upstream.ChainException import io.emeraldpay.dshackle.upstream.ChainRequest import io.emeraldpay.dshackle.upstream.ChainResponse import io.emeraldpay.dshackle.upstream.rpcclient.ListParams +import io.emeraldpay.dshackle.upstream.rpcclient.ObjectParams +import io.emeraldpay.dshackle.upstream.rpcclient.RippleCommandParams import org.slf4j.LoggerFactory import reactor.core.publisher.Flux import reactor.core.publisher.Mono @@ -47,8 +49,10 @@ class WsSubscriptionsImpl( log.warn("Failed to establish subscription: ${it.error?.message}") Mono.error(ChainException(it.id, it.error!!)) } else { - val id = if (it.getResultAsRawString() == "{}") { - request.id.toString() // in case empty result - match by request id + val rawResult = it.getResultAsRawString() + val id = if (rawResult.startsWith("{") || rawResult == "{}") { + // Ripple returns object result; use stream type as subscription ID + extractRippleStreamType(request) ?: request.id.toString() } else { it.getResultAsProcessedString() } @@ -60,6 +64,26 @@ class WsSubscriptionsImpl( return WsSubscriptions.SubscribeData(message, conn.connectionId(), subscriptionId) } + /** + * Extract expected event type from Ripple subscribe request. + * For `subscribe` with `streams: ["ledger"]`, returns "ledgerClosed" as this is the event type + * that will be received in subscription messages. + */ + private fun extractRippleStreamType(request: ChainRequest): String? { + if (request.method == "subscribe") { + val params = request.params + val streams: List<*>? = when (params) { + is RippleCommandParams -> params.params["streams"] as? List<*> + is ObjectParams -> params.obj["streams"] as? List<*> + else -> null + } + if (streams?.contains("ledger") == true) { + return "ledgerClosed" + } + } + return null + } + override fun unsubscribe(request: ChainRequest): Mono { if (request.params is ListParams && (request.params.list.isEmpty() || request.params.list.contains("")) ) { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/EthereumWsIngressSubscription.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/EthereumWsIngressSubscription.kt index 0114dd057..a7ed95704 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/EthereumWsIngressSubscription.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/EthereumWsIngressSubscription.kt @@ -34,7 +34,7 @@ class EthereumWsIngressSubscription( } @Suppress("UNCHECKED_CAST") - override fun get(topic: String, params: Any?, unsubscribeMethod: String): SubscriptionConnect? { + override fun get(topic: String, params: Any?, unsubscribeMethod: String): SubscriptionConnect? { if (topic == EthereumEgressSubscription.METHOD_PENDING_TXES) { return pendingTxes as SubscriptionConnect } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/WebsocketPendingTxes.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/WebsocketPendingTxes.kt index ddd71c835..b2097626a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/WebsocketPendingTxes.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/subscribe/WebsocketPendingTxes.kt @@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory import reactor.core.publisher.Flux import reactor.core.publisher.Mono import java.time.Duration +import java.util.concurrent.TimeoutException class WebsocketPendingTxes( private val chain: Chain, @@ -35,6 +36,7 @@ class WebsocketPendingTxes( companion object { private val log = LoggerFactory.getLogger(WebsocketPendingTxes::class.java) + private val IDLE_TIMEOUT: Duration = Duration.ofSeconds(85) } override fun createConnection(): Flux { @@ -42,7 +44,15 @@ class WebsocketPendingTxes( return sub .data .flatMapMany { it.t2 } - .timeout(Duration.ofSeconds(85), Mono.empty()) + // Surface stalls as TimeoutException so the surrounding DurableFlux re-invokes + // createConnection() and issues a fresh eth_subscribe. Without this, a silent + // upstream pubsub leaves the shared Flux completed and clients stop receiving events. + .timeout( + IDLE_TIMEOUT, + Mono.error( + TimeoutException("No events from eth_subscribe newPendingTransactions in $IDLE_TIMEOUT, forcing resubscribe"), + ), + ) .map { // comes as a JS string, i.e., within quotes val value = ByteArray(it.size - 2) @@ -60,7 +70,6 @@ class WebsocketPendingTxes( log.info("unsubscribed from ${sub.subId.get()}") } } - .doOnError { t -> log.warn("Invalid pending transaction", t) } - .onErrorResume { Mono.empty() } + .doOnError { t -> log.warn("Error during pending tx subscription: {}", t.message) } } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/AbstractChainSpecific.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/AbstractChainSpecific.kt index 12e3d753e..053d0c1e5 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/AbstractChainSpecific.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/AbstractChainSpecific.kt @@ -30,7 +30,6 @@ import io.emeraldpay.dshackle.upstream.calls.CallSelector import io.emeraldpay.dshackle.upstream.ethereum.WsSubscriptions import io.emeraldpay.dshackle.upstream.finalization.FinalizationDetector import io.emeraldpay.dshackle.upstream.finalization.NoopFinalizationDetector -import org.springframework.cloud.sleuth.Tracer import reactor.core.publisher.Mono import reactor.core.scheduler.Scheduler import java.util.function.Supplier @@ -48,7 +47,7 @@ abstract class AbstractChainSpecific : ChainSpecific { return NoopFinalizationDetector() } - override fun makeCachingReaderBuilder(tracer: Tracer): CachingReaderBuilder { + override fun makeCachingReaderBuilder(): CachingReaderBuilder { return { _, _, _ -> NoopCachingReader } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/ChainSpecific.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/ChainSpecific.kt index fed279b89..1e03b59e4 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/ChainSpecific.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/ChainSpecific.kt @@ -1,5 +1,7 @@ package io.emeraldpay.dshackle.upstream.generic +import io.emeraldpay.dshackle.BlockchainType.AVM +import io.emeraldpay.dshackle.BlockchainType.AZTEC import io.emeraldpay.dshackle.BlockchainType.BITCOIN import io.emeraldpay.dshackle.BlockchainType.COSMOS import io.emeraldpay.dshackle.BlockchainType.ETHEREUM @@ -32,6 +34,8 @@ import io.emeraldpay.dshackle.upstream.UpstreamRpcMethodsDetector import io.emeraldpay.dshackle.upstream.UpstreamSettingsDetector import io.emeraldpay.dshackle.upstream.UpstreamValidator import io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult +import io.emeraldpay.dshackle.upstream.avm.AvmChainSpecific +import io.emeraldpay.dshackle.upstream.aztec.AztecChainSpecific import io.emeraldpay.dshackle.upstream.beaconchain.BeaconChainSpecific import io.emeraldpay.dshackle.upstream.calls.CallMethods import io.emeraldpay.dshackle.upstream.calls.CallSelector @@ -48,7 +52,6 @@ import io.emeraldpay.dshackle.upstream.solana.SolanaChainSpecific import io.emeraldpay.dshackle.upstream.starknet.StarknetChainSpecific import io.emeraldpay.dshackle.upstream.ton.TonHttpSpecific import org.apache.commons.collections4.Factory -import org.springframework.cloud.sleuth.Tracer import reactor.core.publisher.Mono import reactor.core.scheduler.Scheduler import java.util.function.Supplier @@ -77,7 +80,7 @@ interface ChainSpecific { fun subscriptionBuilder(headScheduler: Scheduler): (Multistream) -> EgressSubscription - fun makeCachingReaderBuilder(tracer: Tracer): CachingReaderBuilder + fun makeCachingReaderBuilder(): CachingReaderBuilder fun validator( chain: Chain, @@ -101,6 +104,12 @@ interface ChainSpecific { fun callSelector(caches: Caches): CallSelector? fun lowerBoundService(chain: Chain, upstream: Upstream): LowerBoundService + + /** + * List of HTTP response header names to forward from upstream to client. + * Override in chain-specific implementations to specify relevant headers. + */ + fun getResponseHeadersToForward(): List = emptyList() } object ChainSpecificRegistry { @@ -108,6 +117,8 @@ object ChainSpecificRegistry { @JvmStatic fun resolve(chain: Chain): ChainSpecific { return when (chain.type) { + AVM -> AvmChainSpecific + AZTEC -> AztecChainSpecific ETHEREUM -> EthereumChainSpecific STARKNET -> StarknetChainSpecific POLKADOT -> PolkadotChainSpecific diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericIngressSubscription.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericIngressSubscription.kt index a48485842..fb5d10c07 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericIngressSubscription.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericIngressSubscription.kt @@ -13,6 +13,7 @@ import reactor.core.publisher.Flux import reactor.core.publisher.Mono import java.time.Duration import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.TimeoutException class GenericIngressSubscription( val chain: Chain, @@ -26,7 +27,7 @@ class GenericIngressSubscription( private val holders = ConcurrentHashMap, SubscriptionConnect>() @Suppress("UNCHECKED_CAST") - override fun get(topic: String, params: Any?, unsubscribeMethod: String): SubscriptionConnect { + override fun get(topic: String, params: Any?, unsubscribeMethod: String): SubscriptionConnect { return holders.computeIfAbsent(topic to params) { key -> GenericSubscriptionConnect( chain, @@ -49,6 +50,7 @@ class GenericSubscriptionConnect( companion object { private val log = LoggerFactory.getLogger(GenericSubscriptionConnect::class.java) + private val IDLE_TIMEOUT: Duration = Duration.ofSeconds(85) } @Suppress("UNCHECKED_CAST") @@ -56,16 +58,16 @@ class GenericSubscriptionConnect( val sub = conn.subscribe(ChainRequest(topic, ListParams(getParams(params) as List))) return sub.data .flatMapMany { it.t2 } + // Some upstreams (notably Solana RPCs) silently stop delivering events on a subscription + // while keeping the WebSocket connection alive. Emit a TimeoutException so the surrounding + // DurableFlux re-invokes createConnection() and re-issues `subscribe` with a fresh subId. .timeout( - Duration.ofSeconds(85), - Mono.empty().doOnEach { - log.warn("Timeout during subscription to $topic after 85 seconds") - }, + IDLE_TIMEOUT, + Mono.error( + TimeoutException("No events from subscription to $topic in $IDLE_TIMEOUT, forcing resubscribe"), + ), ) - .onErrorResume { - log.error("Error during subscription to $topic", it) - Mono.empty() - } + .doOnError { log.warn("Error during subscription to $topic: {}", it.message) } .doFinally { if (unsubscribeMethod != "") { conn.unsubscribe( diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericUpstream.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericUpstream.kt index ecd3e63af..0db63e420 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericUpstream.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericUpstream.kt @@ -43,6 +43,7 @@ import reactor.core.Disposable import reactor.core.publisher.Flux import reactor.core.publisher.Sinks import reactor.core.scheduler.Schedulers +import java.net.URI import java.time.Duration import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicBoolean @@ -64,6 +65,7 @@ open class GenericUpstream( lowerBoundServiceBuilder: LowerBoundServiceBuilder, finalizationDetectorBuilder: FinalizationDetectorBuilder, versionRules: Supplier, + private val additionalSettings: UpstreamsConfig.AdditionalSettings?, ) : DefaultUpstream(id, hash, null, UpstreamAvailability.OK, options, role, targets, node, chainConfig, chain), Lifecycle { constructor( @@ -96,9 +98,12 @@ open class GenericUpstream( lowerBoundServiceBuilder, finalizationDetectorBuilder, versionRules, + config.additionalSettings, ) { rpcMethodsDetector = upstreamRpcMethodsDetectorBuilder(this, config) detectRpcMethods(config, buildMethods) + rpcConnectionUrl = (config.connection as? UpstreamsConfig.RpcConnection) + ?.let { it.rpc?.url ?: it.ws?.url } } private val validator: UpstreamValidator? = validatorBuilder(chain, this, getOptions(), chainConfig, versionRules) @@ -108,6 +113,8 @@ open class GenericUpstream( private val settingsDetectorSubscription = AtomicReference() private val hasLiveSubscriptionHead: AtomicBoolean = AtomicBoolean(getOptions().disableLivenessSubscriptionValidation) + private val hasPendingTxs = AtomicBoolean(getOptions().disablePendingTxValidation) + protected val connector: GenericConnector = connectorFactory.create(this, chain) .also { upConnector -> Gauge.builder("upstream_head", upConnector.getHead()) { @@ -118,9 +125,13 @@ open class GenericUpstream( .register(Metrics.globalRegistry) } private val livenessSubscription = AtomicReference() + private val pendingTxSubscription = AtomicReference() private val settingsDetector = upstreamSettingsDetectorBuilder(chain, this) private var rpcMethodsDetector: UpstreamRpcMethodsDetector? = null + // configured RPC/WS URL (carries query flags like ?hl=) + private var rpcConnectionUrl: URI? = null + private val lowerBoundService = lowerBoundServiceBuilder(chain, this) private val started = AtomicBoolean(false) @@ -148,11 +159,14 @@ open class GenericUpstream( // outdated, looks like applicable only for bitcoin and our ws_head trick override fun getCapabilities(): Set { - return if (hasLiveSubscriptionHead.get()) { - setOf(Capability.RPC, Capability.BALANCE, Capability.WS_HEAD) - } else { - setOf(Capability.RPC, Capability.BALANCE) + val caps = mutableSetOf(Capability.RPC, Capability.BALANCE) + if (hasLiveSubscriptionHead.get()) { + caps.add(Capability.WS_HEAD) } + if (hasPendingTxs.get()) { + caps.add(Capability.WS_PENDING_TX) + } + return caps } override fun isGrpc(): Boolean { @@ -176,6 +190,8 @@ open class GenericUpstream( ) } + fun getRpcConnectionUrl(): URI? = rpcConnectionUrl + @Suppress("UNCHECKED_CAST") override fun cast(selfType: Class): T { if (!selfType.isAssignableFrom(this.javaClass)) { @@ -356,6 +372,14 @@ open class GenericUpstream( ), ) } + if (!getOptions().disablePendingTxValidation) { + pendingTxSubscription.set( + connector.pendingTxEvents().subscribe { + hasPendingTxs.set(it) + sendUpstreamStateEvent(UPDATED) + }, + ) + } detectSettings() if (!getOptions().disableBoundValidation) { @@ -378,6 +402,7 @@ open class GenericUpstream( lowerBlockDetectorSubscription.getAndSet(null)?.dispose() finalizationDetectorSubscription.getAndSet(null)?.dispose() settingsDetectorSubscription.getAndSet(null)?.dispose() + pendingTxSubscription.getAndSet(null)?.dispose() connector.getHead().stop() } @@ -443,6 +468,10 @@ open class GenericUpstream( return lowerBoundService.predictLowerBound(type, timeOffsetSeconds) } + override fun getAdditionalSettings(): UpstreamsConfig.AdditionalSettings? { + return additionalSettings + } + fun isValid(): Boolean = isUpstreamValid.get() companion object { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericConnector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericConnector.kt index 9d58c153b..bb12a6b5d 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericConnector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericConnector.kt @@ -15,4 +15,6 @@ interface GenericConnector : Lifecycle { fun getIngressReader(): ChainReader fun getIngressSubscription(): IngressSubscription + + fun pendingTxEvents(): Flux } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericRpcConnector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericRpcConnector.kt index 7e0a60b12..d8a5be096 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericRpcConnector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericRpcConnector.kt @@ -13,11 +13,15 @@ import io.emeraldpay.dshackle.upstream.Lifecycle import io.emeraldpay.dshackle.upstream.MergedHead import io.emeraldpay.dshackle.upstream.NoIngressSubscription import io.emeraldpay.dshackle.upstream.ethereum.AlwaysHeadLivenessValidator +import io.emeraldpay.dshackle.upstream.ethereum.BASE_TX_LIMIT import io.emeraldpay.dshackle.upstream.ethereum.GenericWsHead import io.emeraldpay.dshackle.upstream.ethereum.HeadLivenessState import io.emeraldpay.dshackle.upstream.ethereum.HeadLivenessValidator import io.emeraldpay.dshackle.upstream.ethereum.HeadLivenessValidatorImpl import io.emeraldpay.dshackle.upstream.ethereum.NoHeadLivenessValidator +import io.emeraldpay.dshackle.upstream.ethereum.NoopPendingTransactionValidator +import io.emeraldpay.dshackle.upstream.ethereum.PendingTransactionValidator +import io.emeraldpay.dshackle.upstream.ethereum.PendingTransactionValidatorImpl import io.emeraldpay.dshackle.upstream.ethereum.WsConnectionPool import io.emeraldpay.dshackle.upstream.ethereum.WsConnectionPoolFactory import io.emeraldpay.dshackle.upstream.ethereum.WsSubscriptions @@ -58,6 +62,7 @@ class GenericRpcConnector( private val head: Head private val liveness: HeadLivenessValidator private val jsonRpcWsClient: JsonRpcWsClient? + private val pendingTxValidator: PendingTransactionValidator companion object { private val log = LoggerFactory.getLogger(GenericRpcConnector::class.java) @@ -136,6 +141,16 @@ class GenericRpcConnector( ) } } + pendingTxValidator = if (chain == Chain.BASE__MAINNET) { + PendingTransactionValidatorImpl( + upstream.getId(), + getIngressReader(), + Duration.ofMinutes(5), + BASE_TX_LIMIT, + ) + } else { + NoopPendingTransactionValidator() + } liveness = if (connectorType != RPC_ONLY && isSpecialChain(chain)) { AlwaysHeadLivenessValidator() @@ -186,6 +201,10 @@ class GenericRpcConnector( return ingressSubscription ?: NoIngressSubscription() } + override fun pendingTxEvents(): Flux { + return pendingTxValidator.pendingTxExists() + } + override fun getHead(): Head { return head } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericWsConnector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericWsConnector.kt index 98794195b..15c1b2d99 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericWsConnector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericWsConnector.kt @@ -6,9 +6,13 @@ import io.emeraldpay.dshackle.upstream.BlockValidator import io.emeraldpay.dshackle.upstream.DefaultUpstream import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.IngressSubscription +import io.emeraldpay.dshackle.upstream.ethereum.BASE_TX_LIMIT import io.emeraldpay.dshackle.upstream.ethereum.GenericWsHead import io.emeraldpay.dshackle.upstream.ethereum.HeadLivenessState import io.emeraldpay.dshackle.upstream.ethereum.HeadLivenessValidatorImpl +import io.emeraldpay.dshackle.upstream.ethereum.NoopPendingTransactionValidator +import io.emeraldpay.dshackle.upstream.ethereum.PendingTransactionValidator +import io.emeraldpay.dshackle.upstream.ethereum.PendingTransactionValidatorImpl import io.emeraldpay.dshackle.upstream.ethereum.WsConnectionPool import io.emeraldpay.dshackle.upstream.ethereum.WsConnectionPoolFactory import io.emeraldpay.dshackle.upstream.ethereum.WsSubscriptionsImpl @@ -36,6 +40,8 @@ class GenericWsConnector( private val head: GenericWsHead private val subscriptions: IngressSubscription private val liveness: HeadLivenessValidatorImpl + private val pendingTxValidator: PendingTransactionValidator + init { pool = wsFactory.create(upstream) reader = JsonRpcWsClient(pool) @@ -54,6 +60,17 @@ class GenericWsConnector( ) liveness = HeadLivenessValidatorImpl(head, expectedBlockTime, headLivenessScheduler, upstream.getId()) subscriptions = chainSpecific.makeIngressSubscription(chain, wsSubscriptions) + + pendingTxValidator = if (chain == Chain.BASE__MAINNET) { + PendingTransactionValidatorImpl( + upstream.getId(), + getIngressReader(), + Duration.ofMinutes(10), + BASE_TX_LIMIT, + ) + } else { + NoopPendingTransactionValidator() + } } override fun headLivenessEvents(): Flux { @@ -81,6 +98,10 @@ class GenericWsConnector( return subscriptions } + override fun pendingTxEvents(): Flux { + return pendingTxValidator.pendingTxExists() + } + override fun getHead(): Head { return head } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcHead.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcHead.kt index 493c24863..2acd0b75b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcHead.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcHead.kt @@ -35,7 +35,7 @@ import reactor.core.publisher.Flux import reactor.core.publisher.Mono import reactor.core.publisher.Sinks import reactor.core.scheduler.Scheduler -import reactor.kotlin.extra.retry.retryExponentialBackoff +import reactor.util.retry.Retry import java.time.Duration import java.util.function.Function @@ -108,14 +108,14 @@ class GrpcHead( blocks = blocks.doOnNext { log.trace("Received block ${it.height}") - }.retryExponentialBackoff( - Long.MAX_VALUE, - Duration.ofMillis(100), - Duration.ofSeconds(60), - true, - ) { - log.debug("Retry grpc head connection ${parent.getId()}") - } + }.retryWhen( + Retry.backoff(Long.MAX_VALUE, Duration.ofMillis(100)) + .maxBackoff(Duration.ofSeconds(60)) + .jitter(0.5) + .doBeforeRetry { + log.debug("Retry grpc head connection ${parent.getId()}") + }, + ) headSubscription = super.follow(blocks) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreamCreator.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreamCreator.kt index 744adef05..012b69f2b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreamCreator.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreamCreator.kt @@ -22,11 +22,11 @@ class GrpcUpstreamCreator( private val authorizationConfig: AuthorizationConfig, private val compressionConfig: CompressionConfig, private val fileResolver: FileResolver, - @Qualifier("grpcChannelExecutor") + @param:Qualifier("grpcChannelExecutor") private val channelExecutor: Executor, - private val grpcTracing: GrpcTracing, - @Qualifier("headScheduler") + @param:Qualifier("headScheduler") private val headScheduler: Scheduler, + private val grpcTracing: GrpcTracing, private val grpcAuthContext: GrpcAuthContext, ) { @Value("\${spring.application.max-metadata-size}") diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt index dbdf582d7..e08198499 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/grpc/GrpcUpstreams.kt @@ -146,7 +146,7 @@ class GrpcUpstreams( return Flux.interval(Duration.ZERO, Duration.ofSeconds(20)) .flatMap { authAndDescribe(grpcUpstreamsAuth) - }.onErrorContinue { t, _ -> + }.onErrorContinue { t, _: Any? -> if (ExceptionUtils.indexOfType(t, IOException::class.java) >= 0) { log.warn("gRPC upstream $host:$port is unavailable. (${t.javaClass}: ${t.message})") known.values.forEach { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/kadena/KadenaChainSpecific.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/kadena/KadenaChainSpecific.kt index d9ee996fd..87b79f232 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/kadena/KadenaChainSpecific.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/kadena/KadenaChainSpecific.kt @@ -107,8 +107,8 @@ object KadenaChainSpecific : AbstractPollChainSpecific() { @JsonIgnoreProperties(ignoreUnknown = true) data class KadenaHeader( - @JsonProperty("height") var height: Long, - @JsonProperty("weight") var weight: String, - @JsonProperty("instance") var instance: String, - @JsonProperty("id") var id: String, + @param:JsonProperty("height") var height: Long, + @param:JsonProperty("weight") var weight: String, + @param:JsonProperty("instance") var instance: String, + @param:JsonProperty("id") var id: String, ) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/kadena/KadenaUpstreamSettingsDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/kadena/KadenaUpstreamSettingsDetector.kt index 0463f35f9..9d0bfc7c2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/kadena/KadenaUpstreamSettingsDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/kadena/KadenaUpstreamSettingsDetector.kt @@ -32,7 +32,7 @@ class KadenaUpstreamSettingsDetector( @JsonIgnoreProperties(ignoreUnknown = true) data class KadenaHeader( - @JsonProperty("instance") var instance: String, + @param:JsonProperty("instance") var instance: String, ) override fun nodeTypeRequest(): NodeTypeRequest = NodeTypeRequest(clientVersionRequest()) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/GoldLowerBounds.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/GoldLowerBounds.kt new file mode 100644 index 000000000..82a8ab78d --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/GoldLowerBounds.kt @@ -0,0 +1,28 @@ +package io.emeraldpay.dshackle.upstream.lowerbound + +import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.config.ChainsConfig + +object GoldLowerBounds { + + private lateinit var bounds: Map> + + fun init(chainConfigs: Collection) { + bounds = chainConfigs.filter { + it.shortNames.isNotEmpty() && + Global.chainById(it.shortNames[0]) != Chain.UNSPECIFIED && + it.goldLowerBounds.isNotEmpty() + }.associate { cfg -> + Global.chainById(cfg.shortNames[0]) to cfg.goldLowerBounds.mapKeys { toLowerBoundType(it.key) } + } + } + + fun getBound(chain: Chain, boundType: LowerBoundType): ChainsConfig.GoldLowerBound? { + return bounds[chain]?.get(boundType) + } + + private fun toLowerBoundType(type: ChainsConfig.LowerBoundType): LowerBoundType { + return LowerBoundType.byName(type.name) + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundData.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundData.kt index 20ccc7a0d..175686f47 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundData.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundData.kt @@ -20,7 +20,23 @@ data class LowerBoundData( } enum class LowerBoundType { - UNKNOWN, STATE, SLOT, BLOCK, TX, LOGS, TRACE, PROOF, BLOB, EPOCH, RECEIPTS + UNKNOWN, STATE, SLOT, BLOCK, TX, LOGS, TRACE, PROOF, BLOB, EPOCH, RECEIPTS; + + companion object { + fun byName(name: String): LowerBoundType { + return entries.firstOrNull { it.name.equals(name, ignoreCase = true) } ?: UNKNOWN + } + } +} + +enum class ManualLowerBoundType { + HEAD, FIXED; + + companion object { + fun byName(name: String): ManualLowerBoundType? { + return entries.firstOrNull { it.name.equals(name, ignoreCase = true) } + } + } } fun BlockchainOuterClass.LowerBoundType.fromProtoType(): LowerBoundType { diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundDetector.kt index 857ee9608..e56226bf0 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundDetector.kt @@ -18,7 +18,7 @@ abstract class LowerBoundDetector( protected val lowerBounds = LowerBounds(chain) private val lowerBoundSink = Sinks.many().multicast().directBestEffort() - fun detectLowerBound(): Flux { + fun detectLowerBound(manualBoundsService: ManualLowerBoundService): Flux { val notProcessing = AtomicBoolean(true) return Flux.merge( @@ -29,15 +29,25 @@ abstract class LowerBoundDetector( ) .filter { notProcessing.get() } .flatMap { - notProcessing.set(false) - internalDetectLowerBound() - .onErrorResume { Mono.just(LowerBoundData.default()) } - .switchIfEmpty(Flux.just(LowerBoundData.default())) - .doFinally { notProcessing.set(true) } + if (types() == manualBoundsService.manualBoundTypes()) { + Flux.fromIterable(types()) + .mapNotNull { manualBoundsService.manualLowerBound(it) } + } else { + notProcessing.set(false) + Flux.merge( + internalDetectLowerBound() + .filter { !manualBoundsService.hasManualBound(it.type) } + .onErrorResume { Mono.just(LowerBoundData.default()) } + .switchIfEmpty(Flux.just(LowerBoundData.default())) + .doFinally { notProcessing.set(true) }, + Flux.fromIterable(types()) + .mapNotNull { manualBoundsService.manualLowerBound(it) }, + ) + } }, ) .filter { - it.lowerBound >= (lowerBounds.getLastBound(it.type)?.lowerBound ?: 0) + (it.lowerBound >= (lowerBounds.getLastBound(it.type)?.lowerBound ?: 0)) || it.lowerBound == 1L } .map { lowerBounds.updateBound(it) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundService.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundService.kt index c0693aa8b..c82ef43b2 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundService.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundService.kt @@ -16,10 +16,14 @@ abstract class LowerBoundService( private val lowerBounds = ConcurrentHashMap() private val detectors: List by lazy { detectors() } + private val manualBoundsService = BaseManualLowerBoundService( + upstream, + upstream.getAdditionalSettings()?.manualLowerBounds ?: emptyMap(), + ) fun detectLowerBounds(): Flux { return Flux.merge( - detectors.map { it.detectLowerBound() }, + detectors.map { it.detectLowerBound(manualBoundsService) }, ) .doOnNext { log.info("Lower bound of type ${it.type} is ${it.lowerBound} for upstream ${upstream.getId()} of chain $chain") diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/ManualLowerBoundService.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/ManualLowerBoundService.kt new file mode 100644 index 000000000..2bbadcdf3 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/ManualLowerBoundService.kt @@ -0,0 +1,87 @@ +package io.emeraldpay.dshackle.upstream.lowerbound + +import io.emeraldpay.dshackle.config.UpstreamsConfig +import io.emeraldpay.dshackle.upstream.Upstream + +interface ManualLowerBoundService { + fun manualLowerBound(type: LowerBoundType): LowerBoundData? + fun hasManualBound(type: LowerBoundType): Boolean + fun manualBoundTypes(): Set +} + +class NoopManualLowerBoundService : ManualLowerBoundService { + override fun manualLowerBound(type: LowerBoundType): LowerBoundData? { + return null + } + + override fun hasManualBound(type: LowerBoundType): Boolean { + return false + } + + override fun manualBoundTypes(): Set { + return emptySet() + } +} + +class BaseManualLowerBoundService( + private val upstream: Upstream, + private val manualLowerBoundSettings: Map, +) : ManualLowerBoundService { + private val providers = manualLowerBoundSettings.mapValues { provider(it.value) } + + override fun manualLowerBound(type: LowerBoundType): LowerBoundData? { + val provider = providers[type] ?: return null + if (!provider.canHandle()) { + return null + } + return LowerBoundData(provider.getManualLowerBound(), type) + } + + override fun hasManualBound(type: LowerBoundType): Boolean { + return manualLowerBoundSettings.contains(type) + } + + override fun manualBoundTypes(): Set { + return manualLowerBoundSettings.keys + } + + private fun provider(setting: UpstreamsConfig.ManualBoundSetting): ManualLowerBoundProvider { + return when (setting.type) { + ManualLowerBoundType.HEAD -> HeadManualLowerBoundProvider(upstream, setting) + ManualLowerBoundType.FIXED -> FixedManualLowerBoundProvider(setting) + } + } +} + +interface ManualLowerBoundProvider { + fun getManualLowerBound(): Long + fun canHandle(): Boolean +} + +class HeadManualLowerBoundProvider( + private val upstream: Upstream, + private val setting: UpstreamsConfig.ManualBoundSetting, +) : ManualLowerBoundProvider { + override fun getManualLowerBound(): Long { + return upstream.getHead().getCurrentHeight()!! + setting.value + } + + override fun canHandle(): Boolean { + return setting.type == ManualLowerBoundType.HEAD && + setting.value < 0 && + upstream.getHead().getCurrentHeight() != null && + upstream.getHead().getCurrentHeight()!! + setting.value >= 0 + } +} + +class FixedManualLowerBoundProvider( + private val setting: UpstreamsConfig.ManualBoundSetting, +) : ManualLowerBoundProvider { + override fun getManualLowerBound(): Long { + return setting.value + } + + override fun canHandle(): Boolean { + return setting.type == ManualLowerBoundType.FIXED && setting.value > 0 + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/near/NearChainSpecific.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/near/NearChainSpecific.kt index 2caeebe3d..0060725f6 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/near/NearChainSpecific.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/near/NearChainSpecific.kt @@ -119,26 +119,26 @@ object NearChainSpecific : AbstractPollChainSpecific() { @JsonIgnoreProperties(ignoreUnknown = true) data class NearBlock( - @JsonProperty("header") var header: NearHeader, + @param:JsonProperty("header") var header: NearHeader, ) @JsonIgnoreProperties(ignoreUnknown = true) data class NearHeader( - @JsonProperty("height") var height: Long, - @JsonProperty("hash") var hash: String, - @JsonProperty("prev_hash") var prevHash: String, - @JsonProperty("timestamp") var timestamp: Long, + @param:JsonProperty("height") var height: Long, + @param:JsonProperty("hash") var hash: String, + @param:JsonProperty("prev_hash") var prevHash: String, + @param:JsonProperty("timestamp") var timestamp: Long, ) @JsonIgnoreProperties(ignoreUnknown = true) data class NearStatus( - @JsonProperty("chain_id") var chainId: String, - @JsonProperty("sync_info") var syncInfo: NearSync, + @param:JsonProperty("chain_id") var chainId: String, + @param:JsonProperty("sync_info") var syncInfo: NearSync, ) @JsonIgnoreProperties(ignoreUnknown = true) data class NearSync( - @JsonProperty("syncing") var syncing: Boolean, - @JsonProperty("earliest_block_height") var earliestHeight: Long, - @JsonProperty("earliest_block_time") var earliestBlockTime: Instant, + @param:JsonProperty("syncing") var syncing: Boolean, + @param:JsonProperty("earliest_block_height") var earliestHeight: Long, + @param:JsonProperty("earliest_block_time") var earliestBlockTime: Instant, ) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/near/NearUpstreamSettingsDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/near/NearUpstreamSettingsDetector.kt index 120667fa1..34873424b 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/near/NearUpstreamSettingsDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/near/NearUpstreamSettingsDetector.kt @@ -32,13 +32,13 @@ class NearUpstreamSettingsDetector( @JsonIgnoreProperties(ignoreUnknown = true) private data class NearVersionResponse( - @JsonProperty("version") + @param:JsonProperty("version") val nearVersion: NearVersion, ) @JsonIgnoreProperties(ignoreUnknown = true) private data class NearVersion( - @JsonProperty("version") + @param:JsonProperty("version") val version: String, ) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/polkadot/PolkadotChainSpecific.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/polkadot/PolkadotChainSpecific.kt index 17a9c8709..967cbab4e 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/polkadot/PolkadotChainSpecific.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/polkadot/PolkadotChainSpecific.kt @@ -158,30 +158,30 @@ object PolkadotChainSpecific : AbstractPollChainSpecific() { @JsonIgnoreProperties(ignoreUnknown = true) data class PolkadotBlockResponse( - @JsonProperty("block") var block: PolkadotBlock, + @param:JsonProperty("block") var block: PolkadotBlock, ) @JsonIgnoreProperties(ignoreUnknown = true) data class PolkadotBlock( - @JsonProperty("header") var header: PolkadotHeader, + @param:JsonProperty("header") var header: PolkadotHeader, ) @JsonIgnoreProperties(ignoreUnknown = true) data class PolkadotHeader( - @JsonProperty("parentHash") var parentHash: String, - @JsonProperty("number") var number: String, - @JsonProperty("stateRoot") var stateRoot: String, - @JsonProperty("extrinsicsRoot") var extrinsicsRoot: String, - @JsonProperty("digest") var digest: PolkadotDigest, + @param:JsonProperty("parentHash") var parentHash: String, + @param:JsonProperty("number") var number: String, + @param:JsonProperty("stateRoot") var stateRoot: String, + @param:JsonProperty("extrinsicsRoot") var extrinsicsRoot: String, + @param:JsonProperty("digest") var digest: PolkadotDigest, ) data class PolkadotDigest( - @JsonProperty("logs") var logs: List, + @param:JsonProperty("logs") var logs: List, ) @JsonIgnoreProperties(ignoreUnknown = true) data class PolkadotHealth( - @JsonProperty("peers") var peers: Long, - @JsonProperty("isSyncing") var isSyncing: Boolean, - @JsonProperty("shouldHavePeers") var shouldHavePeers: Boolean, + @param:JsonProperty("peers") var peers: Long, + @param:JsonProperty("isSyncing") var isSyncing: Boolean, + @param:JsonProperty("shouldHavePeers") var shouldHavePeers: Boolean, ) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/RestHttpReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/RestHttpReader.kt index 1b0a2b247..ee83d6e72 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/RestHttpReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/restclient/RestHttpReader.kt @@ -1,11 +1,15 @@ package io.emeraldpay.dshackle.upstream.restclient +import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.config.AuthConfig +import io.emeraldpay.dshackle.upstream.ChainCallError import io.emeraldpay.dshackle.upstream.ChainRequest import io.emeraldpay.dshackle.upstream.ChainResponse import io.emeraldpay.dshackle.upstream.HttpReader import io.emeraldpay.dshackle.upstream.RequestMetrics +import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError +import io.emeraldpay.dshackle.upstream.generic.ChainSpecificRegistry import io.emeraldpay.dshackle.upstream.rpcclient.ResponseRpcParser import io.emeraldpay.dshackle.upstream.rpcclient.RestParams import io.emeraldpay.dshackle.upstream.stream.AggregateResponse @@ -17,7 +21,9 @@ import io.netty.handler.codec.http.HttpMethod import org.apache.commons.lang3.time.StopWatch import reactor.core.publisher.Flux import reactor.core.publisher.Mono +import reactor.core.scheduler.Scheduler import reactor.kotlin.core.publisher.switchIfEmpty +import reactor.netty.http.client.HttpClientResponse import java.util.concurrent.TimeUnit class RestHttpReader( @@ -25,12 +31,23 @@ class RestHttpReader( maxConnections: Int, queueSize: Int, metrics: RequestMetrics, + private val httpScheduler: Scheduler, + private val chain: Chain, basicAuth: AuthConfig.ClientBasicAuth? = null, tlsCAAuth: ByteArray? = null, -) : HttpReader(target, maxConnections, queueSize, metrics, basicAuth, tlsCAAuth) { + customHeaders: Map = emptyMap(), + bearerAuth: AuthConfig.ClientBearerAuth? = null, +) : HttpReader(target, maxConnections, queueSize, metrics, basicAuth, tlsCAAuth, customHeaders, bearerAuth) { private val parser = ResponseRpcParser() private val requestParser = RestRequestParser + private val headersToForward = ChainSpecificRegistry.resolve(chain).getResponseHeadersToForward() + + private fun extractResponseHeaders(header: HttpClientResponse): Map { + return headersToForward + .mapNotNull { name -> header.responseHeaders().get(name)?.let { name to it } } + .toMap() + } override fun internalRead(key: ChainRequest): Mono { val startTime = StopWatch() @@ -43,18 +60,26 @@ class RestHttpReader( .flatMap(this::execute) .doOnNext { if (startTime.isStarted) { - metrics?.timer?.record(startTime.nanoTime, TimeUnit.NANOSECONDS) + metrics?.timer(key.method)?.record(startTime.nanoTime, TimeUnit.NANOSECONDS) } } .handle { it, sink -> when (it) { - is StreamResponse -> sink.next(ChainResponse(it.stream, key.id)) + is StreamResponse -> sink.next(ChainResponse(it.stream, key.id, it.headers)) is AggregateResponse -> { - if (it.code != 200) { + if (it.code == 503) { + // 503 bodies (e.g. Cloudflare) are typically plain text / HTML, not JSON, + // so skip JSON parsing to avoid a misleading "Unrecognized token" error + val error = ChainCallError( + RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE, + "HTTP Code: 503", + ) + sink.next(ChainResponse(null, error, it.headers)) + } else if (it.code != 200) { val error = parser.readError(Global.objectMapper.createParser(it.response)) - sink.next(ChainResponse(null, error)) + sink.next(ChainResponse(null, error, it.headers)) } else { - sink.next(ChainResponse(it.response, null)) + sink.next(ChainResponse(it.response, null, it.headers)) } } else -> sink.error(IllegalStateException("Wrong response type")) @@ -85,26 +110,30 @@ class RestHttpReader( return if (!key.isStreamed) { response.response { header, bytes -> val statusCode = header.status().code() + val responseHeaders = extractResponseHeaders(header) - bytes.aggregate().asByteArray().map { - AggregateResponse(it, statusCode) + bytes.aggregate().asByteArray().publishOn(httpScheduler).map { + AggregateResponse(it, statusCode, responseHeaders) }.switchIfEmpty { - Mono.just(AggregateResponse(ByteArray(0), statusCode)) + Mono.just(AggregateResponse(ByteArray(0), statusCode, responseHeaders)) } }.single() } else { response.responseConnection { t, u -> + val responseHeaders = extractResponseHeaders(t) + if (t.status().code() != 200) { - u.inbound().receive().aggregate().asByteArray() - .map { AggregateResponse(it, t.status().code()) } + u.inbound().receive().aggregate().asByteArray().publishOn(httpScheduler) + .map { AggregateResponse(it, t.status().code(), responseHeaders) } } else { Mono.just( StreamResponse( Flux.concat( - u.inbound().receive().asByteArray() + u.inbound().receive().asByteArray().publishOn(httpScheduler) .map { Chunk(it, false) }, Mono.just(Chunk(ByteArray(0), true)), ), + responseHeaders, ), ) } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ripple/RippleChainSpecific.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ripple/RippleChainSpecific.kt index a2a869fff..3ce071bbc 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/ripple/RippleChainSpecific.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/ripple/RippleChainSpecific.kt @@ -3,7 +3,6 @@ package io.emeraldpay.dshackle.upstream.ripple import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.module.kotlin.readValue import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.config.ChainsConfig.ChainConfig @@ -13,71 +12,84 @@ import io.emeraldpay.dshackle.foundation.ChainOptions.Options import io.emeraldpay.dshackle.reader.ChainReader import io.emeraldpay.dshackle.upstream.BasicUpstreamSettingsDetector import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.EgressSubscription import io.emeraldpay.dshackle.upstream.GenericSingleCallValidator +import io.emeraldpay.dshackle.upstream.IngressSubscription +import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.NodeTypeRequest import io.emeraldpay.dshackle.upstream.SingleValidator import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.UpstreamAvailability import io.emeraldpay.dshackle.upstream.UpstreamSettingsDetector import io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult +import io.emeraldpay.dshackle.upstream.ethereum.WsSubscriptions import io.emeraldpay.dshackle.upstream.generic.AbstractPollChainSpecific +import io.emeraldpay.dshackle.upstream.generic.GenericEgressSubscription +import io.emeraldpay.dshackle.upstream.generic.GenericIngressSubscription import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundService import io.emeraldpay.dshackle.upstream.rpcclient.ListParams -import org.slf4j.LoggerFactory +import io.emeraldpay.dshackle.upstream.rpcclient.RippleCommandParams import reactor.core.publisher.Flux import reactor.core.publisher.Mono +import reactor.core.scheduler.Scheduler import java.math.BigInteger import java.time.Instant object RippleChainSpecific : AbstractPollChainSpecific() { - private val log = LoggerFactory.getLogger(RippleChainSpecific::class.java) - override fun parseBlock(data: ByteArray, upstreamId: String, api: ChainReader): Mono { + // Parse ledger_closed response: { "ledger_hash": "...", "ledger_index": 123 } val jsonNode = Global.objectMapper.readTree(data) - - val block: RippleClosedLedger = if (jsonNode.has("ledger")) { - Global.objectMapper.treeToValue(jsonNode.get("ledger"), RippleClosedLedger::class.java) - } else { - val result = Global.objectMapper.readValue(data, RippleBlock::class.java) - result.closed.ledger - } - - var height: Long = 0 - try { - height = block.ledgerIndex.toLong() - } catch (e: NumberFormatException) { - log.error("Invalid ledgerIndex ${block.ledgerIndex}, upstreamId:$upstreamId") - } + val ledgerHash = jsonNode.get("ledger_hash").asText() + val ledgerIndex = jsonNode.get("ledger_index").asLong() return Mono.just( BlockContainer( - height = height, - hash = BlockId.from(block.ledgerHash), + height = ledgerIndex, + hash = BlockId.from(ledgerHash), difficulty = BigInteger.ZERO, timestamp = Instant.EPOCH, full = false, json = data, - parsed = block, + parsed = jsonNode, transactions = emptyList(), upstreamId = upstreamId, - parentHash = BlockId.from(block.parentHash), + parentHash = null, ), ) } override fun getFromHeader(data: ByteArray, upstreamId: String, api: ChainReader): Mono { - throw NotImplementedError() - } + // Parse Ripple ledger stream event: { "type": "ledgerClosed", "ledger_hash": "...", "ledger_index": 123, ... } + val event = Global.objectMapper.readValue(data, RippleLedgerStreamEvent::class.java) - override fun listenNewHeadsRequest(): ChainRequest { - throw NotImplementedError() - } + // Ripple epoch starts at 2000-01-01 00:00:00 UTC (946684800 seconds after Unix epoch) + val timestamp = event.ledgerTime?.let { + Instant.ofEpochSecond(it + 946684800L) + } ?: Instant.EPOCH - override fun unsubscribeNewHeadsRequest(subId: Any): ChainRequest { - throw NotImplementedError() + return Mono.just( + BlockContainer( + height = event.ledgerIndex, + hash = BlockId.from(event.ledgerHash), + difficulty = BigInteger.ZERO, + timestamp = timestamp, + full = false, + json = data, + parsed = event, + transactions = emptyList(), + upstreamId = upstreamId, + parentHash = null, // Not available in stream event + ), + ) } + override fun listenNewHeadsRequest(): ChainRequest = + ChainRequest("subscribe", RippleCommandParams("streams" to listOf("ledger"))) + + override fun unsubscribeNewHeadsRequest(subId: Any): ChainRequest = + ChainRequest("unsubscribe", RippleCommandParams("streams" to listOf("ledger"))) + override fun upstreamValidators( chain: Chain, upstream: Upstream, @@ -132,11 +144,19 @@ object RippleChainSpecific : AbstractPollChainSpecific() { } override fun latestBlockRequest(): ChainRequest = - ChainRequest("ledger", ListParams()) + ChainRequest("ledger_closed", ListParams()) override fun upstreamSettingsDetector(chain: Chain, upstream: Upstream): UpstreamSettingsDetector? { return RippleUpstreamSettingsDetector(upstream) } + + override fun makeIngressSubscription(chain: Chain, ws: WsSubscriptions): IngressSubscription { + return GenericIngressSubscription(chain, ws, listOf("ledger")) + } + + override fun subscriptionBuilder(headScheduler: Scheduler): (Multistream) -> EgressSubscription { + return { ms -> GenericEgressSubscription(ms, headScheduler) } + } } class RippleUpstreamSettingsDetector(val upstream: Upstream) : BasicUpstreamSettingsDetector(upstream) { @@ -181,124 +201,101 @@ class RippleUpstreamSettingsDetector(val upstream: Upstream) : BasicUpstreamSett @JsonIgnoreProperties(ignoreUnknown = true) data class RippleInfoWrapper( - @JsonProperty("info") var info: RippleInfo, + @param:JsonProperty("info") var info: RippleInfo, ) @JsonIgnoreProperties(ignoreUnknown = true) data class RippleInfo( - @JsonProperty("clio_version") var clio: String?, - @JsonProperty("build_version") var buildVersion: String?, -) - -@JsonIgnoreProperties(ignoreUnknown = true) -data class RippleBlock( - @JsonProperty("closed") var closed: RippleClosed, - @JsonProperty("open") var open: RippleOpen?, - @JsonProperty("status") var status: String?, -) - -@JsonIgnoreProperties(ignoreUnknown = true) -data class RippleClosed( - @JsonProperty("ledger") var ledger: RippleClosedLedger, -) - -@JsonIgnoreProperties(ignoreUnknown = true) -data class RippleOpen( - @JsonProperty("ledger") var ledger: RippleOpenLedger, -) - -@JsonIgnoreProperties(ignoreUnknown = true) -data class RippleClosedLedger( - @JsonProperty("account_hash") var accountHash: String, - @JsonProperty("close_flags") var closeFlags: Short, - @JsonProperty("close_time") var closeTime: Long, - @JsonProperty("close_time_human") var closeTimeHuman: String, - @JsonProperty("close_time_iso") var closeTimeIso: String, - @JsonProperty("close_time_resolution") var closeTimeResolution: Short, - @JsonProperty("closed") var closed: Boolean, - @JsonProperty("ledger_hash") var ledgerHash: String, - @JsonProperty("ledger_index") var ledgerIndex: String, - @JsonProperty("parent_close_time") var parentCloseTime: Long, - @JsonProperty("parent_hash") var parentHash: String, - @JsonProperty("total_coins") var totalCoins: String, - @JsonProperty("transaction_hash") var transactionHash: String, -) - -@JsonIgnoreProperties(ignoreUnknown = true) -data class RippleOpenLedger( - @JsonProperty("closed") var closed: Boolean, - @JsonProperty("ledger_index") var ledgerIndex: String, - @JsonProperty("parent_hash") var parentHash: String, + @param:JsonProperty("clio_version") var clio: String?, + @param:JsonProperty("build_version") var buildVersion: String?, ) @JsonIgnoreProperties(ignoreUnknown = true) data class RippleState( - @JsonProperty("state") var state: RippleServerState, - @JsonProperty("status") var status: String? = null, + @param:JsonProperty("state") var state: RippleServerState, + @param:JsonProperty("status") var status: String? = null, ) @JsonIgnoreProperties(ignoreUnknown = true) data class RippleServerState( - @JsonProperty("build_version") val buildVersion: String, - @JsonProperty("complete_ledgers") val completeLedgers: String, - @JsonProperty("initial_sync_duration_us") val initialSyncDurationUs: String?, - @JsonProperty("io_latency_ms") val ioLatencyMs: Int, - @JsonProperty("jq_trans_overflow") val jqTransOverflow: String, - @JsonProperty("last_close") val lastClose: LastClose, - @JsonProperty("load_base") val loadBase: Int, - @JsonProperty("load_factor") val loadFactor: Int, - @JsonProperty("load_factor_fee_escalation") val loadFactorFeeEscalation: Int, - @JsonProperty("load_factor_fee_queue") val loadFactorFeeQueue: Int, - @JsonProperty("load_factor_fee_reference") val loadFactorFeeReference: Int, - @JsonProperty("load_factor_server") val loadFactorServer: Int, - @JsonProperty("network_id") val networkId: Int, - @JsonProperty("peer_disconnects") val peerDisconnects: String?, - @JsonProperty("peer_disconnects_resources") val peerDisconnectsResources: String?, - @JsonProperty("peers") val peers: Int, - @JsonProperty("ports") val ports: List, - @JsonProperty("pubkey_node") val pubkeyNode: String?, - @JsonProperty("server_state") val serverState: String?, - @JsonProperty("server_state_duration_us") val serverStateDurationUs: String?, - @JsonProperty("state_accounting") val stateAccounting: StateAccounting?, - @JsonProperty("time") val time: String, - @JsonProperty("uptime") val uptime: Long, - @JsonProperty("validated_ledger") val validatedLedger: ValidatedLedger, - @JsonProperty("validation_quorum") val validationQuorum: Int, + @param:JsonProperty("build_version") val buildVersion: String, + @param:JsonProperty("complete_ledgers") val completeLedgers: String, + @param:JsonProperty("initial_sync_duration_us") val initialSyncDurationUs: String?, + @param:JsonProperty("io_latency_ms") val ioLatencyMs: Int, + @param:JsonProperty("jq_trans_overflow") val jqTransOverflow: String, + @param:JsonProperty("last_close") val lastClose: LastClose, + @param:JsonProperty("load_base") val loadBase: Int, + @param:JsonProperty("load_factor") val loadFactor: Int, + @param:JsonProperty("load_factor_fee_escalation") val loadFactorFeeEscalation: Int, + @param:JsonProperty("load_factor_fee_queue") val loadFactorFeeQueue: Int, + @param:JsonProperty("load_factor_fee_reference") val loadFactorFeeReference: Int, + @param:JsonProperty("load_factor_server") val loadFactorServer: Int, + @param:JsonProperty("network_id") val networkId: Int, + @param:JsonProperty("peer_disconnects") val peerDisconnects: String?, + @param:JsonProperty("peer_disconnects_resources") val peerDisconnectsResources: String?, + @param:JsonProperty("peers") val peers: Int, + @param:JsonProperty("ports") val ports: List, + @param:JsonProperty("pubkey_node") val pubkeyNode: String?, + @param:JsonProperty("server_state") val serverState: String?, + @param:JsonProperty("server_state_duration_us") val serverStateDurationUs: String?, + @param:JsonProperty("state_accounting") val stateAccounting: StateAccounting?, + @param:JsonProperty("time") val time: String, + @param:JsonProperty("uptime") val uptime: Long, + @param:JsonProperty("validated_ledger") val validatedLedger: ValidatedLedger, + @param:JsonProperty("validation_quorum") val validationQuorum: Int, ) @JsonIgnoreProperties(ignoreUnknown = true) data class LastClose( - @JsonProperty("converge_time") val convergeTime: Int, - @JsonProperty("proposers") val proposers: Int, + @param:JsonProperty("converge_time") val convergeTime: Int, + @param:JsonProperty("proposers") val proposers: Int, ) @JsonIgnoreProperties(ignoreUnknown = true) data class Port( - @JsonProperty("port") val port: String, - @JsonProperty("protocol") val protocol: List, + @param:JsonProperty("port") val port: String, + @param:JsonProperty("protocol") val protocol: List, ) @JsonIgnoreProperties(ignoreUnknown = true) data class StateAccounting( - @JsonProperty("connected") val connected: StateDuration, - @JsonProperty("disconnected") val disconnected: StateDuration, - @JsonProperty("full") val full: StateDuration, - @JsonProperty("syncing") val syncing: StateDuration, - @JsonProperty("tracking") val tracking: StateDuration, + @param:JsonProperty("connected") val connected: StateDuration, + @param:JsonProperty("disconnected") val disconnected: StateDuration, + @param:JsonProperty("full") val full: StateDuration, + @param:JsonProperty("syncing") val syncing: StateDuration, + @param:JsonProperty("tracking") val tracking: StateDuration, ) @JsonIgnoreProperties(ignoreUnknown = true) data class StateDuration( - @JsonProperty("duration_us") val durationUs: String, - @JsonProperty("transitions") val transitions: String, + @param:JsonProperty("duration_us") val durationUs: String, + @param:JsonProperty("transitions") val transitions: String, ) @JsonIgnoreProperties(ignoreUnknown = true) data class ValidatedLedger( - @JsonProperty("base_fee") val baseFee: Int, - @JsonProperty("close_time") val closeTime: Long, - @JsonProperty("hash") val hash: String, - @JsonProperty("reserve_base") val reserveBase: Long, - @JsonProperty("reserve_inc") val reserveInc: Long, - @JsonProperty("seq") val seq: Long, + @param:JsonProperty("base_fee") val baseFee: Int, + @param:JsonProperty("close_time") val closeTime: Long, + @param:JsonProperty("hash") val hash: String, + @param:JsonProperty("reserve_base") val reserveBase: Long, + @param:JsonProperty("reserve_inc") val reserveInc: Long, + @param:JsonProperty("seq") val seq: Long, +) + +/** + * Ripple ledger stream subscription event. + * Received when subscribed to the "ledger" stream via WebSocket. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +data class RippleLedgerStreamEvent( + @param:JsonProperty("type") val type: String, // "ledgerClosed" + @param:JsonProperty("ledger_hash") val ledgerHash: String, + @param:JsonProperty("ledger_index") val ledgerIndex: Long, + @param:JsonProperty("ledger_time") val ledgerTime: Long? = null, + @param:JsonProperty("txn_count") val txnCount: Int? = null, + @param:JsonProperty("validated_ledgers") val validatedLedgers: String? = null, + @param:JsonProperty("reserve_base") val reserveBase: Long? = null, + @param:JsonProperty("reserve_inc") val reserveInc: Long? = null, + @param:JsonProperty("fee_base") val feeBase: Int? = null, + @param:JsonProperty("fee_ref") val feeRef: Int? = null, ) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/CallParams.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/CallParams.kt index bd031edd7..e9764fc8f 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/CallParams.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/CallParams.kt @@ -39,6 +39,23 @@ data class ObjectParams(val obj: Map) : JsonRpcParams() { } } +/** + * Ripple native WebSocket command format. + * Unlike JSON-RPC, Ripple uses "command" field and flat structure. + */ +data class RippleCommandParams(val params: Map) : CallParams { + constructor(vararg pairs: Pair) : this(mapOf(*pairs)) + + override fun toJson(id: Int, method: String): ByteArray { + val json = mutableMapOf( + "id" to id, + "command" to method, + ) + json.putAll(params) + return Global.objectMapper.writeValueAsBytes(json) + } +} + data class RestParams( val headers: List>, val queryParams: List>, diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcGrpcClient.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcGrpcClient.kt index 4d0528ec0..641cf3dd0 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcGrpcClient.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcGrpcClient.kt @@ -114,7 +114,7 @@ class JsonRpcGrpcClient( } .doOnNext { if (timer.isStarted) { - metrics?.timer?.record(timer.getTime(TimeUnit.NANOSECONDS), TimeUnit.NANOSECONDS) + metrics?.timer()?.record(timer.getTime(TimeUnit.NANOSECONDS), TimeUnit.NANOSECONDS) } } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpReader.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpReader.kt index bdf09ad9d..e3e58b0c0 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpReader.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpReader.kt @@ -29,20 +29,24 @@ import io.emeraldpay.dshackle.upstream.stream.StreamResponse import io.netty.buffer.Unpooled import org.apache.commons.lang3.time.StopWatch import reactor.core.publisher.Mono +import reactor.core.scheduler.Scheduler import java.util.concurrent.TimeUnit import java.util.function.Function /** * JSON RPC client */ -class JsonRpcHttpReader( +class JsonRpcHttpReader @JvmOverloads constructor( target: String, maxConnections: Int, queueSize: Int, metrics: RequestMetrics, + private val httpScheduler: Scheduler, basicAuth: AuthConfig.ClientBasicAuth? = null, tlsCAAuth: ByteArray? = null, -) : HttpReader(target, maxConnections, queueSize, metrics, basicAuth, tlsCAAuth) { + customHeaders: Map = emptyMap(), + bearerAuth: AuthConfig.ClientBearerAuth? = null, +) : HttpReader(target, maxConnections, queueSize, metrics, basicAuth, tlsCAAuth, customHeaders, bearerAuth) { private val parser = ResponseRpcParser() private val streamParser = JsonRpcStreamParser() @@ -59,15 +63,20 @@ class JsonRpcHttpReader( response.response { header, bytes -> val statusCode = header.status().code() - bytes.aggregate().asByteArray().map { - AggregateResponse(it, statusCode) - } + bytes.aggregate().asByteArray() + .publishOn(httpScheduler) + .map { + AggregateResponse(it, statusCode) + } }.single() } else { response.responseConnection { t, u -> streamParser.streamParse( t.status().code(), - u.inbound().receive().asByteArray(), + u.inbound() + .receive() + .asByteArray() + .publishOn(httpScheduler), ) }.single() } @@ -84,7 +93,7 @@ class JsonRpcHttpReader( .flatMap(this@JsonRpcHttpReader::execute) .doOnNext { if (startTime.isStarted) { - metrics?.timer?.record(startTime.nanoTime, TimeUnit.NANOSECONDS) + metrics?.timer(key.method)?.record(startTime.nanoTime, TimeUnit.NANOSECONDS) } } .transform(asJsonRpcResponse(key)) @@ -99,8 +108,8 @@ class JsonRpcHttpReader( resp.map { when (it) { is AggregateResponse -> { - val parsed = parser.parse(it.response) val statusCode = it.code + val parsed = parser.parse(it.response) if (statusCode != 200) { if (parsed.hasError() && parsed.error!!.code != RpcResponseError.CODE_UPSTREAM_INVALID_RESPONSE) { // extracted the error details from the HTTP Body diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseParser.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseParser.kt index b51ab4380..de1891fae 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseParser.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseParser.kt @@ -32,7 +32,15 @@ abstract class ResponseParser { private val log = LoggerFactory.getLogger(ResponseParser::class.java) } - private val jsonFactory = JsonFactory() + // Some upstreams (e.g. Moca's Tendermint EVM) embed raw control characters + // such as LF inside JSON string values (notably in `web3_clientVersion` + // results). Default Jackson rejects those with "Illegal unquoted character" + // before the response ever reaches the upstream-settings detector, so we + // relax just this one rule at the response-parsing layer to keep the + // payload flowing. + private val jsonFactory = JsonFactory().apply { + enable(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS) + } abstract fun build(state: Preparsed): T @@ -97,7 +105,8 @@ abstract class ResponseParser { state.copy(result = result) } } else if (field == "error") { - val err = readError(parser) + val errorValue = Global.getErrorValueAsIs(json) + val err = readError(parser, errorValue) if (err != null) { return state.copy(error = err) } @@ -154,6 +163,10 @@ abstract class ResponseParser { } fun readError(parser: JsonParser): ChainCallError? { + return readError(parser, null) + } + + fun readError(parser: JsonParser, errorAsIs: ByteArray?): ChainCallError? { var code = 0 var message = "" var details: Any? = null @@ -204,7 +217,7 @@ abstract class ResponseParser { } } } - return ChainCallError(code, message, details) + return ChainCallError(code, message, details, errorAsIs) } data class Preparsed( diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseWSParser.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseWSParser.kt index b99c65c2a..3f5a7dcde 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseWSParser.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/ResponseWSParser.kt @@ -55,6 +55,18 @@ class ResponseWSParser : ResponseParser() { val method = parser.valueAsString return state.copy(subMethod = method) } + // Handle Ripple subscription format: { "type": "ledgerClosed", "ledger_hash": "...", ... } + if ("type" == field) { + parser.nextToken() + val type = parser.valueAsString + // "ledgerClosed" is a Ripple subscription notification + if (type == "ledgerClosed") { + // Use type as subscription identifier, entire JSON as result + return state.copy(subId = type, result = json) + } + // "response" is a normal RPC response, let it pass through + return state + } if ("params" == field) { // example: // newHeads diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/stream/JsonRpcStreamParser.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/stream/JsonRpcStreamParser.kt index dd9f262f1..25587a241 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/stream/JsonRpcStreamParser.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/rpcclient/stream/JsonRpcStreamParser.kt @@ -3,6 +3,7 @@ package io.emeraldpay.dshackle.upstream.rpcclient.stream import com.fasterxml.jackson.core.JsonFactory import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.core.JsonToken +import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.upstream.ChainCallError import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError import io.emeraldpay.dshackle.upstream.rpcclient.ResponseRpcParser @@ -233,7 +234,8 @@ class JsonRpcStreamParser( return response } } else if (parser.currentName == "error") { - return SingleResponse(response?.result, responseRpcParser.readError(parser)) + val errorValue = Global.getErrorValueAsIs(firstBytes) + return SingleResponse(response?.result, responseRpcParser.readError(parser, errorValue)) } } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/DisabledSigner.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/DisabledSigner.kt new file mode 100644 index 000000000..17cd084c4 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/DisabledSigner.kt @@ -0,0 +1,15 @@ +package io.emeraldpay.dshackle.upstream.signature + +import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException +import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError + +class DisabledSigner : ResponseSigner { + override val enabled: Boolean = false + + override fun sign(nonce: Long, message: ByteArray, source: String): ResponseSigner.Signature { + throw RpcException( + RpcResponseError.CODE_INTERNAL_ERROR, + "Response signing requested via nonce but signing key is not configured", + ) + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/EcdsaSigner.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/EcdsaSigner.kt deleted file mode 100644 index 902ff4e36..000000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/EcdsaSigner.kt +++ /dev/null @@ -1,57 +0,0 @@ -package io.emeraldpay.dshackle.upstream.signature - -import org.apache.commons.codec.binary.Hex -import java.security.MessageDigest -import java.security.Signature -import java.security.interfaces.ECPrivateKey - -class EcdsaSigner( - private val privateKey: ECPrivateKey, - val keyId: Long, -) : ResponseSigner { - - companion object { - const val SIGN_SCHEME = "SHA256withECDSA" - const val MSG_PREFIX = "DSHACKLESIG" - const val MSG_SEPARATOR = '/' - } - - override fun sign(nonce: Long, message: ByteArray, source: String): ResponseSigner.Signature { - val sig = Signature.getInstance(SIGN_SCHEME, "BC") - sig.initSign(privateKey) - val wrapped = wrapMessage(nonce, message, source) - sig.update(wrapped.toByteArray()) - val value = sig.sign() - return ResponseSigner.Signature(value, source, keyId) - } - - /** - * To avoid various attacks, such as various kinds of padding and message alternating attacks, - * we (1) tag the original message to specify the source, (2) ensure no parts of the message can affect each other - * and (3) ensure the message cannot break the wrapping. - * - * We're doing that by converting the message as `"DSHACKLESIG/" || str(nonce) || "/" || hex(sha256(msg))` - * - * I.e.: - * - three elements in the wrapped message - * - separated by "/" which is not a part of any element - * - first element is DSHACKLESIG tag - * - second is the nonce value encode as decimal string - * - third is SHA256 hash of the original message encoded as hex string - */ - fun wrapMessage(nonce: Long, message: ByteArray, source: String): String { - val sha256 = MessageDigest.getInstance("SHA-256") - // we create it with max capacity that we expect for the result, which is total lengths of its parts - val formatterMsg = StringBuilder(11 + 1 + 18 + 1 + 64 + 1 + 64) - formatterMsg.append(MSG_PREFIX) - .append(MSG_SEPARATOR) - .append(nonce.toString()) - .append(MSG_SEPARATOR) - // We expect that the id is short enough (less than 64 symbols) and also it doesn't contain the `/` symbol - // which is verified in UpstreamConfigReader and DefaultUpstream constructor - .append(source) - .append(MSG_SEPARATOR) - .append(Hex.encodeHexString(sha256.digest(message))) - return formatterMsg.toString() - } -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/NoSigner.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/NoSigner.kt deleted file mode 100644 index 5c963b0a6..000000000 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/NoSigner.kt +++ /dev/null @@ -1,7 +0,0 @@ -package io.emeraldpay.dshackle.upstream.signature - -class NoSigner : ResponseSigner { - override fun sign(nonce: Long, message: ByteArray, source: String): ResponseSigner.Signature? { - return null - } -} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/ResponseSigner.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/ResponseSigner.kt index 395499413..96cd5cbc4 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/ResponseSigner.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/ResponseSigner.kt @@ -2,7 +2,13 @@ package io.emeraldpay.dshackle.upstream.signature interface ResponseSigner { - fun sign(nonce: Long, message: ByteArray, source: String): Signature? + /** + * `true` when an actual signing key is configured and [sign] can produce signatures. + * `false` for placeholder signers that reject any signing attempt (e.g. when auth is disabled). + */ + val enabled: Boolean + + fun sign(nonce: Long, message: ByteArray, source: String): Signature data class Signature( val value: ByteArray, diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/ResponseSignerFactory.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/ResponseSignerFactory.kt index 22690b532..d7080150d 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/ResponseSignerFactory.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/ResponseSignerFactory.kt @@ -1,66 +1,77 @@ package io.emeraldpay.dshackle.upstream.signature -import io.emeraldpay.dshackle.config.SignatureConfig +import io.emeraldpay.dshackle.config.AuthorizationConfig import org.apache.commons.codec.binary.Hex -import org.bouncycastle.jce.ECNamedCurveTable -import org.bouncycastle.jce.spec.ECPublicKeySpec -import org.bouncycastle.math.ec.ECPoint -import org.bouncycastle.util.io.pem.PemObject -import org.bouncycastle.util.io.pem.PemReader +import org.bouncycastle.openssl.PEMParser import org.slf4j.LoggerFactory -import org.springframework.beans.factory.FactoryBean -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Repository +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.stereotype.Component +import java.io.StringReader import java.nio.ByteBuffer import java.nio.file.Files -import java.nio.file.Path +import java.nio.file.Paths import java.security.KeyFactory import java.security.MessageDigest import java.security.PublicKey -import java.security.interfaces.ECPrivateKey +import java.security.interfaces.RSAPrivateCrtKey +import java.security.interfaces.RSAPrivateKey import java.security.spec.PKCS8EncodedKeySpec +import java.security.spec.RSAPublicKeySpec -@Repository +@Configuration +open class SignatureBeans { + @Bean + open fun signer(factory: ResponseSignerFactory): ResponseSigner = + factory.createSigner() +} + +@Component open class ResponseSignerFactory( - @Autowired private val config: SignatureConfig, -) : FactoryBean { + private val authorizationConfig: AuthorizationConfig, +) { companion object { private val log = LoggerFactory.getLogger(ResponseSignerFactory::class.java) } - fun readKey(algorithm: SignatureConfig.Algorithm, keyPath: String): Pair { - val reader = PemReader(Files.newBufferedReader(Path.of(keyPath))) - return readKey(algorithm, reader.readPemObject()) + fun createSigner(): ResponseSigner { + if (!authorizationConfig.enabled) { + log.info("Response signing disabled: auth is not enabled") + return DisabledSigner() + } + val path = authorizationConfig.serverConfig.providerPrivateKeyPath + if (path.isBlank()) { + log.warn("Response signing disabled: auth.server.provider-private-key is not set") + return DisabledSigner() + } + + val (privateKey, keyId) = readRsaKey(path) + return RsaSigner(privateKey, keyId) } - private fun readKey(algorithm: SignatureConfig.Algorithm, pem: PemObject): Pair { - val keyFactory = KeyFactory.getInstance("EC") - val key = when (algorithm) { - SignatureConfig.Algorithm.NIST_P256 -> { - val keySpec = PKCS8EncodedKeySpec(pem.content) - keyFactory.generatePrivate(keySpec) - } - } + internal fun readRsaKey(path: String): Pair { + val pemContent = StringReader(Files.readString(Paths.get(path))) + val pemObject = PEMParser(pemContent).readPemObject() + ?: throw IllegalStateException("Cannot parse PEM key at $path") - if (key !is ECPrivateKey) { - throw IllegalStateException("Only EC keys are allowed") - } + val keyFactory = KeyFactory.getInstance("RSA") + val privateKey = keyFactory.generatePrivate(PKCS8EncodedKeySpec(pemObject.content)) - if (algorithm == SignatureConfig.Algorithm.NIST_P256 && key.params.toString().indexOf(SignatureConfig.Algorithm.NIST_P256.getCurveName()) < 0) { - throw IllegalStateException("Key is not NIST P256, generate NIST P256 or use another algorithm") + if (privateKey !is RSAPrivateKey) { + throw IllegalStateException("Only RSA keys are supported for response signing") } - val publicKey = extractPublicKey(keyFactory, key, algorithm) - val id = getPublicKeyId(publicKey) - - return Pair(key, id) + val publicKey = extractPublicKey(keyFactory, privateKey) + val keyId = getPublicKeyId(publicKey) + return Pair(privateKey, keyId) } - fun extractPublicKey(keyFactory: KeyFactory, privateKey: ECPrivateKey, algorithm: SignatureConfig.Algorithm): PublicKey { - val ecSpec = ECNamedCurveTable.getParameterSpec(algorithm.getCurveName()) - val q: ECPoint = ecSpec.g.multiply(privateKey.s) - return keyFactory.generatePublic(ECPublicKeySpec(q, ecSpec)) + private fun extractPublicKey(keyFactory: KeyFactory, privateKey: RSAPrivateKey): PublicKey { + val crt = privateKey as? RSAPrivateCrtKey + ?: throw IllegalStateException("RSA private key does not expose public exponent; use a PKCS#8 key that contains CRT parameters") + val spec = RSAPublicKeySpec(crt.modulus, crt.publicExponent) + return keyFactory.generatePublic(spec) } private fun getPublicKeyId(publicKey: PublicKey): Long { @@ -69,20 +80,4 @@ open class ResponseSignerFactory( log.info("Using key to sign responses: ${Hex.encodeHexString(fullId).substring(0..15)}") return ByteBuffer.wrap(fullId).asLongBuffer().get() } - - override fun getObject(): ResponseSigner { - if (!config.enabled) { - return NoSigner() - } - if (config.privateKey == null) { - log.warn("Private Key for response signature is not set") - return NoSigner() - } - val key = readKey(config.algorithm, config.privateKey!!) - return EcdsaSigner(key.first, key.second) - } - - override fun getObjectType(): Class<*>? { - return ResponseSigner::class.java - } } diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/RsaSigner.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/RsaSigner.kt new file mode 100644 index 000000000..f54396661 --- /dev/null +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/signature/RsaSigner.kt @@ -0,0 +1,50 @@ +package io.emeraldpay.dshackle.upstream.signature + +import org.apache.commons.codec.binary.Hex +import java.security.MessageDigest +import java.security.Signature +import java.security.interfaces.RSAPrivateKey + +class RsaSigner( + private val privateKey: RSAPrivateKey, + val keyId: Long, +) : ResponseSigner { + + companion object { + const val SIGN_SCHEME = "SHA256withRSA" + const val MSG_PREFIX = "DSHACKLESIG" + const val MSG_SEPARATOR = '/' + } + + override val enabled: Boolean = true + + override fun sign(nonce: Long, message: ByteArray, source: String): ResponseSigner.Signature { + val sig = Signature.getInstance(SIGN_SCHEME, "BC") + sig.initSign(privateKey) + val wrapped = wrapMessage(nonce, message, source) + sig.update(wrapped.toByteArray()) + val value = sig.sign() + return ResponseSigner.Signature(value, source, keyId) + } + + /** + * Wrapping format: `"DSHACKLESIG/" || str(nonce) || "/" || source || "/" || hex(sha256(msg))` + * + * `nonce` carries an unsigned uint64 bit pattern (proto3 uint64 ↔ Java long). + * Format it as unsigned decimal so verifiers reconstruct the same wrap for + * nonces ≥ 2^63 — otherwise `Long.toString` emits a negative number and + * signature verification fails. + */ + fun wrapMessage(nonce: Long, message: ByteArray, source: String): String { + val sha256 = MessageDigest.getInstance("SHA-256") + val formatterMsg = StringBuilder(11 + 1 + 20 + 1 + 64 + 1 + 64) + formatterMsg.append(MSG_PREFIX) + .append(MSG_SEPARATOR) + .append(java.lang.Long.toUnsignedString(nonce)) + .append(MSG_SEPARATOR) + .append(source) + .append(MSG_SEPARATOR) + .append(Hex.encodeHexString(sha256.digest(message))) + return formatterMsg.toString() + } +} diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaChainSpecific.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaChainSpecific.kt index 62e25582b..5f1ea2c23 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaChainSpecific.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaChainSpecific.kt @@ -30,91 +30,143 @@ import org.slf4j.LoggerFactory import reactor.core.publisher.Mono import reactor.core.scheduler.Scheduler import java.math.BigInteger +import java.nio.ByteBuffer import java.time.Instant - +import java.util.concurrent.ConcurrentHashMap + +/** + * Solana chain-specific implementation using slotSubscribe for head detection. + * + * Uses lightweight slotSubscribe WebSocket subscription instead of expensive blockSubscribe: + * - ~50 bytes per notification vs ~1KB for blockSubscribe + * - Stable API (no special node flags required) + * - Universal provider support + * - Throttled getEpochInfo calls (every N slots) to get actual slot and block height + * - Synthetic hash based on slot for ForkChoice deduplication + */ object SolanaChainSpecific : AbstractChainSpecific() { private val log = LoggerFactory.getLogger(SolanaChainSpecific::class.java) + // Throttle: check actual block height every N slots + private const val HEIGHT_CHECK_INTERVAL = 5 + + // Cache per upstream for throttling + private val lastKnownHeights = ConcurrentHashMap() + private val lastCheckedSlots = ConcurrentHashMap() + + private val getEpochInfoReq = ChainRequest("getEpochInfo", ListParams(mapOf("commitment" to "confirmed"))) + override fun getLatestBlock(api: ChainReader, upstreamId: String): Mono { - return api.read(ChainRequest("getSlot", ListParams())).flatMap { - val slot = it.getResultAsProcessedString().toLong() - api.read( - ChainRequest( - "getBlocks", - ListParams( - slot - 10, - slot, - ), - ), - ).flatMap { - val response = Global.objectMapper.readValue(it.getResult(), LongArray::class.java) - if (response == null || response.isEmpty()) { - Mono.empty() - } else { - api.read( - ChainRequest( - "getBlock", - ListParams( - response.max(), - mapOf( - "commitment" to "confirmed", - "showRewards" to false, - "transactionDetails" to "none", - "maxSupportedTransactionVersion" to 0, - ), - ), - ), - ).map { - val raw = it.getResult() - val block = Global.objectMapper.readValue(it.getResult(), SolanaBlock::class.java) - makeBlock(raw, block, upstreamId, response.max()) - }.onErrorResume { - log.debug("error during getting last solana block - ${it.message}") - Mono.empty() + return api.read(getEpochInfoReq) + .map { response -> + val epochInfo = Global.objectMapper.readValue( + response.getResult(), + SolanaEpochInfo::class.java, + ) + lastKnownHeights[upstreamId] = epochInfo.blockHeight + lastCheckedSlots[upstreamId] = epochInfo.absoluteSlot + makeBlockFromSlot(epochInfo.absoluteSlot, epochInfo.absoluteSlot - 1, epochInfo.blockHeight, upstreamId, ByteArray(0)) + } + .onErrorResume { error -> + log.debug("error during getting latest solana block - ${error.message}") + Mono.empty() + } + } + + override fun getFromHeader(data: ByteArray, upstreamId: String, api: ChainReader): Mono { + return try { + val notification = Global.objectMapper.readValue(data, SolanaSlotNotification::class.java) + val slot = notification.slot + + val lastChecked = lastCheckedSlots[upstreamId] ?: 0L + val lastHeight = lastKnownHeights[upstreamId] + val shouldCheckHeight = slot - lastChecked >= HEIGHT_CHECK_INTERVAL + + // Optimistic height estimation: assume 1:1 slot-to-block ratio + val estimatedHeight = if (lastHeight != null && lastChecked > 0) { + lastHeight + (slot - lastChecked) + } else { + null + } + + if (shouldCheckHeight || estimatedHeight == null) { + // Verify actual height using getEpochInfo (single call for both slot and height) + api.read(getEpochInfoReq) + .map { response -> + val epochInfo = Global.objectMapper.readValue( + response.getResult(), + SolanaEpochInfo::class.java, + ) + val actualSlot = epochInfo.absoluteSlot + val actualHeight = epochInfo.blockHeight + + if (estimatedHeight != null && estimatedHeight != actualHeight) { + log.debug( + "Height drift detected for {}: estimated={}, actual={}, diff={}", + upstreamId, + estimatedHeight, + actualHeight, + estimatedHeight - actualHeight, + ) + } + + lastKnownHeights[upstreamId] = actualHeight + lastCheckedSlots[upstreamId] = actualSlot + makeBlockFromSlot(actualSlot, actualSlot - 1, actualHeight, upstreamId, data) } - } + .onErrorResume { error -> + log.warn("Failed to get block height, using estimated value: ${error.message}") + val height = estimatedHeight ?: lastHeight ?: slot + Mono.just(makeBlockFromSlot(slot, notification.parent, height, upstreamId, data)) + } + } else { + // Use optimistic estimated height + Mono.just(makeBlockFromSlot(slot, notification.parent, estimatedHeight, upstreamId, data)) } + } catch (e: Exception) { + log.error("Failed to parse slotSubscribe notification", e) + Mono.empty() } } - override fun getFromHeader(data: ByteArray, upstreamId: String, api: ChainReader): Mono { - val res = Global.objectMapper.readValue(data, SolanaWrapper::class.java) - return Mono.just(makeBlock(data, res.value.block, upstreamId, res.context.slot)) + override fun listenNewHeadsRequest(): ChainRequest { + return ChainRequest("slotSubscribe", ListParams()) + } + + override fun unsubscribeNewHeadsRequest(subId: Any): ChainRequest { + return ChainRequest("slotUnsubscribe", ListParams(subId)) } - private fun makeBlock(raw: ByteArray, block: SolanaBlock, upstreamId: String, slot: Long): BlockContainer { + private fun makeBlockFromSlot(slot: Long, parentSlot: Long, height: Long, upstreamId: String, data: ByteArray): BlockContainer { + // Synthetic hash from slot for ForkChoice deduplication + val syntheticHash = BlockId.from( + ByteBuffer.allocate(32).putLong(slot).array(), + ) + // Synthetic parent hash from parent slot for chain tracking + val syntheticParentHash = BlockId.from( + ByteBuffer.allocate(32).putLong(parentSlot).array(), + ) + return BlockContainer( - height = block.height, - hash = BlockId.fromBase64(block.hash), + height = height, + hash = syntheticHash, difficulty = BigInteger.ZERO, - timestamp = Instant.ofEpochMilli(block.timestamp), + timestamp = Instant.now(), full = false, - json = raw, - parsed = block, + json = data, + parsed = null, transactions = emptyList(), upstreamId = upstreamId, - parentHash = BlockId.fromBase64(block.parent), + parentHash = syntheticParentHash, slot = slot, ) } - override fun listenNewHeadsRequest(): ChainRequest { - return ChainRequest( - "blockSubscribe", - ListParams( - "all", - mapOf( - "commitment" to "confirmed", - "showRewards" to false, - "transactionDetails" to "none", - ), - ), - ) - } - - override fun unsubscribeNewHeadsRequest(subId: Any): ChainRequest { - return ChainRequest("blockUnsubscribe", ListParams(subId)) + // For testing only - clear height cache + internal fun clearCache() { + lastKnownHeights.clear() + lastCheckedSlots.clear() } override fun upstreamValidators( @@ -165,26 +217,28 @@ object SolanaChainSpecific : AbstractChainSpecific() { } } +// slotSubscribe response format @JsonIgnoreProperties(ignoreUnknown = true) -data class SolanaWrapper( - @JsonProperty("context") var context: SolanaContext, - @JsonProperty("value") var value: SolanaResult, -) - -@JsonIgnoreProperties(ignoreUnknown = true) -data class SolanaContext( - @JsonProperty("slot") var slot: Long, +data class SolanaSlotNotification( + @param:JsonProperty("slot") val slot: Long, + @param:JsonProperty("parent") val parent: Long, + @param:JsonProperty("root") val root: Long, ) +// getEpochInfo response format @JsonIgnoreProperties(ignoreUnknown = true) -data class SolanaResult( - @JsonProperty("block") var block: SolanaBlock, +data class SolanaEpochInfo( + @param:JsonProperty("absoluteSlot") val absoluteSlot: Long, + @param:JsonProperty("blockHeight") val blockHeight: Long, + @param:JsonProperty("epoch") val epoch: Long, + @param:JsonProperty("slotIndex") val slotIndex: Long, + @param:JsonProperty("slotsInEpoch") val slotsInEpoch: Long, ) +// getBlock response format (used by SolanaLowerBoundSlotDetector) @JsonIgnoreProperties(ignoreUnknown = true) data class SolanaBlock( - @JsonProperty("blockHeight") var height: Long, - @JsonProperty("blockTime") var timestamp: Long, - @JsonProperty("blockhash") var hash: String, - @JsonProperty("previousBlockhash") var parent: String, + @param:JsonProperty("blockHeight") val height: Long, + @param:JsonProperty("blockhash") val hash: String, + @param:JsonProperty("blockTime") val time: Long, ) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaUpstreamSettingsDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaUpstreamSettingsDetector.kt index b0e1ee99e..b118a58fe 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaUpstreamSettingsDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaUpstreamSettingsDetector.kt @@ -32,7 +32,7 @@ class SolanaUpstreamSettingsDetector( @JsonIgnoreProperties(ignoreUnknown = true) private data class SolanaVersion( - @JsonProperty("solana-core") + @param:JsonProperty("solana-core") val version: String, ) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetChainSpecific.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetChainSpecific.kt index 5e508e9bb..94b23f3da 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetChainSpecific.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetChainSpecific.kt @@ -119,14 +119,14 @@ object StarknetChainSpecific : AbstractPollChainSpecific() { @JsonIgnoreProperties(ignoreUnknown = true) data class StarknetBlock( - @JsonProperty("block_hash") var hash: String, - @JsonProperty("block_number") var number: Long, - @JsonProperty("timestamp") var timestamp: Instant, - @JsonProperty("parent_hash") var parent: String, + @param:JsonProperty("block_hash") var hash: String, + @param:JsonProperty("block_number") var number: Long, + @param:JsonProperty("timestamp") var timestamp: Instant, + @param:JsonProperty("parent_hash") var parent: String, ) @JsonIgnoreProperties(ignoreUnknown = true) data class StarknetSyncing( - @JsonProperty("current_block_num") var current: Long, - @JsonProperty("highest_block_num") var highest: Long, + @param:JsonProperty("current_block_num") var current: Long, + @param:JsonProperty("highest_block_num") var highest: Long, ) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetUpstreamSettingsDetector.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetUpstreamSettingsDetector.kt index b77509125..2ba903034 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetUpstreamSettingsDetector.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetUpstreamSettingsDetector.kt @@ -18,7 +18,7 @@ class StarknetUpstreamSettingsDetector( ) } - private fun detectNodeType(): Flux?> { + private fun detectNodeType(): Flux> { return upstream .getIngressReader() .read(pathfinderVersionRequest()) diff --git a/src/main/kotlin/io/emeraldpay/dshackle/upstream/stream/Responses.kt b/src/main/kotlin/io/emeraldpay/dshackle/upstream/stream/Responses.kt index 1b2e7a4f0..621c3619a 100644 --- a/src/main/kotlin/io/emeraldpay/dshackle/upstream/stream/Responses.kt +++ b/src/main/kotlin/io/emeraldpay/dshackle/upstream/stream/Responses.kt @@ -35,11 +35,13 @@ data class SingleResponse( data class StreamResponse( val stream: Flux, + val headers: Map = emptyMap(), ) : Response() data class AggregateResponse( val response: ByteArray, val code: Int, + val headers: Map = emptyMap(), ) : Response() { override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index db5bfba84..336256b85 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,26 +1,9 @@ + + spring: application: max-metadata-size: ${MAX_METADATA_SIZE:16384} name: ${DSHACKLE_APP_NAME:dshackle} - zipkin: - enabled: ${ENABLE_COLLECT_SPANS:false} - base-url: ${ZIPKIN_URL:https://trace.drpc.dev/} - sleuth: - span-filter: - additional-span-name-patterns-to-ignore: - - "^grpcChannelExecutor$" - - ".+NativeSubscribe" - - ".+SubscribeHead" - - ".+SubscribeBalance" - - ".+SubscribeTxStatus" - - ".+SubscribeStatus" - - ".+SubscribeNodeStatus" - - ".+Describe" - - ".+ServerReflectionInfo" - -spans: - collect: - long-span-threshold: ${LONG_SPAN_THRESHOLD:1000} compatibility: enabled: ${DSHACKLE_COMPATIBILITY_ENABLED:true} diff --git a/src/main/resources/logback-spring.xml b/src/main/resources/logback-spring.xml index 7bd936616..af4bb0cdb 100644 --- a/src/main/resources/logback-spring.xml +++ b/src/main/resources/logback-spring.xml @@ -4,7 +4,7 @@ + value="%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) [%X{traceId},%X{spanId}] %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}"/> diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/AuthConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/AuthConfigReaderSpec.groovy index c31a95a90..51593497f 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/config/AuthConfigReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/config/AuthConfigReaderSpec.groovy @@ -35,6 +35,29 @@ class AuthConfigReaderSpec extends Specification { act.password == "258fe4149c199ad8f2811a68f20154fc" } + def "Read bearer-auth for client"() { + setup: + def yaml = + "bearer-auth:\n" + + " token: 9c199ad8f281f20154fc258fe41a6814" + when: + def act = reader.readClientBearerAuth(reader.readNode(yaml)) + then: + act != null + act.token == "9c199ad8f281f20154fc258fe41a6814" + } + + def "Read bearer-auth without token as null"() { + setup: + def yaml = + "bearer-auth:\n" + + " something: else" + when: + def act = reader.readClientBearerAuth(reader.readNode(yaml)) + then: + act == null + } + def "Read tls for client"() { setup: def yaml = diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/SignatureConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/SignatureConfigReaderSpec.groovy deleted file mode 100644 index 2beb419f1..000000000 --- a/src/test/groovy/io/emeraldpay/dshackle/config/SignatureConfigReaderSpec.groovy +++ /dev/null @@ -1,40 +0,0 @@ -package io.emeraldpay.dshackle.config - -import io.emeraldpay.dshackle.test.TestingCommons -import spock.lang.Specification - -class SignatureConfigReaderSpec extends Specification { - - def "Parse enabled"() { - setup: - def config = "signed-response:\n" + - " enabled: true\n" + - " algorithm: NIST_P256\n" + - " private-key: /root/key.pem\n" - - when: - def reader = new SignatureConfigReader(TestingCommons.fileResolver()) - def act = reader.read(new ByteArrayInputStream(config.bytes)) - - then: - act.enabled - act.privateKey == "/root/key.pem" - act.algorithm == SignatureConfig.Algorithm.NIST_P256 - } - - def "No path when disabled"() { - setup: - def config = "signed-response:\n" + - " enabled: false\n" + - " private-key: /root/key.pem\n" - - when: - def reader = new SignatureConfigReader(TestingCommons.fileResolver()) - def act = reader.read(new ByteArrayInputStream(config.bytes)) - - then: - !act.enabled - act.privateKey == null - } - -} diff --git a/src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy index 2f8b83ed0..d0824fffb 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/config/UpstreamsConfigReaderSpec.groovy @@ -75,6 +75,38 @@ class UpstreamsConfigReaderSpec extends Specification { } } + def "Parse config with bearer auth"() { + setup: + def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-bearer-auth.yaml") + when: + def act = reader.readInternal(config) + then: + act != null + act.upstreams.size() == 2 + with(act.upstreams.get(0)) { + id == "local" + connection instanceof UpstreamsConfig.RpcConnection + with((UpstreamsConfig.RpcConnection) connection) { + rpc.basicAuth == null + rpc.bearerAuth != null + rpc.bearerAuth.token == "9c199ad8f281f20154fc258fe41a6814" + ws != null + ws.basicAuth == null + ws.bearerAuth != null + ws.bearerAuth.token == "258fe4149c199ad8f2811a68f20154fc" + } + } + with(act.upstreams.get(1)) { + id == "both-auth" + connection instanceof UpstreamsConfig.RpcConnection + with((UpstreamsConfig.RpcConnection) connection) { + // when both are configured basic-auth wins and bearer-auth is dropped + rpc.basicAuth != null + rpc.bearerAuth == null + } + } + } + def "Parse websocket-only config"() { setup: def config = this.class.getClassLoader().getResourceAsStream("configs/upstreams-ws-only.yaml") @@ -636,7 +668,7 @@ class UpstreamsConfigReaderSpec extends Specification { def options = partialOptions.buildOptions() then: options == new ChainOptions.Options( - false, false, 30, Duration.ofSeconds(60), null, true, 1, true, true, true, true, 1_000_000, false, false, true, false + false, false, 30, Duration.ofSeconds(60), null, true, 1, true, true, true, true, 1_000_000, false, false, true, false, false ) } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/quorum/QuorumRequestReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/quorum/QuorumRequestReaderSpec.groovy index cd2449719..4c334363a 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/quorum/QuorumRequestReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/quorum/QuorumRequestReaderSpec.groovy @@ -29,7 +29,6 @@ import io.emeraldpay.dshackle.upstream.UpstreamAvailability import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError -import org.springframework.cloud.sleuth.Tracer import reactor.core.publisher.Mono import reactor.test.StepVerifier import spock.lang.Specification @@ -55,7 +54,7 @@ class QuorumRequestReaderSpec extends Specification { Chain.ETHEREUM__MAINNET, [up], Selector.empty ) - def reader = new QuorumRequestReader(apis, new AlwaysQuorum(), Stub(Tracer)) + def reader = new QuorumRequestReader(apis, new AlwaysQuorum()) when: def act = reader.read(new ChainRequest("eth_test", new ListParams())) @@ -87,7 +86,7 @@ class QuorumRequestReaderSpec extends Specification { Chain.ETHEREUM__MAINNET, [up], Selector.empty ) - def reader = new QuorumRequestReader(apis, new AlwaysQuorum(), Stub(Tracer)) + def reader = new QuorumRequestReader(apis, new AlwaysQuorum()) when: def act = reader.read(new ChainRequest("eth_test", new ListParams())) @@ -125,7 +124,7 @@ class QuorumRequestReaderSpec extends Specification { Chain.ETHEREUM__MAINNET, [up], Selector.empty ) - def reader = new QuorumRequestReader(apis, new AlwaysQuorum(), Stub(Tracer)) + def reader = new QuorumRequestReader(apis, new AlwaysQuorum()) when: def act = reader.read(new ChainRequest("eth_test", new ListParams())) @@ -158,7 +157,7 @@ class QuorumRequestReaderSpec extends Specification { Chain.ETHEREUM__MAINNET, [up], Selector.empty ) - def reader = new QuorumRequestReader(apis, new NotNullQuorum(), Stub(Tracer)) + def reader = new QuorumRequestReader(apis, new NotNullQuorum()) when: def act = reader.read(new ChainRequest("eth_test", new ListParams())) @@ -191,7 +190,7 @@ class QuorumRequestReaderSpec extends Specification { Chain.ETHEREUM__MAINNET, [up], Selector.empty ) - def reader = new QuorumRequestReader(apis, new NotNullQuorum(), Stub(Tracer)) + def reader = new QuorumRequestReader(apis, new NotNullQuorum()) when: def act = reader.read(new ChainRequest("eth_test", new ListParams())) @@ -224,7 +223,7 @@ class QuorumRequestReaderSpec extends Specification { Chain.ETHEREUM__MAINNET, [up], Selector.empty ) - def reader = new QuorumRequestReader(apis, new NotNullQuorum(), Stub(Tracer)) + def reader = new QuorumRequestReader(apis, new NotNullQuorum()) when: def act = reader.read(new ChainRequest("eth_test", new ListParams())) @@ -255,7 +254,7 @@ class QuorumRequestReaderSpec extends Specification { Chain.ETHEREUM__MAINNET, [up], Selector.empty ) - def reader = new QuorumRequestReader(apis, new AlwaysQuorum(), Stub(Tracer)) + def reader = new QuorumRequestReader(apis, new AlwaysQuorum()) when: def act = reader.read(new ChainRequest("eth_test", new ListParams())) @@ -289,7 +288,7 @@ class QuorumRequestReaderSpec extends Specification { Chain.ETHEREUM__MAINNET, [up], Selector.empty ) - def reader = new QuorumRequestReader(apis, new NotLaggingQuorum(1), Stub(Tracer)) + def reader = new QuorumRequestReader(apis, new NotLaggingQuorum(1)) when: def act = reader.read(new ChainRequest("eth_test", new ListParams())) @@ -315,7 +314,7 @@ class QuorumRequestReaderSpec extends Specification { Chain.ETHEREUM__MAINNET, [up], Selector.empty ) - def reader = new QuorumRequestReader(apis, new AlwaysQuorum(), Stub(Tracer)) + def reader = new QuorumRequestReader(apis, new AlwaysQuorum()) when: def act = reader.read(new ChainRequest("eth_test", new ListParams())) @@ -365,7 +364,7 @@ class QuorumRequestReaderSpec extends Specification { new Selector.HeightMatcher(100000000), ) )) - def reader = new QuorumRequestReader(apis, new AlwaysQuorum(), Stub(Tracer)) + def reader = new QuorumRequestReader(apis, new AlwaysQuorum()) when: def act = reader.read(new ChainRequest("eth_test", new ListParams())) diff --git a/src/test/groovy/io/emeraldpay/dshackle/reader/BroadcastReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/reader/BroadcastReaderSpec.groovy index 37caa0e2c..7695c5516 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/reader/BroadcastReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/reader/BroadcastReaderSpec.groovy @@ -7,7 +7,7 @@ import io.emeraldpay.dshackle.upstream.ChainException import io.emeraldpay.dshackle.upstream.ChainRequest import io.emeraldpay.dshackle.upstream.ChainResponse import io.emeraldpay.dshackle.upstream.rpcclient.ListParams -import org.springframework.cloud.sleuth.Tracer +import io.emeraldpay.dshackle.upstream.signature.DisabledSigner import reactor.core.publisher.Mono import reactor.test.StepVerifier import spock.lang.Specification @@ -43,7 +43,7 @@ class BroadcastReaderSpec extends Specification { Mono.just(new ChainResponse(result, null)) } } - def reader = new BroadcastReader([up, up1, up2], new Selector.EmptyMatcher(), null, new BroadcastQuorum(), Stub(Tracer)) + def reader = new BroadcastReader([up, up1, up2], new Selector.EmptyMatcher(), new DisabledSigner(), new BroadcastQuorum()) when: def act = reader.read(new ChainRequest("eth_sendRawTransaction", new ListParams(["0x1"]))) then: @@ -81,7 +81,7 @@ class BroadcastReaderSpec extends Specification { 1 * read(new ChainRequest("eth_sendRawTransaction", new ListParams(["0x1"]))) >> Mono.error(new ChainException(1, "too low")) } } - def reader = new BroadcastReader([up, up1, up2], new Selector.EmptyMatcher(), null, new BroadcastQuorum(), Stub(Tracer)) + def reader = new BroadcastReader([up, up1, up2], new Selector.EmptyMatcher(), new DisabledSigner(), new BroadcastQuorum()) when: def act = reader.read(new ChainRequest("eth_sendRawTransaction", new ListParams(["0x1"]))) then: @@ -114,7 +114,7 @@ class BroadcastReaderSpec extends Specification { 0 * getId() >> "id" 0 * getIngressReader() >> Mock(Reader) } - def reader = new BroadcastReader([up, up1, up2], new Selector.EmptyMatcher(), null, new BroadcastQuorum(), Stub(Tracer)) + def reader = new BroadcastReader([up, up1, up2], new Selector.EmptyMatcher(), new DisabledSigner(), new BroadcastQuorum()) when: def act = reader.read(new ChainRequest("eth_sendRawTransaction", new ListParams(["0x1"]))) then: @@ -152,7 +152,7 @@ class BroadcastReaderSpec extends Specification { Mono.error(new ChainException(1, "too low")) } } - def reader = new BroadcastReader([up, up1, up2], new Selector.EmptyMatcher(), null, new BroadcastQuorum(), Stub(Tracer)) + def reader = new BroadcastReader([up, up1, up2], new Selector.EmptyMatcher(), new DisabledSigner(), new BroadcastQuorum()) when: def act = reader.read(new ChainRequest("eth_sendRawTransaction", new ListParams(["0x1"]))) then: @@ -178,7 +178,7 @@ class BroadcastReaderSpec extends Specification { 0 * getId() >> "id" 0 * getIngressReader() >> Mock(Reader) } - def reader = new BroadcastReader([up, up1, up2], new Selector.EmptyMatcher(), null, new BroadcastQuorum(), Stub(Tracer)) + def reader = new BroadcastReader([up, up1, up2], new Selector.EmptyMatcher(), new DisabledSigner(), new BroadcastQuorum()) when: def act = reader .read(new ChainRequest("eth_sendRawTransaction", new ListParams(["0x1"]))) diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeCallSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeCallSpec.groovy index ff32787b9..8815e2e35 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeCallSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeCallSpec.groovy @@ -39,7 +39,6 @@ import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.signature.ResponseSigner import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcResponseError -import org.springframework.cloud.sleuth.Tracer import reactor.core.publisher.Flux import reactor.core.publisher.Mono import reactor.test.StepVerifier @@ -67,7 +66,7 @@ class NativeCallSpec extends Specification { config.cache = cacheConfig config.passthrough = passthrough - new NativeCall(upstreams, signer, config, Stub(Tracer)) + new NativeCall(upstreams, signer, config) } def "Tries router first"() { diff --git a/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeSubscribeSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeSubscribeSpec.groovy index 2b69bed3e..70c18801e 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeSubscribeSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/rpc/NativeSubscribeSpec.groovy @@ -22,7 +22,7 @@ import io.emeraldpay.dshackle.test.MultistreamHolderMock import io.emeraldpay.dshackle.upstream.Selector import io.emeraldpay.dshackle.upstream.ethereum.EthereumEgressSubscription import io.emeraldpay.dshackle.upstream.generic.GenericMultistream -import io.emeraldpay.dshackle.upstream.signature.NoSigner +import io.emeraldpay.dshackle.upstream.signature.DisabledSigner import reactor.core.publisher.Flux import reactor.test.StepVerifier import spock.lang.Specification @@ -30,7 +30,7 @@ import spock.lang.Specification import java.time.Duration class NativeSubscribeSpec extends Specification { - def signer = new NoSigner() + def signer = new DisabledSigner() def "Call with empty params when not provided"() { setup: @@ -45,10 +45,6 @@ class NativeSubscribeSpec extends Specification { } def up = Mock(GenericMultistream) { 1 * it.start() - _ * it.getSubscriptionTopics() >> { - println("getSubscriptionTopics called") - return ["newHeads"] - } _ * it.tryProxySubscribe(_, _) >> { println("tryProxySubscribe called") return null diff --git a/src/test/groovy/io/emeraldpay/dshackle/startup/configure/UpstreamCreatorLabelsSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/startup/configure/UpstreamCreatorLabelsSpec.groovy new file mode 100644 index 000000000..29a4060aa --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/startup/configure/UpstreamCreatorLabelsSpec.groovy @@ -0,0 +1,74 @@ +package io.emeraldpay.dshackle.startup.configure + +import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.config.ChainsConfig +import io.emeraldpay.dshackle.config.UpstreamsConfig +import io.emeraldpay.dshackle.foundation.ChainOptions +import io.emeraldpay.dshackle.upstream.CallTargetsHolder +import io.emeraldpay.dshackle.upstream.signature.DisabledSigner +import io.emeraldpay.dshackle.upstream.signature.ResponseSigner +import io.emeraldpay.dshackle.upstream.signature.RsaSigner +import spock.lang.Specification + +import java.security.interfaces.RSAPrivateKey + +class UpstreamCreatorLabelsSpec extends Specification { + + static class TestCreator extends UpstreamCreator { + TestCreator(ChainsConfig chainsConfig, CallTargetsHolder callTargets, ResponseSigner signer) { + super(chainsConfig, callTargets, signer) + } + + @Override + protected UpstreamCreationData createUpstream( + UpstreamsConfig.Upstream upstreamsConfig, + Chain chain, + ChainOptions.Options options, + ChainsConfig.ChainConfig chainConf) { + return UpstreamCreationData.default() + } + + UpstreamsConfig.Labels callBuildLabels(Map src) { + return buildUpstreamLabels(src) + } + } + + TestCreator makeCreator(ResponseSigner signer) { + return new TestCreator(Mock(ChainsConfig), Mock(CallTargetsHolder), signer) + } + + def "Adds secure-signed label when signer is enabled"() { + setup: + def creator = makeCreator(new RsaSigner(Stub(RSAPrivateKey), 1L)) + + when: + def labels = creator.callBuildLabels(["provider": "drpc"]) + + then: + labels["provider"] == "drpc" + labels["secure-signed"] == "true" + } + + def "Does not add secure-signed label when signer is disabled"() { + setup: + def creator = makeCreator(new DisabledSigner()) + + when: + def labels = creator.callBuildLabels(["provider": "drpc"]) + + then: + labels["provider"] == "drpc" + !labels.containsKey("secure-signed") + } + + def "Does not override user-provided secure-signed label"() { + setup: + def creator = makeCreator(new RsaSigner(Stub(RSAPrivateKey), 1L)) + + when: + def labels = creator.callBuildLabels(["secure-signed": "false"]) + + then: + labels["secure-signed"] == "false" + } +} diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/ApiReaderMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/ApiReaderMock.groovy index 89b0fec72..4a4067e6d 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/ApiReaderMock.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/ApiReaderMock.groovy @@ -227,7 +227,15 @@ class ApiReaderMock implements Reader { @Override ByteBufFlux receive() { - throw new UnsupportedOperationException() + return ByteBufFlux + .fromString( + Flux.merge( + jsonResponses, + responses.map { + Global.objectMapper.writeValueAsString(it) + } + ) + ) } @Override diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/GenericConnectorMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/GenericConnectorMock.groovy index f66ddf659..ea92c16a8 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/GenericConnectorMock.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/GenericConnectorMock.groovy @@ -14,11 +14,13 @@ class GenericConnectorMock implements GenericConnector { Reader api Head head Flux liveness + Flux pendingTxs GenericConnectorMock(Reader api, Head head) { this.api = api this.head = head this.liveness = Flux.just(HeadLivenessState.NON_CONSECUTIVE) + this.pendingTxs = Flux.just(false) } @Override @@ -51,4 +53,9 @@ class GenericConnectorMock implements GenericConnector { IngressSubscription getIngressSubscription() { return NoEthereumIngressSubscription.DEFAULT } + + @Override + Flux pendingTxEvents() { + return pendingTxs + } } \ No newline at end of file diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/GenericUpstreamMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/GenericUpstreamMock.groovy index b22bff24f..7fd0eb6bd 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/GenericUpstreamMock.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/GenericUpstreamMock.groovy @@ -79,6 +79,7 @@ class GenericUpstreamMock extends GenericUpstream { io.emeraldpay.dshackle.upstream.starknet.StarknetChainSpecific.INSTANCE.&lowerBoundService, io.emeraldpay.dshackle.upstream.starknet.StarknetChainSpecific.INSTANCE.&finalizationDetectorBuilder, [get: { null }] as java.util.function.Supplier, + null, ) this.ethereumHeadMock = this.getHead() as EthereumHeadMock setLag(0) diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/MultistreamHolderMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/MultistreamHolderMock.groovy index ee98b05e9..c2ab614f8 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/MultistreamHolderMock.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/MultistreamHolderMock.groovy @@ -28,7 +28,6 @@ import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods import io.emeraldpay.dshackle.upstream.ethereum.EthereumCachingReader import io.emeraldpay.dshackle.upstream.ethereum.EthereumChainSpecific import org.jetbrains.annotations.NotNull -import org.springframework.cloud.sleuth.brave.bridge.BraveTracer import reactor.core.scheduler.Schedulers class MultistreamHolderMock implements MultistreamHolder { @@ -49,7 +48,7 @@ class MultistreamHolderMock implements MultistreamHolder { upstreams[chain] = new GenericMultistream( chain, Schedulers.immediate(), null, new ArrayList(), Caches.default(), Schedulers.boundedElastic(), - EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(TestingCommons.tracerMock()), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(), EthereumChainSpecific.INSTANCE.&localReaderBuilder, io.emeraldpay.dshackle.upstream.starknet.StarknetChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic()), ) @@ -102,7 +101,7 @@ class MultistreamHolderMock implements MultistreamHolder { EthereumMultistreamMock(@NotNull Chain chain, @NotNull List upstreams, @NotNull Caches caches) { super(chain, Schedulers.immediate(), null, upstreams, caches, Schedulers.boundedElastic(), - EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(new BraveTracer(null, null, null)), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(), EthereumChainSpecific.INSTANCE.&localReaderBuilder, EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic()), ) diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy index b0af252bd..6c2b9b524 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/test/TestingCommons.groovy @@ -52,10 +52,6 @@ class TestingCommons { return new ApiReaderMock() } - static TracerMock tracerMock() { - return new TracerMock(null, null, null) - } - static GenericUpstreamMock upstream() { return new GenericUpstreamMock(Chain.ETHEREUM__MAINNET, api()) } @@ -99,7 +95,7 @@ class TestingCommons { static Multistream multistream(GenericUpstreamMock up) { return new GenericMultistream(Chain.ETHEREUM__MAINNET, Schedulers.immediate(), null, new ArrayList(), Caches.default(), Schedulers.boundedElastic(), - EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(tracerMock()), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(), EthereumChainSpecific.INSTANCE.&localReaderBuilder, EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic()), ).tap { @@ -126,7 +122,7 @@ class TestingCommons { static Multistream multistreamWithoutUpstreams(Chain chain) { return new GenericMultistream(chain, Schedulers.immediate(), null, [], emptyCaches().getCaches(chain), Schedulers.boundedElastic(), - EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(tracerMock()), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(), EthereumChainSpecific.INSTANCE.&localReaderBuilder, EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic()), ) @@ -134,7 +130,7 @@ class TestingCommons { static Multistream multistreamClassicWithoutUpstreams(Chain chain) { return new GenericMultistream(chain, Schedulers.immediate(), null, [], emptyCaches().getCaches(chain), Schedulers.boundedElastic(), - EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(tracerMock()), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(), EthereumChainSpecific.INSTANCE.&localReaderBuilder, EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic()), ) diff --git a/src/test/groovy/io/emeraldpay/dshackle/test/TracerMock.groovy b/src/test/groovy/io/emeraldpay/dshackle/test/TracerMock.groovy deleted file mode 100644 index 7d0efbbe9..000000000 --- a/src/test/groovy/io/emeraldpay/dshackle/test/TracerMock.groovy +++ /dev/null @@ -1,64 +0,0 @@ -package io.emeraldpay.dshackle.test - -import brave.Tracer -import org.springframework.cloud.sleuth.CurrentTraceContext -import org.springframework.cloud.sleuth.Span -import org.springframework.cloud.sleuth.TraceContext -import org.springframework.cloud.sleuth.brave.bridge.BraveBaggageManager -import org.springframework.cloud.sleuth.brave.bridge.BraveSpan -import org.springframework.cloud.sleuth.brave.bridge.BraveTracer -import spock.mock.DetachedMockFactory - -class TracerMock extends BraveTracer { - private final spanMock = new SpanMock(null) - private static final mockFactory = new DetachedMockFactory() - - TracerMock(Tracer tracer, CurrentTraceContext context, BraveBaggageManager braveBaggageManager) { - super(tracer, context, braveBaggageManager) - } - - @Override - Span nextSpan(Span parent) { - return spanMock - } - - @Override - Span currentSpan() { - return spanMock - } - - @Override - SpanInScope withSpan(Span span) { - return mockFactory.Stub(SpanInScope) - } - - private static class SpanMock extends BraveSpan { - SpanMock(brave.Span delegate) { - super(delegate) - } - - @Override - Span start() { - return this - } - - @Override - Span name(String name) { - return this - } - - @Override - Span tag(String key, String value) { - return this - } - - @Override - void end() { - } - - @Override - TraceContext context() { - return mockFactory.Stub(TraceContext) - } - } -} diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy index 417a3227d..0a9bdda7d 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/FilteredApisSpec.groovy @@ -80,6 +80,7 @@ class FilteredApisSpec extends Specification { cs.&lowerBoundService, cs.&finalizationDetectorBuilder, [get: { null }] as java.util.function.Supplier, + null, ) } def matcher = new Selector.LabelMatcher("test", ["foo"]) diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy index e17746d23..58df1dd21 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/MultistreamSpec.groovy @@ -55,7 +55,7 @@ class MultistreamSpec extends Specification { def up2 = new GenericUpstreamMock("test1", Chain.ETHEREUM__MAINNET, TestingCommons.api(), new DirectCallMethods(["eth_test2", "eth_test3"])) def aggr = new GenericMultistream(Chain.ETHEREUM__MAINNET, Schedulers.immediate(), null, [up1, up2], Caches.default(), Schedulers.boundedElastic(), - EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(TestingCommons.tracerMock()), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(), EthereumChainSpecific.INSTANCE.&localReaderBuilder, EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic())) when: @@ -190,7 +190,7 @@ class MultistreamSpec extends Specification { def up3 = TestingCommons.upstream("test-3", "external") def multistream = new GenericMultistream(Chain.ETHEREUM__MAINNET, Schedulers.immediate(), null, [up1, up2, up3], Caches.default(), Schedulers.boundedElastic(), - EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(TestingCommons.tracerMock()), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(), EthereumChainSpecific.INSTANCE.&localReaderBuilder, EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic())) @@ -263,7 +263,7 @@ class MultistreamSpec extends Specification { def up2 = new GenericUpstreamMock("test2", Chain.ETHEREUM__MAINNET, TestingCommons.api(), new DirectCallMethods(["eth_test1", "eth_test2"])) def ms = new GenericMultistream(Chain.ETHEREUM__MAINNET, Schedulers.immediate(), null, new ArrayList(), Caches.default(), Schedulers.boundedElastic(), - EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(TestingCommons.tracerMock()), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(), EthereumChainSpecific.INSTANCE.&localReaderBuilder, EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic())) when: @@ -294,7 +294,7 @@ class MultistreamSpec extends Specification { def up2 = new GenericUpstreamMock("test2", Chain.ETHEREUM__MAINNET, TestingCommons.api(), new DirectCallMethods(["eth_test1", "eth_test2"])) def ms = new GenericMultistream(Chain.ETHEREUM__MAINNET, Schedulers.immediate(), null, new ArrayList(), Caches.default(), Schedulers.boundedElastic(), - EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(TestingCommons.tracerMock()), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(), EthereumChainSpecific.INSTANCE.&localReaderBuilder, EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic())) def head1 = createBlock(250, "0x0d050c785de17179f935b9b93aca09c442964cc59972c71ae68e74731448401b") @@ -329,7 +329,7 @@ class MultistreamSpec extends Specification { def up3 = TestingCommons.upstream("test-3", "external") def multistream = new GenericMultistream(Chain.ETHEREUM__MAINNET, Schedulers.immediate(), null, new ArrayList(), Caches.default(), Schedulers.boundedElastic(), - EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(TestingCommons.tracerMock()), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(), EthereumChainSpecific.INSTANCE.&localReaderBuilder, EthereumChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic())) multistream.processUpstreamsEvents( @@ -367,7 +367,7 @@ class MultistreamSpec extends Specification { TestEthereumPosMultistream(@NotNull Chain chain, @NotNull List upstreams, @NotNull Caches caches) { super(chain, Schedulers.immediate(), null, upstreams, caches, Schedulers.boundedElastic(), - EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(TestingCommons.tracerMock()), + EthereumChainSpecific.INSTANCE.makeCachingReaderBuilder(), EthereumChainSpecific.INSTANCE.&localReaderBuilder, StarknetChainSpecific.INSTANCE.subscriptionBuilder(Schedulers.boundedElastic())) } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClientSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClientSpec.groovy index ea686f168..5975f71b5 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClientSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/EsploraClientSpec.groovy @@ -1,6 +1,7 @@ package io.emeraldpay.dshackle.upstream.bitcoin import io.emeraldpay.dshackle.upstream.bitcoin.data.EsploraUnspent +import org.bitcoinj.base.AddressParser /** * Copyright (c) 2020 EmeraldPay, Inc @@ -18,13 +19,9 @@ import io.emeraldpay.dshackle.upstream.bitcoin.data.EsploraUnspent * limitations under the License. */ -import org.bitcoinj.core.Address -import org.bitcoinj.params.MainNetParams -import org.bitcoinj.params.TestNet3Params import org.mockserver.integration.ClientAndServer import org.mockserver.model.HttpRequest import org.mockserver.model.HttpResponse -import org.springframework.util.SocketUtils import reactor.test.StepVerifier import spock.lang.Specification @@ -33,11 +30,9 @@ import java.time.Duration class EsploraClientSpec extends Specification { ClientAndServer mockServer - int port = 23001 def setup() { - port = SocketUtils.findAvailableTcpPort(23001) - mockServer = ClientAndServer.startClientAndServer(port); + mockServer = ClientAndServer.startClientAndServer(0) } def cleanup() { @@ -54,9 +49,9 @@ class EsploraClientSpec extends Specification { ).respond( HttpResponse.response(responseJson) ) - def client = new EsploraClient(new URI("http://localhost:${port}"), null, null) + def client = new EsploraClient(new URI("http://localhost:${mockServer.port}"), null, null) when: - def act = client.getUtxo(Address.fromString(new MainNetParams(), "35vktkPo4wdK8Twu4VMiuPLdCx23XEykGY")) + def act = client.getUtxo(AddressParser.getDefault().parseAddress("35vktkPo4wdK8Twu4VMiuPLdCx23XEykGY")) then: StepVerifier.create(act) @@ -77,7 +72,7 @@ class EsploraClientSpec extends Specification { .verify(Duration.ofSeconds(3)) when: - def actTotal = client.getUtxo(Address.fromString(new MainNetParams(), "35vktkPo4wdK8Twu4VMiuPLdCx23XEykGY")) + def actTotal = client.getUtxo(AddressParser.getDefault().parseAddress("35vktkPo4wdK8Twu4VMiuPLdCx23XEykGY")) .block() .sum { it.value } @@ -95,9 +90,9 @@ class EsploraClientSpec extends Specification { ).respond( HttpResponse.response(responseJson) ) - def client = new EsploraClient(new URI("http://localhost:${port}"), null, null) + def client = new EsploraClient(new URI("http://localhost:${mockServer.port}"), null, null) when: - def act = client.getTransactions(Address.fromString(TestNet3Params.get(), "tb1qyatuwvkfx8thy2ntmtuea6v42vp3zefqvll8kx")) + def act = client.getTransactions(AddressParser.getDefault().parseAddress("tb1qyatuwvkfx8thy2ntmtuea6v42vp3zefqvll8kx")) then: StepVerifier.create(act) @@ -125,9 +120,9 @@ class EsploraClientSpec extends Specification { ).respond( HttpResponse.response("[]") ) - def client = new EsploraClient(new URI("http://localhost:${port}"), null, null) + def client = new EsploraClient(new URI("http://localhost:${mockServer.port}"), null, null) when: - def act = client.getTransactions(Address.fromString(TestNet3Params.get(), "tb1qyatuwvkfx8thy2ntmtuea6v42vp3zefqvll8kx")) + def act = client.getTransactions(AddressParser.getDefault().parseAddress("tb1qyatuwvkfx8thy2ntmtuea6v42vp3zefqvll8kx")) then: StepVerifier.create(act) diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReaderSpec.groovy index 578ad8321..c80f238a6 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/RpcUnspentReaderSpec.groovy @@ -19,8 +19,7 @@ import io.emeraldpay.dshackle.reader.Reader import io.emeraldpay.dshackle.upstream.ChainRequest import io.emeraldpay.dshackle.upstream.ChainResponse import io.emeraldpay.dshackle.upstream.rpcclient.ListParams -import org.bitcoinj.core.Address -import org.bitcoinj.params.MainNetParams +import org.bitcoinj.base.AddressParser import reactor.core.publisher.Mono import spock.lang.Specification @@ -38,7 +37,7 @@ class RpcUnspentReaderSpec extends Specification { def reader = new RpcUnspentReader(upstreams) when: - def act = reader.read(Address.fromString(new MainNetParams(), "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK")).block() + def act = reader.read(AddressParser.getDefault().parseAddress("1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK")).block() then: // cat src/test/resources/bitcoin/unspent-one-addr.json | jq '. | length' @@ -72,7 +71,7 @@ class RpcUnspentReaderSpec extends Specification { def reader = new RpcUnspentReader(upstreams) when: - def act = reader.read(Address.fromString(new MainNetParams(), "35hK24tcLEWcgNA4JxpvbkNkoAcDGqQPsP")).block() + def act = reader.read(AddressParser.getDefault().parseAddress("35hK24tcLEWcgNA4JxpvbkNkoAcDGqQPsP")).block() then: // cat src/test/resources/bitcoin/unspent-two-addr.json | jq '[.[] | select(.address == "35hK24tcLEWcgNA4JxpvbkNkoAcDGqQPsP")] | length' @@ -107,7 +106,7 @@ class RpcUnspentReaderSpec extends Specification { def reader = new RpcUnspentReader(upstreams) when: - def act = reader.read(Address.fromString(new MainNetParams(), "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK")).block() + def act = reader.read(AddressParser.getDefault().parseAddress("1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK")).block() then: // cat src/test/resources/bitcoin/unspent-two-addr.json | jq '[.[] | select(.address == "1K7xkspJg7DDKNwzXgoRSDCUxiFsRegsSK")] | length' diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddressesSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddressesSpec.groovy index c3f6d096f..8ddd648d8 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddressesSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/bitcoin/XpubAddressesSpec.groovy @@ -15,8 +15,7 @@ */ package io.emeraldpay.dshackle.upstream.bitcoin -import org.bitcoinj.core.Address -import org.bitcoinj.params.MainNetParams +import org.bitcoinj.base.AddressParser import reactor.core.publisher.Mono import reactor.test.StepVerifier import spock.lang.Specification @@ -119,7 +118,7 @@ class XpubAddressesSpec extends Specification { def "list when only fist is active"() { setup: AddressActiveCheck check = Mock(AddressActiveCheck) { - 1 * isActive(Address.fromString(MainNetParams.get(), "bc1qaexx257l7sgm62szw2ulj6n2v99t5ph9ekkul3")) >> Mono.just(true) + 1 * isActive(AddressParser.getDefault().parseAddress("bc1qaexx257l7sgm62szw2ulj6n2v99t5ph9ekkul3")) >> Mono.just(true) 20 * isActive(_) >> Mono.just(false) } XpubAddresses addresses = new XpubAddresses(check) @@ -143,7 +142,7 @@ class XpubAddressesSpec extends Specification { def "list when only 3rd is active"() { setup: AddressActiveCheck check = Mock(AddressActiveCheck) { - 1 * isActive(Address.fromString(MainNetParams.get(), "bc1qah84zz7aavf3f5eyx5f29809y6xugqyphq0wtz")) >> Mono.just(true) + 1 * isActive(AddressParser.getDefault().parseAddress("bc1qah84zz7aavf3f5eyx5f29809y6xugqyphq0wtz")) >> Mono.just(true) // 2 times before, 20 times after 22 * isActive(_) >> Mono.just(false) } @@ -169,11 +168,11 @@ class XpubAddressesSpec extends Specification { setup: AddressActiveCheck check = Mock(AddressActiveCheck) { // 2 - 1 * isActive(Address.fromString(MainNetParams.get(), "bc1qah84zz7aavf3f5eyx5f29809y6xugqyphq0wtz")) >> Mono.just(true) + 1 * isActive(AddressParser.getDefault().parseAddress("bc1qah84zz7aavf3f5eyx5f29809y6xugqyphq0wtz")) >> Mono.just(true) // 11 - 1 * isActive(Address.fromString(MainNetParams.get(), "bc1q3kqug4cx95a02yhwn6geelftmw3zklrgmhjll8")) >> Mono.just(true) + 1 * isActive(AddressParser.getDefault().parseAddress("bc1q3kqug4cx95a02yhwn6geelftmw3zklrgmhjll8")) >> Mono.just(true) // 22 - 1 * isActive(Address.fromString(MainNetParams.get(), "bc1qf7m2rrmrksj34vhgxmm04y43dlj7c5f58f8ku6")) >> Mono.just(true) + 1 * isActive(AddressParser.getDefault().parseAddress("bc1qf7m2rrmrksj34vhgxmm04y43dlj7c5f58f8ku6")) >> Mono.just(true) // 0..1 = 2 // + 3..10 = 8 // + 12..21 = 10 diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/calls/DefaultEthereumMethodsSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/calls/DefaultEthereumMethodsSpec.groovy index b672cac25..7ffa434a4 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/calls/DefaultEthereumMethodsSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/calls/DefaultEthereumMethodsSpec.groovy @@ -58,11 +58,24 @@ class DefaultEthereumMethodsSpec extends Specification { where: chain | methods Chain.POLYGON__MAINNET | ["bor_getAuthor", - "bor_getCurrentValidators", - "bor_getCurrentProposer", - "bor_getRootHash", - "bor_getSignersAtHash", - "eth_getRootHash"] + "bor_getCurrentValidators", + "bor_getCurrentProposer", + "bor_getRootHash", + "bor_getSigners", + "bor_getSignersAtHash", + "bor_getSnapshot", + "bor_getSnapshotAtHash", + "eth_getRootHash"] + Chain.SHIBARIUM__MAINNET | ["bor_getAuthor", + "bor_getCurrentValidators", + "bor_getCurrentProposer", + "bor_getRootHash", + "bor_getSigners", + "bor_getSignersAtHash", + "bor_getSnapshot", + "bor_getSnapshotAtHash", + "eth_getRootHash", + "eth_getTransactionReceiptsByBlock"] Chain.OPTIMISM__MAINNET | ["rollup_gasPrices"] } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumCachingReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumCachingReaderSpec.groovy index 8db74557d..5fcf7ec96 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumCachingReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumCachingReaderSpec.groovy @@ -8,19 +8,18 @@ import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.data.DefaultContainer import io.emeraldpay.dshackle.reader.RequestReader import io.emeraldpay.dshackle.reader.RequestReaderFactory -import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.upstream.* import io.emeraldpay.dshackle.upstream.calls.DefaultEthereumMethods -import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson -import io.emeraldpay.dshackle.upstream.finalization.FinalizationType -import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import io.emeraldpay.dshackle.upstream.ethereum.domain.Address import io.emeraldpay.dshackle.upstream.ethereum.domain.BlockHash import io.emeraldpay.dshackle.upstream.ethereum.domain.TransactionId import io.emeraldpay.dshackle.upstream.ethereum.domain.Wei +import io.emeraldpay.dshackle.upstream.ethereum.json.BlockJson import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionJson import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionLogJson import io.emeraldpay.dshackle.upstream.ethereum.json.TransactionReceiptJson +import io.emeraldpay.dshackle.upstream.finalization.FinalizationType +import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import org.apache.commons.collections4.Factory import reactor.core.publisher.Mono import reactor.test.StepVerifier @@ -48,7 +47,7 @@ class EthereumDirectReaderSpec extends Specification { 1 * create() >> new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET) } EthereumDirectReader reader = new EthereumDirectReader( - Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls, TestingCommons.tracerMock() + Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls ) reader.requestReaderFactory = Mock(RequestReaderFactory) { 1 * create({ it.upstreamFilter.sort == Selector.Sort.safe }) >> Mock(RequestReader) { @@ -83,7 +82,7 @@ class EthereumDirectReaderSpec extends Specification { 1 * create() >> new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET) } EthereumDirectReader reader = new EthereumDirectReader( - Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls, TestingCommons.tracerMock() + Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls ) reader.requestReaderFactory = Mock(RequestReaderFactory) { 1 * create(_) >> Mock(RequestReader) { @@ -110,7 +109,7 @@ class EthereumDirectReaderSpec extends Specification { 1 * create() >> new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET) } EthereumDirectReader reader = new EthereumDirectReader( - Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls, TestingCommons.tracerMock() + Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls ) reader.requestReaderFactory = Mock(RequestReaderFactory) { 1 * create(_) >> Mock(RequestReader) { @@ -143,7 +142,7 @@ class EthereumDirectReaderSpec extends Specification { 1 * create() >> new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET) } EthereumDirectReader reader = new EthereumDirectReader( - Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls, TestingCommons.tracerMock() + Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls ) reader.requestReaderFactory = Mock(RequestReaderFactory) { 1 * create(_) >> Mock(RequestReader) { @@ -175,7 +174,7 @@ class EthereumDirectReaderSpec extends Specification { 1 * create() >> new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET) } EthereumDirectReader reader = new EthereumDirectReader( - Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls, TestingCommons.tracerMock() + Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls ) reader.requestReaderFactory = Mock(RequestReaderFactory) { 1 * create(_) >> Mock(RequestReader) { @@ -208,7 +207,7 @@ class EthereumDirectReaderSpec extends Specification { 1 * create() >> new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET) } EthereumDirectReader reader = new EthereumDirectReader( - Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls, TestingCommons.tracerMock() + Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls ) reader.requestReaderFactory = Mock(RequestReaderFactory) { 1 * create(_) >> Mock(RequestReader) { @@ -241,7 +240,7 @@ class EthereumDirectReaderSpec extends Specification { 1 * create() >> new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET) } EthereumDirectReader reader = new EthereumDirectReader( - Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls, TestingCommons.tracerMock() + Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls ) reader.requestReaderFactory = Mock(RequestReaderFactory) { 1 * create(_) >> Mock(RequestReader) { @@ -275,7 +274,7 @@ class EthereumDirectReaderSpec extends Specification { 1 * cacheReceipt(Caches.Tag.REQUESTED, { DefaultContainer data -> data.txId.toHex() == hash1.substring(2) && data.height == 100 }) } EthereumDirectReader reader = new EthereumDirectReader( - Stub(Multistream), caches, new CurrentBlockCache(), calls, TestingCommons.tracerMock() + Stub(Multistream), caches, new CurrentBlockCache(), calls ) reader.requestReaderFactory = Mock(RequestReaderFactory) { 1 * create(_) >> Mock(RequestReader) { @@ -300,7 +299,7 @@ class EthereumDirectReaderSpec extends Specification { 1 * create() >> new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET) } EthereumDirectReader reader = new EthereumDirectReader( - Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls, TestingCommons.tracerMock() + Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls ) reader.requestReaderFactory = Mock(RequestReaderFactory) { 1 * create(_) >> Mock(RequestReader) { @@ -330,7 +329,7 @@ class EthereumDirectReaderSpec extends Specification { 1 * create() >> new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET) } EthereumDirectReader reader = new EthereumDirectReader( - up, Caches.default(), new CurrentBlockCache(), calls, TestingCommons.tracerMock() + up, Caches.default(), new CurrentBlockCache(), calls ) reader.requestReaderFactory = Mock(RequestReaderFactory) { 1 * create(_) >> Mock(RequestReader) { @@ -361,7 +360,7 @@ class EthereumDirectReaderSpec extends Specification { 1 * create() >> new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET) } EthereumDirectReader reader = new EthereumDirectReader( - up, Caches.default(), new CurrentBlockCache(), calls, TestingCommons.tracerMock() + up, Caches.default(), new CurrentBlockCache(), calls ) reader.requestReaderFactory = Mock(RequestReaderFactory) { 1 * create(_) >> Mock(RequestReader) { @@ -399,7 +398,7 @@ class EthereumDirectReaderSpec extends Specification { Global.objectMapper.writeValueAsBytes(json), null, 1, data, null) ) EthereumDirectReader ethereumDirectReader = new EthereumDirectReader( - Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls, TestingCommons.tracerMock() + Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls ) ethereumDirectReader.requestReaderFactory = Mock(RequestReaderFactory) { 2 * create(_) >> Mock(RequestReader) { @@ -439,7 +438,7 @@ class EthereumDirectReaderSpec extends Specification { Global.objectMapper.writeValueAsBytes(json), null, 1, data, null) ) EthereumDirectReader ethereumDirectReader = new EthereumDirectReader( - Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls, TestingCommons.tracerMock() + Stub(Multistream), Caches.default(), new CurrentBlockCache(), calls ) ethereumDirectReader.requestReaderFactory = Mock(RequestReaderFactory) { 2 * create(_) >> Mock(RequestReader) { @@ -472,7 +471,7 @@ class EthereumDirectReaderSpec extends Specification { 4 * create() >> new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET) } EthereumDirectReader reader = new EthereumDirectReader( - up, Caches.default(), new CurrentBlockCache(), calls, TestingCommons.tracerMock() + up, Caches.default(), new CurrentBlockCache(), calls ) reader.requestReaderFactory = Mock(RequestReaderFactory) { 4 * create(_) >> Mock(RequestReader) { diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumEgressSubscriptionSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumEgressSubscriptionSpec.groovy index 452ae5b3c..6bde1fda1 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumEgressSubscriptionSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumEgressSubscriptionSpec.groovy @@ -17,6 +17,8 @@ package io.emeraldpay.dshackle.upstream.ethereum import io.emeraldpay.dshackle.test.TestingCommons +import io.emeraldpay.dshackle.upstream.ethereum.subscribe.AggregatedPendingTxes +import io.emeraldpay.dshackle.upstream.ethereum.subscribe.NoPendingTxes import io.emeraldpay.dshackle.upstream.generic.GenericMultistream import io.emeraldpay.dshackle.upstream.ethereum.subscribe.PendingTxesSource import io.emeraldpay.dshackle.upstream.ethereum.domain.Address @@ -216,6 +218,7 @@ class EthereumEgressSubscriptionSpec extends Specification { when: def up3 = TestingCommons.upstream("test") up3.getConnectorMock().setLiveness(Flux.just(HeadLivenessState.OK)) + up3.getConnectorMock().setPendingTxs(Flux.just(true)) up3.stop() up3.start() def ethereumSubscribe3 = new EthereumEgressSubscription(TestingCommons.multistream(up3) as GenericMultistream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) @@ -230,5 +233,23 @@ class EthereumEgressSubscriptionSpec extends Specification { then: ethereumSubscribe4.getAvailableTopics().toSet() == [EthereumEgressSubscription.METHOD_NEW_HEADS].toSet() + when: + def up5 = TestingCommons.upstream("test") + up5.getConnectorMock().setLiveness(Flux.just(HeadLivenessState.OK)) + up5.stop() + up5.start() + def ethereumSubscribe5 = new EthereumEgressSubscription(TestingCommons.multistream(up5) as GenericMultistream, Schedulers.boundedElastic(), Stub(PendingTxesSource)) + then: + ethereumSubscribe5.getAvailableTopics().toSet() == [EthereumEgressSubscription.METHOD_LOGS, EthereumEgressSubscription.METHOD_NEW_HEADS].toSet() + + when: + def up6 = TestingCommons.upstream("test") + up6.getConnectorMock().setLiveness(Flux.just(HeadLivenessState.OK)) + up6.getConnectorMock().setPendingTxs(Flux.just(true)) + up6.stop() + up6.start() + def ethereumSubscribe6 = new EthereumEgressSubscription(TestingCommons.multistream(up6) as GenericMultistream, Schedulers.boundedElastic(), new NoPendingTxes()) + then: + ethereumSubscribe6.getAvailableTopics().toSet() == [EthereumEgressSubscription.METHOD_LOGS, EthereumEgressSubscription.METHOD_NEW_HEADS].toSet() } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLocalReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLocalReaderSpec.groovy index 4109030ee..d970c4bea 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLocalReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumLocalReaderSpec.groovy @@ -21,7 +21,6 @@ class EthereumLocalReaderSpec extends Specification { TestingCommons.multistream(TestingCommons.api()), Caches.default(), ConstantFactory.constantFactory(new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET)), - TestingCommons.tracerMock() ), methods ) @@ -31,22 +30,26 @@ class EthereumLocalReaderSpec extends Specification { act.resultAsProcessedString == "0x0000000000000000000000000000000000000000" } - def "Returns empty if nonce set"() { + def "Serves non-hardcoded call when nonce is set"() { setup: def methods = new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET) + def api = TestingCommons.api() + api.answer("eth_getTransactionByHash", + ["0x0000000000000000000000000000000000000000000000000000000000000001"], null) def router = new EthereumLocalReader( new EthereumCachingReader( - TestingCommons.multistream(TestingCommons.api()), + TestingCommons.multistream(api), Caches.default(), ConstantFactory.constantFactory(new DefaultEthereumMethods(Chain.ETHEREUM__MAINNET)), - TestingCommons.tracerMock() ), methods ) when: - def act = router.read(new ChainRequest("eth_getTransactionByHash", new ListParams(["test"]), 10)) + def act = router.read(new ChainRequest("eth_getTransactionByHash", + new ListParams(["0x0000000000000000000000000000000000000000000000000000000000000001"]), + 10)) .block(Duration.ofSeconds(1)) then: - act == null + act != null } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamSettingsDetectorSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamSettingsDetectorSpec.groovy index eb4b04c72..13e868ad6 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamSettingsDetectorSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/EthereumUpstreamSettingsDetectorSpec.groovy @@ -239,4 +239,81 @@ class EthereumUpstreamSettingsDetectorSpec extends Specification { .expectComplete() .verify(Duration.ofSeconds(1)) } + + // Regression: Moca Tendermint EVM returns a JSON string with raw, unescaped LFs: + // "Version dev ()\nCompiled at using Go go1.23.11 (amd64)" + // Jackson's default parser rejects this with "Illegal unquoted character (code 10)", + // which previously caused EthereumUpstreamSettingsDetector to fail node type detection. + def "Detect node type when client version contains unescaped control chars (Moca Tendermint)"() { + setup: + def rawVersion = "Version dev ()\nCompiled at using Go go1.23.11 (amd64)" + def jsonResultBytes = ('"' + rawVersion + '"').getBytes("UTF-8") + def up = Mock(DefaultUpstream) { + getId() >> "tiernet-us-east-bcn-05-moca-mainnet" + 6 * getIngressReader() >> Mock(Reader) { + 1 * read(new ChainRequest("web3_clientVersion", new ListParams())) >> + Mono.just(new ChainResponse(jsonResultBytes, null)) + 1 * read(new ChainRequest("eth_blockNumber", new ListParams())) >> + Mono.just(new ChainResponse("\"0x10df3e5\"".getBytes(), null)) + 1 * read(new ChainRequest("eth_getBalance", new ListParams(["0x0000000000000000000000000000000000000000", "0x10dccd5"]))) >> + Mono.error(new RuntimeException()) + 1 * read(new ChainRequest("eth_getBalance", new ListParams(["0x0000000000000000000000000000000000000000", "0x2710"]))) >> + Mono.just(new ChainResponse("".getBytes(), null)) + 1 * read(new ChainRequest("eth_call", new ListParams([ + "to": "0x53Daa71B04d589429f6d3DF52db123913B818F22", + "data": "0x51be4eaa", + ], + "latest", + [ + "0x53Daa71B04d589429f6d3DF52db123913B818F22": [ + "code": "0x6080604052348015600f57600080fd5b506004361060285760003560e01c806351be4eaa14602d575b600080fd5b60336047565b604051603e91906066565b60405180910390f35b60005a905090565b6000819050919050565b606081604f565b82525050565b6000602082019050607960008301846059565b9291505056fea26469706673582212201c0202887c1afe66974b06ee355dee07542bbc424cf4d1659c91f56c08c3dcc064736f6c63430008130033", + ], + ], + ))) >> + Mono.just(new ChainResponse("".getBytes(), null)) + 1 * read(new ChainRequest("eth_getBlockByNumber", new ListParams(["pending", false]))) >> + Mono.just(new ChainResponse("{}".getBytes(), null)) + } + getLabels() >> [] + } + def detector = new EthereumUpstreamSettingsDetector(up, Chain.ETHEREUM__MAINNET) + when: + def act = detector.internalDetectLabels() + then: + // The detector must not crash on the unescaped LF. With ALLOW_UNQUOTED_CONTROL_CHARS + // the JSON parses, and the resulting string runs through the existing + // slash/semver/dot logic: no slash, not semver-like, contains a dot -> + // client_type falls back to "default client" and client_version is the raw + // version string (preserved as-is, including the embedded LF). + StepVerifier.create(act) + .expectNext(new Pair("client_type", "default client")) + .expectNext(new Pair("client_version", rawVersion)) + .expectNext(new Pair("archive", "false")) + .expectNext(new Pair("flashblocks", "false")) + .expectComplete() + .verify(Duration.ofSeconds(1)) + } + + def "detectClientVersion handles unescaped control chars in version string"() { + setup: + def rawVersion = "Version dev ()\nCompiled at using Go go1.23.11 (amd64)" + def jsonResultBytes = ('"' + rawVersion + '"').getBytes("UTF-8") + def up = Mock(DefaultUpstream) { + 2 * getIngressReader() >> Mock(Reader) { + 1 * read(new ChainRequest("web3_clientVersion", new ListParams())) >> + Mono.just(new ChainResponse(jsonResultBytes, null)) + } + 1 * getLabels() >> List.of() + } + def detector = new EthereumUpstreamSettingsDetector(up, Chain.ETHEREUM__MAINNET) + when: + def act = detector.detectClientVersion() + then: + // parseClientVersion only strips the outer JSON quotes; the embedded LF is + // passed through unchanged. The important behavior is that it does not throw. + StepVerifier.create(act) + .expectNext(rawVersion) + .expectComplete() + .verify(Duration.ofSeconds(1)) + } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplRealSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplRealSpec.groovy index 969b65fcd..d2fa73086 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplRealSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplRealSpec.groovy @@ -43,6 +43,7 @@ class WsConnectionImplRealSpec extends Specification { Chain.ETHEREUM__MAINNET, "ws://localhost:${port}".toURI(), "http://localhost:${port}".toURI(), + Schedulers.boundedElastic(), Schedulers.boundedElastic() ) ).create(upstream).getConnection() diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplSpec.groovy index eeff55eee..18601baec 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/WsConnectionImplSpec.groovy @@ -46,7 +46,8 @@ class WsConnectionImplSpec extends Specification { Chain.ETHEREUM__MAINNET, new URI("http://localhost"), new URI("http://localhost"), - Schedulers.boundedElastic() + Schedulers.boundedElastic(), + Schedulers.boundedElastic(), ) ) def apiMock = TestingCommons.api() @@ -81,7 +82,8 @@ class WsConnectionImplSpec extends Specification { Chain.ETHEREUM__MAINNET, new URI("http://localhost"), new URI("http://localhost"), - Schedulers.boundedElastic() + Schedulers.boundedElastic(), + Schedulers.boundedElastic(), ) ) def apiMock = TestingCommons.api() @@ -114,7 +116,8 @@ class WsConnectionImplSpec extends Specification { Chain.ETHEREUM__MAINNET, new URI("http://localhost"), new URI("http://localhost"), - Schedulers.boundedElastic() + Schedulers.boundedElastic(), + Schedulers.boundedElastic(), ) ) def apiMock = TestingCommons.api() diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/WebsocketPendingTxesSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/WebsocketPendingTxesSpec.groovy index 68d453e09..f1dd5a021 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/WebsocketPendingTxesSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/ethereum/subscribe/WebsocketPendingTxesSpec.groovy @@ -22,10 +22,12 @@ import io.emeraldpay.dshackle.upstream.ethereum.WsSubscriptions import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import reactor.core.publisher.Flux import reactor.core.publisher.Mono +import reactor.test.StepVerifier import reactor.util.function.Tuples import spock.lang.Specification import java.time.Duration +import java.util.concurrent.TimeoutException import java.util.concurrent.atomic.AtomicReference class WebsocketPendingTxesSpec extends Specification { @@ -55,4 +57,26 @@ class WebsocketPendingTxesSpec extends Specification { "0x67f22a3b441ea312306f97694ca8159f8d6faaccf0f5ce6442c84b13991f1d23", ] } + + def "Emits TimeoutException when upstream goes silent past idle timeout"() { + // Pins the fix for the silent-stall bug: a healthy WS that simply stops delivering + // newPendingTransactions events must surface a TimeoutException so DurableFlux + // re-issues eth_subscribe. + setup: + def ws = Stub(WsSubscriptions) { + subscribe(new ChainRequest("eth_subscribe", new ListParams(["newPendingTransactions"]))) >> new WsSubscriptions.SubscribeData( + Mono.just(Tuples.of("sub-1", Flux.never())), "conn-1", new AtomicReference("sub-1") + ) + unsubscribe(_) >> Mono.empty() + } + def pending = new WebsocketPendingTxes(Chain.ETHEREUM__MAINNET, ws) + + expect: + StepVerifier.withVirtualTime { pending.createConnection() } + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(84)) + .thenAwait(Duration.ofSeconds(2)) + .expectError(TimeoutException) + .verify(Duration.ofSeconds(5)) + } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcErrorSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcErrorSpec.groovy index 1c2a7deb5..ad81f8b1b 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcErrorSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcErrorSpec.groovy @@ -2,7 +2,6 @@ package io.emeraldpay.dshackle.upstream.rpcclient import io.emeraldpay.dshackle.upstream.ChainCallError import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException -import nl.jqno.equalsverifier.EqualsVerifier import spock.lang.Specification class JsonRpcErrorSpec extends Specification { @@ -24,11 +23,4 @@ class JsonRpcErrorSpec extends Specification { act.message == "test test" act.details == "foo bar" } - - def "Equals"() { - when: - def v = EqualsVerifier.forClass(ChainCallError) - then: - v.verify() - } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpReaderSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpReaderSpec.groovy index 8589b780c..588a3e54f 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpReaderSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/JsonRpcHttpReaderSpec.groovy @@ -16,6 +16,7 @@ package io.emeraldpay.dshackle.upstream.rpcclient +import io.emeraldpay.dshackle.config.AuthConfig import io.emeraldpay.dshackle.test.TestingCommons import io.emeraldpay.dshackle.upstream.ChainException import io.emeraldpay.dshackle.upstream.ChainRequest @@ -26,7 +27,7 @@ import io.micrometer.core.instrument.Timer import org.mockserver.integration.ClientAndServer import org.mockserver.model.HttpRequest import org.mockserver.model.HttpResponse -import org.springframework.util.SocketUtils +import reactor.core.scheduler.Schedulers import spock.lang.Specification import java.time.Duration @@ -34,7 +35,6 @@ import java.time.Duration class JsonRpcHttpReaderSpec extends Specification { ClientAndServer mockServer - int port = 19332 RequestMetrics metrics = new RequestMetrics( Timer.builder("test1").register(TestingCommons.meterRegistry), Counter.builder("test2").register(TestingCommons.meterRegistry), @@ -42,8 +42,7 @@ class JsonRpcHttpReaderSpec extends Specification { ) def setup() { - port = SocketUtils.findAvailableTcpPort(19332) - mockServer = ClientAndServer.startClientAndServer(port); + mockServer = ClientAndServer.startClientAndServer(0); } def cleanup() { @@ -52,7 +51,7 @@ class JsonRpcHttpReaderSpec extends Specification { def "Make a request"() { setup: - JsonRpcHttpReader client = new JsonRpcHttpReader("localhost:${port}", 50, 50, metrics,null, null) + JsonRpcHttpReader client = new JsonRpcHttpReader("localhost:${mockServer.port}", 50, 50, metrics, Schedulers.boundedElastic(),null, null, [:]) def resp = '{' + ' "jsonrpc": "2.0",' + ' "result": "0x98de45",' + @@ -71,10 +70,31 @@ class JsonRpcHttpReaderSpec extends Specification { new String(act.result) == '"0x98de45"' } - def "Produces RPC Exception on error status code"() { + def "Make a request with bearer auth"() { setup: - def client = new JsonRpcHttpReader("localhost:${port}", 50, 50, metrics, null, null) + def bearerAuth = new AuthConfig.ClientBearerAuth("test-token-123") + JsonRpcHttpReader client = new JsonRpcHttpReader("localhost:${mockServer.port}", 50, 50, metrics, Schedulers.boundedElastic(), null, null, [:], bearerAuth) + def resp = '{' + + ' "jsonrpc": "2.0",' + + ' "result": "0x98de45",' + + ' "error": null,' + + ' "id": 15' + + '}' + mockServer.when( + HttpRequest.request().withHeader("authorization", "Bearer test-token-123") + ).respond( + HttpResponse.response(resp) + ) + when: + def act = client.read(new ChainRequest("test", new ListParams())).block(Duration.ofSeconds(5)) + then: + act.error == null + new String(act.result) == '"0x98de45"' + } + def "Produces RPC Exception on error status code"() { + setup: + def client = new JsonRpcHttpReader("localhost:${mockServer.port}", 50, 50, metrics, Schedulers.boundedElastic(), null, null, [:]) mockServer.when( HttpRequest.request() ).respond( @@ -97,7 +117,7 @@ class JsonRpcHttpReaderSpec extends Specification { def "Tries to extract message if HTTP error if it still contains a JSON RPC message"() { setup: - def client = new JsonRpcHttpReader("localhost:${port}", 50, 50, metrics, null, null) + def client = new JsonRpcHttpReader("localhost:${mockServer.port}", 50, 50, metrics, Schedulers.boundedElastic(), null, null, [:]) mockServer.when( HttpRequest.request() diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/ResponseRpcParserSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/ResponseRpcParserSpec.groovy index 5d4796c52..04d842b97 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/ResponseRpcParserSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/ResponseRpcParserSpec.groovy @@ -235,4 +235,18 @@ class ResponseRpcParserSpec extends Specification { !act.hasResult() } + // Regression: Moca's Tendermint EVM returns a web3_clientVersion result + // that contains a raw, unescaped LF (CTRL-CHAR, code 10) inside the JSON + // string. Default Jackson rejects that with "Illegal unquoted character", + // which used to break upstream node-type detection. + def "Parse string response with unescaped control chars"() { + setup: + def json = '{"jsonrpc":"2.0","id":1,"result":"Version dev ()\nCompiled at using Go go1.23.11 (amd64)"}' + when: + def act = parser.parse(json.getBytes("UTF-8")) + then: + act.error == null + new String(act.result) == '"Version dev ()\nCompiled at using Go go1.23.11 (amd64)"' + } + } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/ResponseWSParserSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/ResponseWSParserSpec.groovy index 460e5e023..a0f91a6f3 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/ResponseWSParserSpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/rpcclient/ResponseWSParserSpec.groovy @@ -103,4 +103,47 @@ class ResponseWSParserSpec extends Specification { act.error == null new String(act.value) == "null" } + + def "Parse Ripple ledgerClosed subscription event"() { + setup: + def msg = '''{ + "type": "ledgerClosed", + "fee_base": 10, + "fee_ref": 10, + "ledger_hash": "17ACB57A0F73B5160713E81FE72B2AC9F6064541004E272BD09F257D57C30C02", + "ledger_index": 6643099, + "ledger_time": 780804221, + "reserve_base": 10000000, + "reserve_inc": 2000000, + "txn_count": 5, + "validated_ledgers": "6643000-6643099" + }''' + when: + def act = parser.parse(msg.bytes) + then: + act.type == ResponseWSParser.Type.SUBSCRIPTION + act.id.asString() == "ledgerClosed" + act.error == null + with(new String(act.value)) { + it.contains("\"ledger_hash\"") + it.contains("17ACB57A0F73B5160713E81FE72B2AC9F6064541004E272BD09F257D57C30C02") + it.contains("\"ledger_index\": 6643099") + } + } + + def "Parse Ripple RPC response with type field"() { + setup: + def msg = '''{ + "id": 1, + "status": "success", + "type": "response", + "result": {} + }''' + when: + def act = parser.parse(msg.bytes) + then: + act.type == ResponseWSParser.Type.RPC + act.id.asNumber() == 1L + act.error == null + } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/DisabledSignerSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/DisabledSignerSpec.groovy new file mode 100644 index 000000000..54f423643 --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/DisabledSignerSpec.groovy @@ -0,0 +1,28 @@ +package io.emeraldpay.dshackle.upstream.signature + +import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException +import spock.lang.Specification + +class DisabledSignerSpec extends Specification { + + def "sign throws RpcException with CODE_INTERNAL_ERROR"() { + setup: + def signer = new DisabledSigner() + + when: + signer.sign(1L, "data".bytes, "upstreamId") + + then: + def ex = thrown(RpcException) + ex.code == -32603 + ex.rpcMessage.contains("signing key is not configured") + } + + def "Signer is not enabled"() { + setup: + def signer = new DisabledSigner() + + expect: + !signer.enabled + } +} diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/EcdsaSignerSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/EcdsaSignerSpec.groovy deleted file mode 100644 index 98a8acbb8..000000000 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/EcdsaSignerSpec.groovy +++ /dev/null @@ -1,135 +0,0 @@ -package io.emeraldpay.dshackle.upstream.signature - -import io.emeraldpay.dshackle.config.SignatureConfig -import io.emeraldpay.dshackle.upstream.Upstream -import org.apache.commons.codec.binary.Hex -import org.bouncycastle.jce.provider.BouncyCastleProvider -import org.bouncycastle.util.io.pem.PemObject -import org.bouncycastle.util.io.pem.PemWriter -import spock.lang.Specification - -import java.security.KeyFactory -import java.security.KeyPairGenerator -import java.security.MessageDigest -import java.security.Security -import java.security.Signature -import java.security.interfaces.ECPrivateKey -import java.security.spec.ECGenParameterSpec -import java.security.spec.PKCS8EncodedKeySpec - -class EcdsaSignerSpec extends Specification { - - def setupSpec() { - Security.addProvider(new BouncyCastleProvider()) - } - - def "Reads private key NIST P256"() { - setup: - def file = File.createTempFile("test", ".pem") - def keygen = KeyPairGenerator.getInstance("EC") - keygen.initialize(new ECGenParameterSpec("secp256r1")) - def key = keygen.generateKeyPair() - def keyBuilder = new PKCS8EncodedKeySpec(key.getPrivate().getEncoded()) - def writer = new PemWriter(new FileWriter(file.path)) - writer.writeObject(new PemObject("PRIVATE KEY", keyBuilder.getEncoded())) - writer.close() - - when: - def signer = new ResponseSignerFactory(new SignatureConfig()) - def act = signer.readKey(SignatureConfig.Algorithm.NIST_P256, file.absolutePath).first - - then: - act == key.getPrivate() - - cleanup: - file.delete() - } - - def "Id is a hash of x509 public key"() { - setup: - def conf = new SignatureConfig() - conf.enabled = true - conf.privateKey = "src/test/resources/signer/test_key" - def signer = new ResponseSignerFactory(conf).getObject() as EcdsaSigner - - // To verify the test, check the hash of test key above: - // - // echo MFYwEAYHKoZIzj0CAQYFK4EEAAoDQgAE3zetdMdyTO/sTFCLeOrI5moiZt2RjfUVdavhorgqd+gxAqM01cf5Q4QZ8INne9RykcQsbLYXQfDXJbGMm5+gdg== | base64 -d - | shasum -a 256 - // d25f1ff2c1a57235a9bc7725cd645ab0e9631475a12402f2881579d3f6887597 - - // - - when: - def id = signer.keyId - - then: - id == 0xed397068b172b393L - } - - def "Wrap message"() { - setup: - def up = Mock(Upstream) { - _ * getId() >> "infura" - } - def signer = new EcdsaSigner(Stub(ECPrivateKey), 100L) - - when: - def act = signer.wrapMessage(10, "test".bytes, up.id) - - then: - act == "DSHACKLESIG/10/infura/9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" - } - - def "Signed message is valid"() { - setup: - def result = "test".bytes - def up = Mock(Upstream) { - _ * getId() >> "infura" - } - - def keyPairGen = KeyPairGenerator.getInstance("EC") - keyPairGen.initialize(new ECGenParameterSpec("secp256r1")) - def pair = keyPairGen.generateKeyPair() - def verifier = Signature.getInstance("SHA256withECDSA") - verifier.initVerify(pair.getPublic()) - verifier.update("DSHACKLESIG/10/infura/9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08".getBytes()) - - def signer = new EcdsaSigner((pair.getPrivate() as ECPrivateKey), 100L) - - when: - def sig = signer.sign(10, result, up.id) - - then: - verifier.verify(sig.value) - } - - def "Signed message is valid - for docs"() { - // it's the example used in docs - setup: - def result = '["0xe670ec64341771606e55d6b4ca35a1a6b75ee3d5145a99d05921026d1527331", true]'.bytes - def up = Mock(Upstream) { - _ * getId() >> "infura" - } - - def sha256 = MessageDigest.getInstance("SHA-256") - - def conf = new SignatureConfig() - conf.enabled = true - conf.privateKey = "src/test/resources/signer/test_key" - def factory = new ResponseSignerFactory(conf) - - def sk = factory.readKey(conf.algorithm, conf.privateKey).first - def pk = factory.extractPublicKey(KeyFactory.getInstance("EC"), sk, SignatureConfig.Algorithm.NIST_P256) - def verifier = Signature.getInstance("SHA256withECDSA") - verifier.initVerify(pk) - verifier.update("DSHACKLESIG/10/infura/${Hex.encodeHexString(sha256.digest(result))}".getBytes()) - - def signer = factory.getObject() as EcdsaSigner - - when: - def sig = signer.sign(10, result, up.id) - println("Signature: ${Hex.encodeHexString(sig.value)}") - - then: - verifier.verify(sig.value) - } -} diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/ResponseSignerFactorySpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/ResponseSignerFactorySpec.groovy index 0e275a7bf..4769bb2fd 100644 --- a/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/ResponseSignerFactorySpec.groovy +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/ResponseSignerFactorySpec.groovy @@ -1,28 +1,90 @@ package io.emeraldpay.dshackle.upstream.signature - -import io.emeraldpay.dshackle.config.SignatureConfig +import io.emeraldpay.dshackle.config.AuthorizationConfig +import io.emeraldpay.dshackle.upstream.ethereum.rpc.RpcException +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.springframework.util.ResourceUtils import spock.lang.Specification +import java.security.Security + class ResponseSignerFactorySpec extends Specification { + def setupSpec() { + Security.addProvider(new BouncyCastleProvider()) + } + + def "DisabledSigner when auth disabled"() { + setup: + def auth = AuthorizationConfig.default() + + when: + def signer = new ResponseSignerFactory(auth).createSigner() + + then: + signer instanceof DisabledSigner + } - def "No signer if not enabled"() { + def "DisabledSigner when provider-private-key path is blank"() { setup: - def conf = new SignatureConfig() + def auth = new AuthorizationConfig( + true, + "owner", + new AuthorizationConfig.ServerConfig("", "classpath:keys/public.pem"), + AuthorizationConfig.ClientConfig.default(), + ) + when: - def signer = new ResponseSignerFactory(conf).getObject() + def signer = new ResponseSignerFactory(auth).createSigner() + then: - signer instanceof NoSigner + signer instanceof DisabledSigner } - def "No signer if privkey is not configured"() { + def "RsaSigner built from valid RSA key"() { setup: - def conf = new SignatureConfig() + def privPath = ResourceUtils.getFile("classpath:keys/priv.p8.key").absolutePath + def pubPath = ResourceUtils.getFile("classpath:keys/public.pem").absolutePath + def auth = new AuthorizationConfig( + true, + "owner", + new AuthorizationConfig.ServerConfig(privPath, pubPath), + AuthorizationConfig.ClientConfig.default(), + ) + when: - def signer = new ResponseSignerFactory(conf).getObject() + def signer = new ResponseSignerFactory(auth).createSigner() + then: - signer instanceof NoSigner + signer instanceof RsaSigner + (signer as RsaSigner).keyId != 0L } + def "Fails on missing key file"() { + setup: + def auth = new AuthorizationConfig( + true, + "owner", + new AuthorizationConfig.ServerConfig("/no/such/file.pem", "classpath:keys/public.pem"), + AuthorizationConfig.ClientConfig.default(), + ) + + when: + new ResponseSignerFactory(auth).createSigner() + + then: + thrown(Exception) + } + + def "DisabledSigner.sign throws RpcException"() { + setup: + def signer = new ResponseSignerFactory(AuthorizationConfig.default()).createSigner() + + when: + signer.sign(1L, "data".bytes, "up") + + then: + def ex = thrown(RpcException) + ex.code == -32603 + } } diff --git a/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/RsaSignerSpec.groovy b/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/RsaSignerSpec.groovy new file mode 100644 index 000000000..d73a1a434 --- /dev/null +++ b/src/test/groovy/io/emeraldpay/dshackle/upstream/signature/RsaSignerSpec.groovy @@ -0,0 +1,112 @@ +package io.emeraldpay.dshackle.upstream.signature + +import org.apache.commons.codec.binary.Hex +import org.bouncycastle.jce.provider.BouncyCastleProvider +import spock.lang.Specification + +import java.security.KeyPairGenerator +import java.security.MessageDigest +import java.security.Security +import java.security.Signature +import java.security.interfaces.RSAPrivateKey + +class RsaSignerSpec extends Specification { + + def setupSpec() { + Security.addProvider(new BouncyCastleProvider()) + } + + def "Wrap message"() { + setup: + def signer = new RsaSigner(Stub(RSAPrivateKey), 100L) + + when: + def act = signer.wrapMessage(10, "test".bytes, "infura") + + then: + act == "DSHACKLESIG/10/infura/9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + } + + def "Signed message is valid"() { + setup: + def result = "test".bytes + def keyGen = KeyPairGenerator.getInstance("RSA") + keyGen.initialize(2048) + def pair = keyGen.generateKeyPair() + + def sha256 = MessageDigest.getInstance("SHA-256") + def verifier = Signature.getInstance("SHA256withRSA", "BC") + verifier.initVerify(pair.getPublic()) + verifier.update("DSHACKLESIG/10/infura/${Hex.encodeHexString(sha256.digest(result))}".getBytes()) + + def signer = new RsaSigner((pair.getPrivate() as RSAPrivateKey), 100L) + + when: + def sig = signer.sign(10, result, "infura") + + then: + verifier.verify(sig.value) + sig.upstreamId == "infura" + sig.keyId == 100L + } + + def "Wrap nonce >= 2^63 as unsigned"() { + setup: + def signer = new RsaSigner(Stub(RSAPrivateKey), 100L) + // bit pattern of 12241848404401059555 (uint64) == -6204895669308492061 (signed Long) + def nonceBits = -6204895669308492061L + + when: + def act = signer.wrapMessage(nonceBits, "test".bytes, "infura") + + then: + act == "DSHACKLESIG/12241848404401059555/infura/9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + !act.contains("-") + } + + def "Signed message is valid for nonce >= 2^63"() { + setup: + def result = "test".bytes + def keyGen = KeyPairGenerator.getInstance("RSA") + keyGen.initialize(2048) + def pair = keyGen.generateKeyPair() + // bit pattern of 12241848404401059555 (uint64) == -6204895669308492061 (signed Long) + def nonceBits = -6204895669308492061L + + def sha256 = MessageDigest.getInstance("SHA-256") + def verifier = Signature.getInstance("SHA256withRSA", "BC") + verifier.initVerify(pair.getPublic()) + verifier.update("DSHACKLESIG/12241848404401059555/infura/${Hex.encodeHexString(sha256.digest(result))}".getBytes()) + + def signer = new RsaSigner((pair.getPrivate() as RSAPrivateKey), 100L) + + when: + def sig = signer.sign(nonceBits, result, "infura") + + then: + verifier.verify(sig.value) + } + + def "Signer is enabled"() { + setup: + def signer = new RsaSigner(Stub(RSAPrivateKey), 1L) + + expect: + signer.enabled + } + + def "Different nonce produces different signature"() { + setup: + def keyGen = KeyPairGenerator.getInstance("RSA") + keyGen.initialize(2048) + def pair = keyGen.generateKeyPair() + def signer = new RsaSigner((pair.getPrivate() as RSAPrivateKey), 1L) + + when: + def sig1 = signer.sign(1, "test".bytes, "up") + def sig2 = signer.sign(2, "test".bytes, "up") + + then: + !Arrays.equals(sig1.value, sig2.value) + } +} diff --git a/src/test/kotlin/io/emeraldpay/dshackle/IntegrationTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/IntegrationTest.kt index 0beb17f6a..83191b308 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/IntegrationTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/IntegrationTest.kt @@ -10,14 +10,13 @@ import io.emeraldpay.dshackle.reader.BroadcastReader import io.emeraldpay.dshackle.reader.RequestReaderFactory import io.emeraldpay.dshackle.upstream.MultistreamHolder import io.emeraldpay.dshackle.upstream.Selector +import io.emeraldpay.dshackle.upstream.signature.DisabledSigner import io.grpc.BindableService import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test -import org.mockito.Mockito.mock import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.boot.test.context.TestConfiguration -import org.springframework.cloud.sleuth.Tracer import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Import import org.springframework.context.annotation.Profile @@ -57,7 +56,6 @@ class IntegrationTest { fun testUpstreamTxMethodQuorumAndReader() { val ms = multistreamHolder.getUpstream(Chain.ETHEREUM__MAINNET) val ethUpstream = ms.getUpstreams()[0] - val tracer = mock() val reqReader = RequestReaderFactory.default() val txQuorum = ethUpstream.getMethods().createQuorumFor("eth_sendRawTransaction") @@ -68,8 +66,7 @@ class IntegrationTest { ms, Selector.UpstreamFilter.default, txQuorum, - null, - tracer, + DisabledSigner(), ), ) val txCountReader = reqReader.create( @@ -77,8 +74,7 @@ class IntegrationTest { ms, Selector.UpstreamFilter.default, txCountQuorum, - null, - tracer, + DisabledSigner(), ), ) diff --git a/src/test/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReaderTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReaderTest.kt new file mode 100644 index 000000000..b28d89f64 --- /dev/null +++ b/src/test/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfigReaderTest.kt @@ -0,0 +1,161 @@ +package io.emeraldpay.dshackle.config + +import io.emeraldpay.dshackle.FileResolver +import io.emeraldpay.dshackle.foundation.ChainOptionsReader +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType +import io.emeraldpay.dshackle.upstream.lowerbound.ManualLowerBoundType +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.io.File + +class UpstreamsConfigReaderTest { + + @Test + fun `should parse additional settings with lower bounds`() { + val yaml = """ + version: v1 + upstreams: + - id: test-upstream + chain: ethereum + additional-settings: + manual-lower-bounds: + state: + type: head + value: -5000000 + block: + type: fixed + value: 1 + tx: + type: fixed + value: 123 + receipts: + type: fixed + value: 15661 + slot: + type: head + value: -34566 + connection: + ethereum: + rpc: + url: "http://localhost:8545" + """.trimIndent() + + val reader = UpstreamsConfigReader( + FileResolver(File(".")), + ChainOptionsReader(), + ) + + val config = reader.readInternal(yaml.byteInputStream()) + + assertNotNull(config) + assertEquals(1, config.upstreams.size) + + val upstream = config.upstreams[0] + assertEquals("test-upstream", upstream.id) + assertEquals( + UpstreamsConfig.AdditionalSettings( + mapOf( + LowerBoundType.SLOT to UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.HEAD, -34566), + LowerBoundType.RECEIPTS to UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.FIXED, 15661), + LowerBoundType.TX to UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.FIXED, 123), + LowerBoundType.BLOCK to UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.FIXED, 1), + LowerBoundType.STATE to UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.HEAD, -5000000), + ), + ), + upstream.additionalSettings, + ) + } + + @Test + fun `should parse customHeaders from YAML`() { + val yaml = """ + version: v1 + upstreams: + - id: test-upstream + chain: ethereum + custom-headers: + X-Custom-Header: "custom-value" + Authorization: "Bearer token" + X-Another-Header: "another-value" + connection: + ethereum: + rpc: + url: "http://localhost:8545" + """.trimIndent() + + val reader = UpstreamsConfigReader( + FileResolver(File(".")), + ChainOptionsReader(), + ) + + val config = reader.readInternal(yaml.byteInputStream()) + + assertNotNull(config) + assertEquals(1, config.upstreams.size) + + val upstream = config.upstreams[0] + assertEquals("test-upstream", upstream.id) + assertEquals(3, upstream.customHeaders.size) + assertEquals("custom-value", upstream.customHeaders["X-Custom-Header"]) + assertEquals("Bearer token", upstream.customHeaders["Authorization"]) + assertEquals("another-value", upstream.customHeaders["X-Another-Header"]) + } + + @Test + fun `should work without customHeaders`() { + val yaml = """ + version: v1 + upstreams: + - id: test-upstream + chain: ethereum + connection: + ethereum: + rpc: + url: "http://localhost:8545" + """.trimIndent() + + val reader = UpstreamsConfigReader( + FileResolver(File(".")), + ChainOptionsReader(), + ) + + val config = reader.readInternal(yaml.byteInputStream()) + + assertNotNull(config) + assertEquals(1, config.upstreams.size) + + val upstream = config.upstreams[0] + assertEquals("test-upstream", upstream.id) + assertTrue(upstream.customHeaders.isEmpty()) + } + + @Test + fun `should trim header names and values`() { + val yaml = """ + version: v1 + upstreams: + - id: test-upstream + chain: ethereum + custom-headers: + " X-Header ": " value " + connection: + ethereum: + rpc: + url: "http://localhost:8545" + """.trimIndent() + + val reader = UpstreamsConfigReader( + FileResolver(File(".")), + ChainOptionsReader(), + ) + + val config = reader.readInternal(yaml.byteInputStream()) + + assertNotNull(config) + val upstream = config.upstreams[0] + assertEquals(1, upstream.customHeaders.size) + assertEquals("value", upstream.customHeaders["X-Header"]) + } +} diff --git a/src/test/kotlin/io/emeraldpay/dshackle/config/hot/CompatibleVersionsRulesTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/config/hot/CompatibleVersionsRulesTest.kt index 4af6e4e5e..31ee97c1c 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/config/hot/CompatibleVersionsRulesTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/config/hot/CompatibleVersionsRulesTest.kt @@ -10,17 +10,17 @@ class CompatibleVersionsRulesTest { val raw = """ rules: - client: "client1" - blacklist: + blacklist: - 1.0.0 - 1.0.1 - whitelist: + whitelist: - 1.0.2 - 1.0.3 - client: "client2" - blacklist: + blacklist: - 1.0.0 - 1.0.1 - whitelist: + whitelist: - 1.0.2 - 1.0.3 """.trimIndent() @@ -42,4 +42,44 @@ class CompatibleVersionsRulesTest { assertEquals("1.0.2", rules[1].whitelist!![0]) assertEquals("1.0.3", rules[1].whitelist!![1]) } + + @Test + fun `test parsing with networks`() { + val raw = """ + rules: + - client: "erigon" + networks: + - ethereum + blacklist: + - v2.40.0 + - 3.1.0 + - client: "reth" + blacklist: + - v1.4.0 + - v1.4.1 + - client: "reth" + networks: + - bsc + blacklist: + - "reth/v1.6.0-2a4968e/x86_64-unknown-linux-gnu" + """.trimIndent() + + val rules = Global.yamlMapper.readValue(raw, CompatibleVersionsRules::class.java)!!.rules + assertEquals(3, rules.size) + + // First rule: erigon with networks + assertEquals("erigon", rules[0].client) + assertEquals(listOf("ethereum"), rules[0].networks) + assertEquals(listOf("v2.40.0", "3.1.0"), rules[0].blacklist) + + // Second rule: reth without networks (applies to all) + assertEquals("reth", rules[1].client) + assertEquals(null, rules[1].networks) + assertEquals(listOf("v1.4.0", "v1.4.1"), rules[1].blacklist) + + // Third rule: reth with networks (bsc only) + assertEquals("reth", rules[2].client) + assertEquals(listOf("bsc"), rules[2].networks) + assertEquals(listOf("reth/v1.6.0-2a4968e/x86_64-unknown-linux-gnu"), rules[2].blacklist) + } } diff --git a/src/test/kotlin/io/emeraldpay/dshackle/config/reload/ReloadConfigTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/config/reload/ReloadConfigTest.kt index f825d7362..4c82b2d12 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/config/reload/ReloadConfigTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/config/reload/ReloadConfigTest.kt @@ -29,7 +29,6 @@ import org.mockito.kotlin.mock import org.mockito.kotlin.never import org.mockito.kotlin.verify import org.mockito.kotlin.whenever -import org.springframework.cloud.sleuth.Tracer import org.springframework.util.ResourceUtils import reactor.core.publisher.Flux import reactor.core.scheduler.Schedulers @@ -194,7 +193,7 @@ class ReloadConfigTest { ArrayList(), Caches.default(), Schedulers.boundedElastic(), - cs.makeCachingReaderBuilder(mock()), + cs.makeCachingReaderBuilder(), cs::localReaderBuilder, cs.subscriptionBuilder(Schedulers.boundedElastic()), ) diff --git a/src/test/kotlin/io/emeraldpay/dshackle/reader/RequestReaderFactoryTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/reader/RequestReaderFactoryTest.kt index 62fac3496..22bd2627a 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/reader/RequestReaderFactoryTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/reader/RequestReaderFactoryTest.kt @@ -4,12 +4,12 @@ import io.emeraldpay.dshackle.quorum.BroadcastQuorum import io.emeraldpay.dshackle.quorum.MaximumValueQuorum import io.emeraldpay.dshackle.upstream.Multistream import io.emeraldpay.dshackle.upstream.Selector +import io.emeraldpay.dshackle.upstream.signature.DisabledSigner import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.MethodSource import org.mockito.kotlin.mock -import org.springframework.cloud.sleuth.Tracer class RequestReaderFactoryTest { private val defaultFactory = RequestReaderFactory.Default() @@ -26,7 +26,6 @@ class RequestReaderFactoryTest { companion object { private val ms = mock() - private val tracer = mock() @JvmStatic fun data(): List { @@ -36,8 +35,7 @@ class RequestReaderFactoryTest { ms, Selector.UpstreamFilter(Selector.empty), MaximumValueQuorum(), - null, - tracer, + DisabledSigner(), ), ), Arguments.of( @@ -45,8 +43,7 @@ class RequestReaderFactoryTest { ms, Selector.UpstreamFilter(Selector.empty), BroadcastQuorum(), - null, - tracer, + DisabledSigner(), ), ), ) diff --git a/src/test/kotlin/io/emeraldpay/dshackle/rpc/NativeCallTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/rpc/NativeCallTest.kt index 95443a3b3..5c3a405f7 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/rpc/NativeCallTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/rpc/NativeCallTest.kt @@ -13,7 +13,6 @@ import org.junit.jupiter.api.Test import org.mockito.kotlin.doReturn import org.mockito.kotlin.mock import org.mockito.kotlin.spy -import org.springframework.cloud.sleuth.Tracer import reactor.core.publisher.Flux import reactor.core.publisher.Mono import reactor.test.StepVerifier @@ -29,7 +28,6 @@ class NativeCallTest { mock(), mock(), MainConfig(), - mock(), ), ) { on { nativeCallResult(request) } doReturn Flux.just( @@ -59,7 +57,6 @@ class NativeCallTest { mock(), mock(), MainConfig(), - mock(), ), ) { on { nativeCallResult(request) } doReturn Flux.just( @@ -95,7 +92,6 @@ class NativeCallTest { mock(), mock(), MainConfig(), - mock(), ), ) { on { nativeCallResult(request) } doReturn Flux.just( diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/UpstreamSettingsDetectorTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/UpstreamSettingsDetectorTest.kt new file mode 100644 index 000000000..ecdf0ce77 --- /dev/null +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/UpstreamSettingsDetectorTest.kt @@ -0,0 +1,51 @@ +package io.emeraldpay.dshackle.upstream + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class UpstreamSettingsDetectorTest { + + @Test + fun `parseLenientJson accepts plain JSON string`() { + val data = "\"Geth/v1.12.0/linux-amd64/go1.20.3\"".toByteArray() + val node = parseLenientJson(data) + assertTrue(node.isTextual) + assertEquals("Geth/v1.12.0/linux-amd64/go1.20.3", node.asText()) + } + + @Test + fun `parseLenientJson accepts JSON string with unescaped LF (Moca Tendermint)`() { + // Real-world Moca Tendermint web3_clientVersion result: + // "Version dev ()\nCompiled at using Go go1.23.11 (amd64)" + // where \n is a real line feed (CTRL-CHAR, code 10), not an escape sequence. + // Default Jackson rejects this with "Illegal unquoted character"; the lenient + // parser must accept it. + val raw = "Version dev ()\nCompiled at using Go go1.23.11 (amd64)" + val data = ("\"" + raw + "\"").toByteArray() + + val node = parseLenientJson(data) + + assertTrue(node.isTextual) + assertEquals(raw, node.asText()) + } + + @Test + fun `parseLenientJson accepts JSON string with unescaped CR and tab`() { + val raw = "Version dev ()\r\n\tCompiled with Go go1.23.11" + val data = ("\"" + raw + "\"").toByteArray() + + val node = parseLenientJson(data) + + assertTrue(node.isTextual) + assertEquals(raw, node.asText()) + } + + @Test + fun `parseLenientJson still parses normal JSON objects`() { + val data = """{"foo":"bar","n":42}""".toByteArray() + val node = parseLenientJson(data) + assertEquals("bar", node.get("foo").asText()) + assertEquals(42, node.get("n").asInt()) + } +} diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/avm/AvmChainSpecificTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/avm/AvmChainSpecificTest.kt new file mode 100644 index 000000000..2aaa86f57 --- /dev/null +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/avm/AvmChainSpecificTest.kt @@ -0,0 +1,93 @@ +package io.emeraldpay.dshackle.upstream.avm + +import io.emeraldpay.dshackle.reader.ChainReader +import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.ChainResponse +import io.emeraldpay.dshackle.upstream.UpstreamAvailability +import org.assertj.core.api.Assertions +import org.junit.jupiter.api.Test +import reactor.core.publisher.Mono + +val avmStatusSynced = """ + { + "last-round": 30000000, + "last-version": "https://github.com/algorandfoundation/specs/tree/somehash", + "next-version": "https://github.com/algorandfoundation/specs/tree/somehash", + "next-version-round": 30000001, + "next-version-supported": true, + "time-since-last-round": 1500000000, + "catchup-time": 0, + "last-catchpoint": "" + } +""".trimIndent() + +val avmStatusCatchingUp = """ + { + "last-round": 30000000, + "last-version": "https://github.com/algorandfoundation/specs/tree/somehash", + "next-version": "https://github.com/algorandfoundation/specs/tree/somehash", + "next-version-round": 30000001, + "next-version-supported": true, + "time-since-last-round": 1500000000, + "catchup-time": 1500000000, + "last-catchpoint": "30000000#QWERTYU" + } +""".trimIndent() + +val avmBlockHeader = """ + { + "block": { + "rnd": 30000000, + "ts": 1696802363, + "prev": "blk-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "seed": "c29tZXNlZWRieXRlc3RoYXRpczMyYnl0ZXNsb25nISEh", + "txn": "dHhucm9vdGhhc2h2YWx1ZWZvcnRlc3RpbmcxMjM0NTY=", + "gen": "mainnet-v1.0", + "proto": "https://github.com/algorandfoundation/specs/tree/somehash" + }, + "cert": {} + } +""".trimIndent() + +class AvmChainSpecificTest { + + @Test + fun parseBlockChainsThroughBlockEndpoint() { + val reader = object : ChainReader { + override fun read(key: ChainRequest): Mono { + Assertions.assertThat(key.method).isEqualTo("GET#/v2/blocks/30000000") + return Mono.just(ChainResponse(avmBlockHeader.toByteArray(), null)) + } + } + + val result = AvmChainSpecific.parseBlock( + avmStatusSynced.toByteArray(), + "upstream-1", + reader, + ).block()!! + + Assertions.assertThat(result.height).isEqualTo(30000000L) + Assertions.assertThat(result.upstreamId).isEqualTo("upstream-1") + Assertions.assertThat(result.timestamp.epochSecond).isEqualTo(1696802363L) + Assertions.assertThat(result.hash.toHex()).isNotEmpty() + Assertions.assertThat(result.parentHash?.toHex()).isNotEmpty() + } + + @Test + fun validateSyncedNode() { + Assertions.assertThat(AvmChainSpecific.validate(avmStatusSynced.toByteArray(), "test")) + .isEqualTo(UpstreamAvailability.OK) + } + + @Test + fun validateCatchingUpNode() { + Assertions.assertThat(AvmChainSpecific.validate(avmStatusCatchingUp.toByteArray(), "test")) + .isEqualTo(UpstreamAvailability.SYNCING) + } + + @Test + fun latestBlockRequestUsesStatusEndpoint() { + val request = AvmChainSpecific.latestBlockRequest() + Assertions.assertThat(request.method).isEqualTo("GET#/v2/status") + } +} diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/aztec/AztecChainSpecificTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/aztec/AztecChainSpecificTest.kt new file mode 100644 index 000000000..fa45433f2 --- /dev/null +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/aztec/AztecChainSpecificTest.kt @@ -0,0 +1,146 @@ +package io.emeraldpay.dshackle.upstream.aztec + +import io.emeraldpay.dshackle.reader.ChainReader +import io.emeraldpay.dshackle.upstream.ChainCallError +import io.emeraldpay.dshackle.upstream.ChainCallUpstreamException +import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.ChainResponse +import org.assertj.core.api.Assertions +import org.junit.jupiter.api.Test +import reactor.core.publisher.Mono +import java.util.concurrent.atomic.AtomicInteger + +// v5 (v5.0.0-rc.1) node_getChainTips response: proposed stays flat, the rest nested. +private val chainTipsResponse = """ + { + "proposed": {"number": 12345, "hash": "0xaaaa"}, + "checkpointed": {"block": {"number": 12340, "hash": "0xbbbb"}, "checkpoint": {"number": 100, "hash": "0x1111"}}, + "proven": {"block": {"number": 12330, "hash": "0xcccc"}, "checkpoint": {"number": 99, "hash": "0x2222"}}, + "finalized": {"block": {"number": 12320, "hash": "0xdddd"}, "checkpoint": {"number": 98, "hash": "0x3333"}} + } +""".trimIndent() + +// legacy node_getL2Tips response (v3-era flat proposed) +private val l2TipsResponse = """ + { + "proposed": {"number": 999, "hash": "0x0999"}, + "proven": {"number": 990, "hash": "0x0990"}, + "checkpointed": {"number": 980, "hash": "0x0980"} + } +""".trimIndent() + +// defensive: some versions nest proposed under .block +private val nestedProposedResponse = """ + { + "proposed": {"block": {"number": 777, "hash": "0x0777"}} + } +""".trimIndent() + +private fun methodNotFound(method: String) = + Mono.error( + ChainCallUpstreamException( + ChainResponse.NumberId(1), + ChainCallError(-32601, "Method not found: $method"), + ), + ) + +class AztecChainSpecificTest { + + @Test + fun latestBlockRequestUsesL2Tips() { + Assertions.assertThat(AztecChainSpecific.latestBlockRequest().method) + .isEqualTo("node_getL2Tips") + } + + @Test + fun parseBlockReadsFlatProposed() { + val result = AztecChainSpecific.parseBlock( + chainTipsResponse.toByteArray(), + "up-flat", + noopReader(), + ).block()!! + + Assertions.assertThat(result.height).isEqualTo(12345L) + Assertions.assertThat(result.hash.toHex()).contains("aaaa") + } + + @Test + fun parseBlockReadsNestedProposed() { + val result = AztecChainSpecific.parseBlock( + nestedProposedResponse.toByteArray(), + "up-nested", + noopReader(), + ).block()!! + + Assertions.assertThat(result.height).isEqualTo(777L) + Assertions.assertThat(result.hash.toHex()).contains("0777") + } + + @Test + fun getLatestBlockUsesL2TipsWhenAvailable() { + val calls = mutableListOf() + val reader = object : ChainReader { + override fun read(key: ChainRequest): Mono { + calls += key.method + return Mono.just(ChainResponse(l2TipsResponse.toByteArray(), null)) + } + } + + val result = AztecChainSpecific.getLatestBlock(reader, "up-legacy").block()!! + + Assertions.assertThat(result.height).isEqualTo(999L) + Assertions.assertThat(calls).containsExactly("node_getL2Tips") + } + + @Test + fun getLatestBlockFallsBackToChainTipsAndCaches() { + val calls = mutableListOf() + val reader = object : ChainReader { + override fun read(key: ChainRequest): Mono { + calls += key.method + return when (key.method) { + "node_getL2Tips" -> methodNotFound("node_getL2Tips") + "node_getChainTips" -> Mono.just(ChainResponse(chainTipsResponse.toByteArray(), null)) + else -> Mono.error(IllegalStateException("unexpected ${key.method}")) + } + } + } + + // first poll: probes legacy, falls back to v5 + val first = AztecChainSpecific.getLatestBlock(reader, "up-v5").block()!! + Assertions.assertThat(first.height).isEqualTo(12345L) + Assertions.assertThat(calls).containsExactly("node_getL2Tips", "node_getChainTips") + + // second poll: must hit the cached working method directly, no dead probe + calls.clear() + val second = AztecChainSpecific.getLatestBlock(reader, "up-v5").block()!! + Assertions.assertThat(second.height).isEqualTo(12345L) + Assertions.assertThat(calls).containsExactly("node_getChainTips") + } + + @Test + fun getLatestBlockDoesNotFallBackOnOtherErrors() { + val attempts = AtomicInteger(0) + val reader = object : ChainReader { + override fun read(key: ChainRequest): Mono { + attempts.incrementAndGet() + return Mono.error( + ChainCallUpstreamException( + ChainResponse.NumberId(1), + ChainCallError(-32000, "internal error"), + ), + ) + } + } + + val thrown = runCatching { AztecChainSpecific.getLatestBlock(reader, "up-err").block() } + Assertions.assertThat(thrown.isFailure).isTrue() + // only the primary method is attempted; no fallback probe on a non-method-not-found error + Assertions.assertThat(attempts.get()).isEqualTo(1) + } + + private fun noopReader() = object : ChainReader { + override fun read(key: ChainRequest): Mono = + Mono.error(IllegalStateException("not expected")) + } +} diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultAvmMethodsTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultAvmMethodsTest.kt new file mode 100644 index 000000000..8f6ed15c7 --- /dev/null +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/calls/DefaultAvmMethodsTest.kt @@ -0,0 +1,119 @@ +package io.emeraldpay.dshackle.upstream.calls + +import io.emeraldpay.dshackle.quorum.AlwaysQuorum +import io.emeraldpay.dshackle.quorum.BroadcastQuorum +import io.emeraldpay.dshackle.quorum.NotNullQuorum +import org.assertj.core.api.Assertions +import org.junit.jupiter.api.Test + +class DefaultAvmMethodsTest { + + private val methods = DefaultAvmMethods() + + @Test + fun rootLevelCommonEndpointsAreCallable() { + Assertions.assertThat(methods.isCallable("GET#/genesis")).isTrue() + Assertions.assertThat(methods.isCallable("GET#/health")).isTrue() + Assertions.assertThat(methods.isCallable("GET#/ready")).isTrue() + Assertions.assertThat(methods.isCallable("GET#/versions")).isTrue() + Assertions.assertThat(methods.isCallable("GET#/metrics")).isTrue() + } + + @Test + fun v2ReadMethodsAreCallable() { + Assertions.assertThat(methods.isCallable("GET#/v2/status")).isTrue() + Assertions.assertThat(methods.isCallable("GET#/v2/blocks/*")).isTrue() + Assertions.assertThat(methods.isCallable("GET#/v2/blocks/*/hash")).isTrue() + Assertions.assertThat(methods.isCallable("GET#/v2/blocks/*/txids")).isTrue() + Assertions.assertThat(methods.isCallable("GET#/v2/blocks/*/lightheader/proof")).isTrue() + Assertions.assertThat(methods.isCallable("GET#/v2/accounts/*")).isTrue() + Assertions.assertThat(methods.isCallable("GET#/v2/accounts/*/transactions/pending")).isTrue() + Assertions.assertThat(methods.isCallable("GET#/v2/transactions/pending")).isTrue() + } + + @Test + fun sendMethodsAreCallable() { + Assertions.assertThat(methods.isCallable("POST#/v2/transactions")).isTrue() + Assertions.assertThat(methods.isCallable("POST#/v2/transactions/async")).isTrue() + } + + @Test + fun spuriousAlgodEndpointsAreNotCallable() { + // These paths don't exist in algod's OpenAPI spec — regression guards. + Assertions.assertThat(methods.isCallable("GET#/v2/genesis")).isFalse() + Assertions.assertThat(methods.isCallable("GET#/v2/versions")).isFalse() + Assertions.assertThat(methods.isCallable("GET#/v2/health")).isFalse() + Assertions.assertThat(methods.isCallable("GET#/v2/ready")).isFalse() + Assertions.assertThat(methods.isCallable("GET#/v2/metrics")).isFalse() + Assertions.assertThat(methods.isCallable("GET#/v2/blocks/*/header")).isFalse() + Assertions.assertThat(methods.isCallable("GET#/v2/blocks/*/transactions")).isFalse() + Assertions.assertThat(methods.isCallable("GET#/v2/lightheader/*")).isFalse() + Assertions.assertThat(methods.isCallable("POST#/v2/transactions/dryrun")).isFalse() + } + + @Test + fun unknownMethodsAreNotCallable() { + Assertions.assertThat(methods.isCallable("GET#/eth/blockNumber")).isFalse() + Assertions.assertThat(methods.isCallable("algod_status")).isFalse() + Assertions.assertThat(methods.isCallable("DELETE#/v2/status")).isFalse() + } + + @Test + fun sendMethodsUseBroadcastQuorum() { + Assertions.assertThat(methods.createQuorumFor("POST#/v2/transactions")) + .isInstanceOf(BroadcastQuorum::class.java) + Assertions.assertThat(methods.createQuorumFor("POST#/v2/transactions/async")) + .isInstanceOf(BroadcastQuorum::class.java) + } + + @Test + fun listReadMethodsUseAlwaysQuorum() { + Assertions.assertThat(methods.createQuorumFor("GET#/v2/status")) + .isInstanceOf(AlwaysQuorum::class.java) + Assertions.assertThat(methods.createQuorumFor("GET#/v2/transactions/pending")) + .isInstanceOf(AlwaysQuorum::class.java) + Assertions.assertThat(methods.createQuorumFor("GET#/v2/ledger/supply")) + .isInstanceOf(AlwaysQuorum::class.java) + Assertions.assertThat(methods.createQuorumFor("GET#/genesis")) + .isInstanceOf(AlwaysQuorum::class.java) + } + + @Test + fun byIdLookupsUseNotNullQuorum() { + Assertions.assertThat(methods.createQuorumFor("GET#/v2/blocks/*")) + .isInstanceOf(NotNullQuorum::class.java) + Assertions.assertThat(methods.createQuorumFor("GET#/v2/blocks/*/hash")) + .isInstanceOf(NotNullQuorum::class.java) + Assertions.assertThat(methods.createQuorumFor("GET#/v2/accounts/*")) + .isInstanceOf(NotNullQuorum::class.java) + Assertions.assertThat(methods.createQuorumFor("GET#/v2/applications/*")) + .isInstanceOf(NotNullQuorum::class.java) + Assertions.assertThat(methods.createQuorumFor("GET#/v2/assets/*")) + .isInstanceOf(NotNullQuorum::class.java) + Assertions.assertThat(methods.createQuorumFor("GET#/v2/transactions/pending/*")) + .isInstanceOf(NotNullQuorum::class.java) + } + + @Test + fun noHardcodedMethods() { + Assertions.assertThat(methods.isHardcoded("GET#/v2/status")).isFalse() + Assertions.assertThat(methods.isHardcoded("GET#/genesis")).isFalse() + } + + @Test + fun defaultGroupReturnsAllSupported() { + Assertions.assertThat(methods.getGroupMethods("default")).isEqualTo(methods.getSupportedMethods()) + Assertions.assertThat(methods.getGroupMethods("unknown")).isEmpty() + } + + @Test + fun supportedMethodsIncludeCoreEndpoints() { + val supported = methods.getSupportedMethods() + Assertions.assertThat(supported).contains( + "GET#/v2/status", + "GET#/genesis", + "POST#/v2/transactions", + "POST#/v2/teal/compile", + ) + } +} diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/error/EthereumStateLowerBoundErrorHandlerTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/error/EthereumStateLowerBoundErrorHandlerTest.kt index 12ea47f9c..3d2ce0c2d 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/error/EthereumStateLowerBoundErrorHandlerTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/error/EthereumStateLowerBoundErrorHandlerTest.kt @@ -1,6 +1,7 @@ package io.emeraldpay.dshackle.upstream.error import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType import io.emeraldpay.dshackle.upstream.rpcclient.ListParams @@ -10,17 +11,18 @@ import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.Arguments.of import org.junit.jupiter.params.provider.MethodSource import org.mockito.Mockito.anyLong -import org.mockito.Mockito.mock import org.mockito.kotlin.any +import org.mockito.kotlin.mock import org.mockito.kotlin.never import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever class EthereumStateLowerBoundErrorHandlerTest { @ParameterizedTest @MethodSource("requests") fun `update lower bound`(request: ChainRequest) { - val upstream = mock() + val upstream = mockUpstreamWithHead(300_000_000L) val handler = EthereumStateLowerBoundErrorHandler handler.handle(upstream, request, "missing trie node d5648cc9aef48154159d53800f2f") @@ -28,6 +30,53 @@ class EthereumStateLowerBoundErrorHandlerTest { verify(upstream).updateLowerBound(213229736, LowerBoundType.STATE) } + @Test + fun `no update lower bound if parsed tag exceeds head`() { + val upstream = mockUpstreamWithHead(100_000_000L) + + EthereumStateLowerBoundErrorHandler.handle( + upstream, + ChainRequest("eth_getBalance", ListParams("0x343", "0xCB5A0A8")), + "missing trie node d5648cc9aef48154159d53800f2f", + ) + + verify(upstream, never()).updateLowerBound(anyLong(), any()) + } + + @Test + fun `no update lower bound if head height is null`() { + val upstream = mockUpstreamWithHead(null) + + EthereumStateLowerBoundErrorHandler.handle( + upstream, + ChainRequest("eth_getBalance", ListParams("0x343", "0xCB5A0A8")), + "missing trie node d5648cc9aef48154159d53800f2f", + ) + + verify(upstream, never()).updateLowerBound(anyLong(), any()) + } + + @Test + fun `no update lower bound for the Base prod incident value`() { + val upstream = mockUpstreamWithHead(30_000_000L) + + EthereumStateLowerBoundErrorHandler.handle( + upstream, + ChainRequest("eth_getBalance", ListParams("0x343", "0xD150C7F1")), + "missing trie node d5648cc9aef48154159d53800f2f", + ) + + verify(upstream, never()).updateLowerBound(anyLong(), any()) + } + + private fun mockUpstreamWithHead(height: Long?): Upstream { + val head = mock() + whenever(head.getCurrentHeight()).thenReturn(height) + val upstream = mock() + whenever(upstream.getHead()).thenReturn(head) + return upstream + } + @Test fun `no update lower bound if error is not about state`() { val upstream = mock() @@ -42,6 +91,20 @@ class EthereumStateLowerBoundErrorHandlerTest { verify(upstream, never()).updateLowerBound(anyLong(), any()) } + @Test + fun `no update lower bound if execution reverted`() { + val upstream = mock() + val request = ChainRequest( + "eth_call", + ListParams("0x343", "0xCB5A0A8"), + ) + val handler = EthereumStateLowerBoundErrorHandler + + handler.handle(upstream, request, "execution reverted: Fallback not supported") + + verify(upstream, never()).updateLowerBound(anyLong(), any()) + } + @Test fun `no update lower bound if there is a non-state method`() { val upstream = mock() diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/error/EthereumTraceLowerBoundErrorHandlerTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/error/EthereumTraceLowerBoundErrorHandlerTest.kt index 2d5d824af..170a402f4 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/error/EthereumTraceLowerBoundErrorHandlerTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/error/EthereumTraceLowerBoundErrorHandlerTest.kt @@ -1,6 +1,7 @@ package io.emeraldpay.dshackle.upstream.error import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType import io.emeraldpay.dshackle.upstream.rpcclient.ListParams @@ -9,15 +10,19 @@ import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.Arguments.of import org.junit.jupiter.params.provider.MethodSource -import org.mockito.Mockito.mock +import org.mockito.Mockito.anyLong +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.never import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever class EthereumTraceLowerBoundErrorHandlerTest { @ParameterizedTest @MethodSource("requests") fun `update lower bound`(request: ChainRequest) { - val upstream = mock() + val upstream = mockUpstreamWithHead(300_000_000L) val handler = EthereumTraceLowerBoundErrorHandler handler.handle(upstream, request, "missing trie node d5648cc9aef48154159d53800f2f") @@ -27,7 +32,7 @@ class EthereumTraceLowerBoundErrorHandlerTest { @Test fun `update lower bound base on regexp`() { - val upstream = mock() + val upstream = mockUpstreamWithHead(300_000_000L) val handler = EthereumTraceLowerBoundErrorHandler handler.handle(upstream, ChainRequest("trace_block", ListParams("0xCB5A0A8")), "block #1 not found") @@ -35,6 +40,40 @@ class EthereumTraceLowerBoundErrorHandlerTest { verify(upstream).updateLowerBound(213229736, LowerBoundType.TRACE) } + @Test + fun `no update lower bound if parsed tag exceeds head`() { + val upstream = mockUpstreamWithHead(100_000_000L) + + EthereumTraceLowerBoundErrorHandler.handle( + upstream, + ChainRequest("trace_block", ListParams("0xCB5A0A8")), + "missing trie node d5648cc9aef48154159d53800f2f", + ) + + verify(upstream, never()).updateLowerBound(anyLong(), any()) + } + + @Test + fun `no update lower bound if head height is null`() { + val upstream = mockUpstreamWithHead(null) + + EthereumTraceLowerBoundErrorHandler.handle( + upstream, + ChainRequest("trace_block", ListParams("0xCB5A0A8")), + "missing trie node d5648cc9aef48154159d53800f2f", + ) + + verify(upstream, never()).updateLowerBound(anyLong(), any()) + } + + private fun mockUpstreamWithHead(height: Long?): Upstream { + val head = mock() + whenever(head.getCurrentHeight()).thenReturn(height) + val upstream = mock() + whenever(upstream.getHead()).thenReturn(head) + return upstream + } + companion object { @JvmStatic fun requests(): List = diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/error/UpstreamErrorHandlerTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/error/UpstreamErrorHandlerTest.kt index 5504d12bf..d2da73d95 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/error/UpstreamErrorHandlerTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/error/UpstreamErrorHandlerTest.kt @@ -1,18 +1,23 @@ package io.emeraldpay.dshackle.upstream.error import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.Head import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import org.junit.jupiter.api.Test -import org.mockito.Mockito.mock +import org.mockito.kotlin.mock import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever class UpstreamErrorHandlerTest { @Test fun `use lower bound error handler`() { + val head = mock() + whenever(head.getCurrentHeight()).thenReturn(300_000_000L) val upstream = mock() + whenever(upstream.getHead()).thenReturn(head) val request = ChainRequest("eth_getCode", ListParams("0x343", "0xCB5A0A8")) val handler = UpstreamErrorHandler diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/ethereum/BasicEthUpstreamRpcMethodsDetectorTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/ethereum/BasicEthUpstreamRpcMethodsDetectorTest.kt index 182ca8b8f..2142824b8 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/ethereum/BasicEthUpstreamRpcMethodsDetectorTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/ethereum/BasicEthUpstreamRpcMethodsDetectorTest.kt @@ -84,6 +84,24 @@ class BasicEthUpstreamRpcMethodsDetectorTest { ChainCallError(32602, "missing value for required argument 0"), ), ) + on { + read(ChainRequest("trace_rawTransaction", ListParams(listOf()))) + } doReturn + Mono.just( + ChainResponse( + null, + ChainCallError(32602, "missing value for required argument 0"), + ), + ) + on { + read(ChainRequest("eth_getStorageValues", ListParams(listOf()))) + } doReturn + Mono.just( + ChainResponse( + null, + ChainCallError(32602, "missing value for required argument 0"), + ), + ) } val upstream = @@ -169,6 +187,24 @@ class BasicEthUpstreamRpcMethodsDetectorTest { ChainCallError(32602, "missing value for required argument 0"), ), ) + on { + read(ChainRequest("trace_rawTransaction", ListParams(listOf()))) + } doReturn + Mono.just( + ChainResponse( + null, + ChainCallError(32602, "missing value for required argument 0"), + ), + ) + on { + read(ChainRequest("eth_getStorageValues", ListParams(listOf()))) + } doReturn + Mono.just( + ChainResponse( + null, + ChainCallError(32602, "missing value for required argument 0"), + ), + ) } val upstream = @@ -180,10 +216,12 @@ class BasicEthUpstreamRpcMethodsDetectorTest { val detector = BasicEthUpstreamRpcMethodsDetector(upstream, config) Assertions.assertThat(detector.detectRpcMethods().block()).apply { isNotNull() - hasSize(6) + hasSize(8) containsEntry("eth_getBlockReceipts", true) containsEntry("trace_callMany", true) + containsEntry("trace_rawTransaction", true) containsEntry("eth_simulateV1", true) + containsEntry("eth_getStorageValues", true) containsEntry("debug_storageRangeAt", true) containsEntry("eth_getTdByNumber", true) containsEntry("eth_callBundle", true) @@ -259,6 +297,24 @@ class BasicEthUpstreamRpcMethodsDetectorTest { ChainCallError(32602, "missing value for required argument 0"), ), ) + on { + read(ChainRequest("trace_rawTransaction", ListParams(listOf()))) + } doReturn + Mono.just( + ChainResponse( + null, + ChainCallError(32602, "missing value for required argument 0"), + ), + ) + on { + read(ChainRequest("eth_getStorageValues", ListParams(listOf()))) + } doReturn + Mono.just( + ChainResponse( + null, + ChainCallError(32602, "missing value for required argument 0"), + ), + ) } val upstream = @@ -286,4 +342,109 @@ class BasicEthUpstreamRpcMethodsDetectorTest { containsEntry("eth_callBundle", true) } } + + @Test + fun `rpc_modules present but trace_rawTransaction and eth_getStorageValues not implemented`() { + val reader = + mock { + on { + read(ChainRequest("rpc_modules", ListParams())) + } doReturn + Mono.just( + ChainResponse( + """{"net": "1.0","debug": "1.0","txpool": "1.0","drpc": "1.0","erigon": "1.0","eth": "1.0","trace": "1.0"}""" + .toByteArray(), + null, + ), + ) + on { + read(ChainRequest("eth_getBlockReceipts", ListParams("latest"))) + } doReturn + Mono.just( + ChainResponse( + """[{"blockHash": "0xd12897f54acaa79f4824aa4f8e1d0f045b5568f5b942073555e9977202c5c474","blockNumber": "0x13c1108"}]""" + .toByteArray(), + null, + ), + ) + on { + read(ChainRequest("trace_callMany", ListParams(listOf(listOf())))) + } doReturn + Mono.just( + ChainResponse( + "[]".toByteArray(), + null, + ), + ) + on { + read(ChainRequest("trace_rawTransaction", ListParams(listOf()))) + } doReturn + Mono.just( + ChainResponse( + null, + ChainCallError(32601, "the method trace_rawTransaction does not exist/is not available"), + ), + ) + on { + read(ChainRequest("eth_simulateV1", ListParams(listOf()))) + } doReturn + Mono.just( + ChainResponse( + "[]".toByteArray(), + null, + ), + ) + on { + read(ChainRequest("eth_getStorageValues", ListParams(listOf()))) + } doReturn + Mono.just( + ChainResponse( + null, + ChainCallError(32601, "the method eth_getStorageValues does not exist/is not available"), + ), + ) + on { + read(ChainRequest("debug_storageRangeAt", ListParams(listOf()))) + } doReturn + Mono.just( + ChainResponse( + null, + ChainCallError(32602, "missing value for required argument 0"), + ), + ) + on { + read(ChainRequest("eth_getTdByNumber", ListParams(listOf()))) + } doReturn + Mono.just( + ChainResponse( + null, + ChainCallError(32602, "missing value for required argument 0"), + ), + ) + on { + read(ChainRequest("eth_callBundle", ListParams(listOf()))) + } doReturn + Mono.just( + ChainResponse( + null, + ChainCallError(32602, "missing value for required argument 0"), + ), + ) + } + + val upstream = + mock { + on { getIngressReader() } doReturn reader + on { getChain() } doReturn Chain.ETHEREUM__MAINNET + } + val config = mock> { } + val detector = BasicEthUpstreamRpcMethodsDetector(upstream, config) + Assertions.assertThat(detector.detectRpcMethods().block()).apply { + isNotNull() + containsEntry("trace_rawTransaction", false) + containsEntry("eth_getStorageValues", false) + containsEntry("trace_callMany", true) + containsEntry("eth_getBlockReceipts", true) + } + } } diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundReceiptsDetectorTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundReceiptsDetectorTest.kt new file mode 100644 index 000000000..f3bbdecab --- /dev/null +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundReceiptsDetectorTest.kt @@ -0,0 +1,56 @@ +package io.emeraldpay.dshackle.upstream.ethereum + +import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.config.ChainsConfig +import io.emeraldpay.dshackle.reader.ChainReader +import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.ChainResponse +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.lowerbound.GoldLowerBounds +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType +import io.emeraldpay.dshackle.upstream.lowerbound.NoopManualLowerBoundService +import io.emeraldpay.dshackle.upstream.rpcclient.ListParams +import org.junit.jupiter.api.Test +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.mock +import org.mockito.kotlin.verify +import reactor.core.publisher.Mono +import reactor.test.StepVerifier +import java.time.Duration + +class EthereumLowerBoundReceiptsDetectorTest { + + @Test + fun `archival bound if there is a gold bound`() { + val reader = mock { + on { read(ChainRequest("eth_getTransactionReceipt", ListParams("goldHash"))) } doReturn + Mono.just(ChainResponse("result".toByteArray(), null)) + } + val upstream = mock { + on { getChain() } doReturn Chain.POLYGON__MAINNET + on { getIngressReader() } doReturn reader + } + GoldLowerBounds.init( + listOf( + ChainsConfig.ChainConfig + .default() + .copy( + shortNames = listOf("polygon"), + goldLowerBounds = mapOf(ChainsConfig.LowerBoundType.RECEIPTS to ChainsConfig.GoldLowerBoundWithHash(10L, "goldHash")), + ), + ), + ) + + val txDetector = EthereumLowerBoundReceiptsDetector(upstream) + + StepVerifier.withVirtualTime { txDetector.detectLowerBound(NoopManualLowerBoundService()) } + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(15)) + .expectNextMatches { it.lowerBound == 1L && it.type == LowerBoundType.RECEIPTS } + .thenCancel() + .verify(Duration.ofSeconds(1)) + + verify(reader).read(ChainRequest("eth_getTransactionReceipt", ListParams("goldHash"))) + verify(upstream).getIngressReader() + } +} diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundTxDetectorTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundTxDetectorTest.kt new file mode 100644 index 000000000..61e38d572 --- /dev/null +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumLowerBoundTxDetectorTest.kt @@ -0,0 +1,56 @@ +package io.emeraldpay.dshackle.upstream.ethereum + +import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.config.ChainsConfig +import io.emeraldpay.dshackle.reader.ChainReader +import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.ChainResponse +import io.emeraldpay.dshackle.upstream.Upstream +import io.emeraldpay.dshackle.upstream.lowerbound.GoldLowerBounds +import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType +import io.emeraldpay.dshackle.upstream.lowerbound.NoopManualLowerBoundService +import io.emeraldpay.dshackle.upstream.rpcclient.ListParams +import org.junit.jupiter.api.Test +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.mock +import org.mockito.kotlin.verify +import reactor.core.publisher.Mono +import reactor.test.StepVerifier +import java.time.Duration + +class EthereumLowerBoundTxDetectorTest { + + @Test + fun `archival bound if there is a gold bound`() { + val reader = mock { + on { read(ChainRequest("eth_getTransactionByHash", ListParams("goldHash"))) } doReturn + Mono.just(ChainResponse("result".toByteArray(), null)) + } + val upstream = mock { + on { getChain() } doReturn Chain.POLYGON__MAINNET + on { getIngressReader() } doReturn reader + } + GoldLowerBounds.init( + listOf( + ChainsConfig.ChainConfig + .default() + .copy( + shortNames = listOf("polygon"), + goldLowerBounds = mapOf(ChainsConfig.LowerBoundType.TX to ChainsConfig.GoldLowerBoundWithHash(10L, "goldHash")), + ), + ), + ) + + val txDetector = EthereumLowerBoundTxDetector(upstream) + + StepVerifier.withVirtualTime { txDetector.detectLowerBound(NoopManualLowerBoundService()) } + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(15)) + .expectNextMatches { it.lowerBound == 1L && it.type == LowerBoundType.TX } + .thenCancel() + .verify(Duration.ofSeconds(1)) + + verify(reader).read(ChainRequest("eth_getTransactionByHash", ListParams("goldHash"))) + verify(upstream).getIngressReader() + } +} diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/ethereum/PendingTransactionValidatorTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/ethereum/PendingTransactionValidatorTest.kt new file mode 100644 index 000000000..6568f8cd6 --- /dev/null +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/ethereum/PendingTransactionValidatorTest.kt @@ -0,0 +1,211 @@ +package io.emeraldpay.dshackle.upstream.ethereum + +import io.emeraldpay.dshackle.reader.ChainReader +import io.emeraldpay.dshackle.upstream.ChainCallError +import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.ChainResponse +import io.emeraldpay.dshackle.upstream.rpcclient.ListParams +import org.junit.jupiter.api.Test +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.mock +import reactor.core.publisher.Mono +import reactor.test.StepVerifier +import java.time.Duration + +class PendingTransactionValidatorTest { + + @Test + fun `noop validator always emits true`() { + val validator = NoopPendingTransactionValidator() + + StepVerifier.create(validator.pendingTxExists()) + .expectNext(true) + .expectComplete() + .verify(Duration.ofSeconds(3)) + } + + @Test + fun `emits true when pending plus queued exceeds limit`() { + val reader = mockReader( + response = txpoolContent(pendingAddresses = 6, queuedAddresses = 0), + ) + val validator = PendingTransactionValidatorImpl( + upstreamId = "test-upstream", + directReader = reader, + interval = Duration.ofSeconds(30), + txLimit = 5, + ) + + StepVerifier.withVirtualTime { validator.pendingTxExists() } + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(15)) + .expectNext(true) + .thenCancel() + .verify(Duration.ofSeconds(3)) + } + + @Test + fun `emits true when pending plus queued is at limit`() { + val reader = mockReader( + response = txpoolContent(pendingAddresses = 3, queuedAddresses = 2), + ) + val validator = PendingTransactionValidatorImpl( + upstreamId = "test-upstream", + directReader = reader, + interval = Duration.ofSeconds(30), + txLimit = 5, + ) + + StepVerifier.withVirtualTime { validator.pendingTxExists() } + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(15)) + .expectNext(true) + .thenCancel() + .verify(Duration.ofSeconds(3)) + } + + @Test + fun `counts both pending and queued buckets`() { + // 3 pending + 4 queued = 7 > limit of 5 + val reader = mockReader( + response = txpoolContent(pendingAddresses = 3, queuedAddresses = 4), + ) + val validator = PendingTransactionValidatorImpl( + upstreamId = "test-upstream", + directReader = reader, + interval = Duration.ofSeconds(30), + txLimit = 5, + ) + + StepVerifier.withVirtualTime { validator.pendingTxExists() } + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(15)) + .expectNext(true) + .thenCancel() + .verify(Duration.ofSeconds(3)) + } + + @Test + fun `handles missing pending field`() { + val reader = mockReader(response = """{"queued": {"0xaa": {}, "0xbb": {}}}""") + val validator = PendingTransactionValidatorImpl( + upstreamId = "test-upstream", + directReader = reader, + interval = Duration.ofSeconds(30), + txLimit = 5, + ) + + // Only 2 queued (no pending node) -> below limit -> false + StepVerifier.withVirtualTime { validator.pendingTxExists() } + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(15)) + .expectNext(false) + .thenCancel() + .verify(Duration.ofSeconds(3)) + } + + @Test + fun `handles missing queued field`() { + val reader = mockReader(response = """{"pending": {"0xaa": {}, "0xbb": {}, "0xcc": {}}}""") + val validator = PendingTransactionValidatorImpl( + upstreamId = "test-upstream", + directReader = reader, + interval = Duration.ofSeconds(30), + txLimit = 2, + ) + + // 3 pending (no queued node) > limit 2 -> true + StepVerifier.withVirtualTime { validator.pendingTxExists() } + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(15)) + .expectNext(true) + .thenCancel() + .verify(Duration.ofSeconds(3)) + } + + @Test + fun `emits false on rpc error`() { + val reader = mock { + on { read(ChainRequest("txpool_content", ListParams())) } doReturn + Mono.just(ChainResponse(null, ChainCallError(-32000, "method not supported"))) + } + val validator = PendingTransactionValidatorImpl( + upstreamId = "test-upstream", + directReader = reader, + interval = Duration.ofSeconds(30), + txLimit = 5, + ) + + StepVerifier.withVirtualTime { validator.pendingTxExists() } + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(15)) + .expectNext(false) + .thenCancel() + .verify(Duration.ofSeconds(3)) + } + + @Test + fun `continues polling after an error response`() { + val errorResponse = Mono.just(ChainResponse(null, ChainCallError(-32000, "method not supported"))) + val okResponse = Mono.just( + ChainResponse( + txpoolContent(pendingAddresses = 10, queuedAddresses = 0).toByteArray(), + null, + ), + ) + val reader = mock { + on { read(ChainRequest("txpool_content", ListParams())) } doReturn errorResponse doReturn okResponse + } + val validator = PendingTransactionValidatorImpl( + upstreamId = "test-upstream", + directReader = reader, + interval = Duration.ofSeconds(30), + txLimit = 5, + ) + + StepVerifier.withVirtualTime { validator.pendingTxExists() } + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(15)) + .expectNext(false) // first poll: error -> false + .expectNoEvent(Duration.ofSeconds(30)) + .expectNext(true) // second poll: success + .thenCancel() + .verify(Duration.ofSeconds(3)) + } + + @Test + fun `polls at configured interval`() { + val reader = mockReader( + response = txpoolContent(pendingAddresses = 10, queuedAddresses = 0), + ) + val validator = PendingTransactionValidatorImpl( + upstreamId = "test-upstream", + directReader = reader, + interval = Duration.ofSeconds(30), + txLimit = 5, + ) + + StepVerifier.withVirtualTime { validator.pendingTxExists() } + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(15)) + .expectNext(true) + .expectNoEvent(Duration.ofSeconds(30)) + .expectNext(true) + .expectNoEvent(Duration.ofSeconds(30)) + .expectNext(true) + .thenCancel() + .verify(Duration.ofSeconds(3)) + } + + private fun mockReader(response: String): ChainReader = + mock { + on { read(ChainRequest("txpool_content", ListParams())) } doReturn + Mono.just(ChainResponse(response.toByteArray(), null)) + } + + private fun txpoolContent(pendingAddresses: Int, queuedAddresses: Int): String { + val pending = (0 until pendingAddresses).joinToString(",") { """"0xp$it": {}""" } + val queued = (0 until queuedAddresses).joinToString(",") { """"0xq$it": {}""" } + return """{"pending": {$pending}, "queued": {$queued}}""" + } +} diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericRpcHeadTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericRpcHeadTest.kt index 225111009..2bb097212 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericRpcHeadTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericRpcHeadTest.kt @@ -133,7 +133,7 @@ class GenericRpcHeadTest { "id", BlockValidator.ALWAYS_VALID, upstream, - Schedulers.boundedElastic(), + Schedulers.immediate(), EthereumChainSpecific, Duration.ofSeconds(5), ) diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericSubscriptionConnectTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericSubscriptionConnectTest.kt index f8b048169..a7af6e938 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericSubscriptionConnectTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/generic/GenericSubscriptionConnectTest.kt @@ -6,13 +6,16 @@ import io.emeraldpay.dshackle.upstream.ethereum.WsSubscriptions import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import org.junit.jupiter.api.Test import org.mockito.Mockito.verify +import org.mockito.kotlin.any import org.mockito.kotlin.doReturn import org.mockito.kotlin.mock +import org.mockito.kotlin.times import reactor.core.publisher.Flux import reactor.core.publisher.Mono import reactor.test.StepVerifier import reactor.util.function.Tuples import java.time.Duration +import java.util.concurrent.TimeoutException import java.util.concurrent.atomic.AtomicReference class GenericSubscriptionConnectTest { @@ -36,4 +39,70 @@ class GenericSubscriptionConnectTest { verify(ws).subscribe(ChainRequest(topic, ListParams(param))) } + + @Test + fun `emits TimeoutException when upstream goes silent past idle timeout`() { + // Simulates the Solana-like failure: WS subscription is established, but upstream stops + // delivering events without closing the connection. The Flux must surface a TimeoutException + // so the surrounding DurableFlux re-subscribes. + val param: List = listOf("all") + val topic = "slotSubscribe" + val ws = mock { + on { subscribe(ChainRequest(topic, ListParams(param))) } doReturn + WsSubscriptions.SubscribeData( + Mono.just(Tuples.of("sub-id-1", Flux.never())), + "conn-1", + AtomicReference("sub-id-1"), + ) + } + + val genericSubscriptionConnect = GenericSubscriptionConnect(Chain.SOLANA__MAINNET, ws, topic, param, "") + + StepVerifier.withVirtualTime { genericSubscriptionConnect.createConnection() } + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(84)) + .thenAwait(Duration.ofSeconds(2)) + .expectError(TimeoutException::class.java) + .verify(Duration.ofSeconds(5)) + } + + @Test + fun `idle timeout triggers resubscribe via DurableFlux retry`() { + // End-to-end on the GenericSubscriptionConnect: when the first subscription stalls, + // DurableFlux must call createConnection() again, which issues a fresh `subscribe` RPC. + val param: List = listOf("all") + val topic = "slotSubscribe" + + val secondAttempt = WsSubscriptions.SubscribeData( + Mono.just(Tuples.of("sub-id-2", Flux.just("recovered".toByteArray()))), + "conn-1", + AtomicReference("sub-id-2"), + ) + val firstAttempt = WsSubscriptions.SubscribeData( + Mono.just(Tuples.of("sub-id-1", Flux.never())), + "conn-1", + AtomicReference("sub-id-1"), + ) + + val ws = mock { + on { subscribe(ChainRequest(topic, ListParams(param))) } + .doReturn(firstAttempt, secondAttempt) + on { unsubscribe(any()) } doReturn Mono.empty() + } + + val genericSubscriptionConnect = GenericSubscriptionConnect(Chain.SOLANA__MAINNET, ws, topic, param, "slotUnsubscribe") + + // GenericPersistentConnect wraps createConnection() with DurableFlux + SharedFluxHolder. + // The first subscription stalls (Flux.never) → timeout (85s) → TimeoutException → + // DurableFlux schedules retry → second subscription delivers "recovered". + StepVerifier.withVirtualTime { genericSubscriptionConnect.connect(io.emeraldpay.dshackle.upstream.Selector.empty) } + .expectSubscription() + .thenAwait(Duration.ofSeconds(90)) + .expectNextMatches { it is ByteArray && String(it) == "recovered" } + .thenCancel() + .verify(Duration.ofSeconds(5)) + + // Verifies we re-subscribed (= sent slotSubscribe again on the same WS). + verify(ws, times(2)).subscribe(ChainRequest(topic, ListParams(param))) + } } diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericConnectorFactoryCreatorTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericConnectorFactoryCreatorTest.kt index d79b7e1e3..b23271067 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericConnectorFactoryCreatorTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/generic/connectors/GenericConnectorFactoryCreatorTest.kt @@ -19,6 +19,7 @@ import org.junit.jupiter.params.provider.MethodSource import org.mockito.Mockito.mockConstruction import org.mockito.kotlin.doReturn import org.mockito.kotlin.mock +import reactor.core.scheduler.Schedulers import reactor.core.scheduler.Schedulers.immediate import java.io.File import java.net.URI @@ -39,6 +40,8 @@ class GenericConnectorFactoryCreatorTest { immediate(), immediate(), MonitoringConfig.default(), + Schedulers.boundedElastic(), + Schedulers.boundedElastic(), ) var args: List<*>? = null diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/kadena/KadenaLowerBoundStateDetectorTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/kadena/KadenaLowerBoundStateDetectorTest.kt index 6217fbc80..5ba02c467 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/kadena/KadenaLowerBoundStateDetectorTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/kadena/KadenaLowerBoundStateDetectorTest.kt @@ -3,6 +3,7 @@ package io.emeraldpay.dshackle.upstream.kadena import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType +import io.emeraldpay.dshackle.upstream.lowerbound.NoopManualLowerBoundService import org.junit.jupiter.api.Test import reactor.test.StepVerifier import java.time.Duration @@ -13,7 +14,7 @@ class KadenaLowerBoundStateDetectorTest { fun `kadena lower block is 1`() { val detector = KadenaLowerBoundStateDetector(Chain.KADENA__MAINNET) - StepVerifier.withVirtualTime { detector.detectLowerBound() } + StepVerifier.withVirtualTime { detector.detectLowerBound(NoopManualLowerBoundService()) } .expectSubscription() .expectNoEvent(Duration.ofSeconds(15)) .expectNext(LowerBoundData(1, LowerBoundType.STATE)) diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/BaseManualLowerBoundServiceTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/BaseManualLowerBoundServiceTest.kt new file mode 100644 index 000000000..ef4fc11ba --- /dev/null +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/BaseManualLowerBoundServiceTest.kt @@ -0,0 +1,176 @@ +package io.emeraldpay.dshackle.upstream.lowerbound + +import io.emeraldpay.dshackle.config.UpstreamsConfig +import io.emeraldpay.dshackle.upstream.Head +import io.emeraldpay.dshackle.upstream.Upstream +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.Arguments +import org.junit.jupiter.params.provider.MethodSource +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.mock + +class BaseManualLowerBoundServiceTest { + + @Test + fun `no manual settings then return nothing`() { + val upstream = mock() + val service = BaseManualLowerBoundService(upstream, emptyMap()) + + assertThat(service.manualBoundTypes()).isEmpty() + LowerBoundType.entries + .forEach { + assertThat(service.manualLowerBound(it)).isNull() + assertThat(service.hasManualBound(it)).isFalse + } + } + + @ParameterizedTest + @MethodSource("headProviderData") + fun `test head provider - canHandle`( + upstream: Upstream, + setting: UpstreamsConfig.ManualBoundSetting, + expected: Boolean, + ) { + val provider = HeadManualLowerBoundProvider(upstream, setting) + + assertThat(provider.canHandle()).isEqualTo(expected) + } + + @ParameterizedTest + @MethodSource("fixedProviderData") + fun `test fixed provider - canHandle`( + setting: UpstreamsConfig.ManualBoundSetting, + expected: Boolean, + ) { + val provider = FixedManualLowerBoundProvider(setting) + + assertThat(provider.canHandle()).isEqualTo(expected) + } + + @Test + fun `get value from head provider`() { + val upstream = mock { + val head = mock { + on { getCurrentHeight() } doReturn 100 + } + on { getHead() } doReturn head + } + val provider = HeadManualLowerBoundProvider(upstream, UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.HEAD, -45)) + + assertThat(provider.getManualLowerBound()).isEqualTo(55) + } + + @Test + fun `get value from fixed provider`() { + val provider = FixedManualLowerBoundProvider(UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.FIXED, 200)) + + assertThat(provider.getManualLowerBound()).isEqualTo(200) + } + + @Test + fun `no supported manual type then null otherwise get a value`() { + val upstream = mock() + val settings = mapOf( + LowerBoundType.STATE to UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.FIXED, 100), + ) + val service = BaseManualLowerBoundService(upstream, settings) + + assertThat(service.manualLowerBound(LowerBoundType.STATE)) + .usingRecursiveComparison() + .ignoringFields("timestamp") + .isEqualTo(LowerBoundData(100, LowerBoundType.STATE)) + assertThat(service.manualBoundTypes()).containsExactly(LowerBoundType.STATE) + assertThat(service.hasManualBound(LowerBoundType.STATE)).isTrue() + LowerBoundType.entries + .filter { it != LowerBoundType.STATE } + .forEach { + assertThat(service.manualLowerBound(it)).isNull() + assertThat(service.hasManualBound(it)).isFalse + } + } + + @Test + fun `can not be handled then null`() { + val upstream = mock() + val settings = mapOf( + LowerBoundType.STATE to UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.FIXED, -100), + ) + val service = BaseManualLowerBoundService(upstream, settings) + + assertThat(service.manualLowerBound(LowerBoundType.STATE)).isNull() + assertThat(service.manualBoundTypes()).containsExactly(LowerBoundType.STATE) + assertThat(service.hasManualBound(LowerBoundType.STATE)).isTrue() + LowerBoundType.entries + .filter { it != LowerBoundType.STATE } + .forEach { + assertThat(service.manualLowerBound(it)).isNull() + assertThat(service.hasManualBound(it)).isFalse + } + } + + companion object { + @JvmStatic + fun fixedProviderData(): List = + listOf( + Arguments.of( + UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.HEAD, 1), + false, + ), + Arguments.of( + UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.FIXED, -121), + false, + ), + Arguments.of( + UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.FIXED, 55), + true, + ), + ) + + @JvmStatic + fun headProviderData(): List = + listOf( + Arguments.of( + mock(), + UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.FIXED, 1), + false, + ), + Arguments.of( + mock(), + UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.HEAD, 1241), + false, + ), + Arguments.of( + mock { + val head = mock { + on { getCurrentHeight() } doReturn null + } + on { getHead() } doReturn head + }, + UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.HEAD, -100), + false, + ), + Arguments.of( + mock { + val head = mock { + on { getCurrentHeight() } doReturn 20 + } + on { getHead() } doReturn head + }, + UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.HEAD, -100), + false, + ), + Arguments.of( + mock { + val head = mock { + on { getCurrentHeight() } doReturn 150 + } + on { getHead() } doReturn head + }, + UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.HEAD, -100), + true, + ), + ) + } +} diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/GoldLowerBoundsTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/GoldLowerBoundsTest.kt new file mode 100644 index 000000000..12d5faa87 --- /dev/null +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/GoldLowerBoundsTest.kt @@ -0,0 +1,63 @@ +package io.emeraldpay.dshackle.upstream.lowerbound + +import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.config.ChainsConfig +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +class GoldLowerBoundsTest { + + @Test + fun `init with all bounds`() { + val cfg = ChainsConfig.ChainConfig.default() + .copy( + shortNames = listOf("polygon"), + goldLowerBounds = ChainsConfig.LowerBoundType.entries + .associateWith { + if (it == ChainsConfig.LowerBoundType.TX || it == ChainsConfig.LowerBoundType.RECEIPTS) { + ChainsConfig.GoldLowerBoundWithHash(5L, "hash_$it") + } else { + ChainsConfig.GoldLowerBound(10L) + } + }, + ) + + GoldLowerBounds.init(listOf(cfg)) + + LowerBoundType.entries + .filter { it != LowerBoundType.UNKNOWN } + .forEach { + val bound = GoldLowerBounds.getBound(Chain.POLYGON__MAINNET, it) + + assertThat(bound).isNotNull + + if (it == LowerBoundType.TX || it == LowerBoundType.RECEIPTS) { + assertThat(bound) + .usingRecursiveComparison() + .isEqualTo(ChainsConfig.GoldLowerBoundWithHash(5L, "hash_$it")) + } else { + assertThat(bound) + .usingRecursiveComparison() + .isEqualTo(ChainsConfig.GoldLowerBound(10L)) + } + } + } + + @Test + fun `no gold bound`() { + val cfg = ChainsConfig.ChainConfig.default() + .copy( + shortNames = listOf("polygon"), + ) + + GoldLowerBounds.init(listOf(cfg)) + + LowerBoundType.entries + .filter { it != LowerBoundType.UNKNOWN } + .forEach { + val bound = GoldLowerBounds.getBound(Chain.POLYGON__MAINNET, it) + + assertThat(bound).isNull() + } + } +} diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundDetectorTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundDetectorTest.kt new file mode 100644 index 000000000..9768f07a0 --- /dev/null +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/LowerBoundDetectorTest.kt @@ -0,0 +1,95 @@ +package io.emeraldpay.dshackle.upstream.lowerbound + +import io.emeraldpay.dshackle.Chain +import org.junit.jupiter.api.Test +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.mock +import reactor.core.publisher.Flux +import reactor.test.StepVerifier +import java.time.Duration + +class LowerBoundDetectorTest { + + @Test + fun `receive only manual bounds`() { + val detector = TestLowerBoundDetector() + val manualBoundService = mock { + on { manualBoundTypes() } doReturn setOf(LowerBoundType.STATE, LowerBoundType.BLOCK) + on { hasManualBound(LowerBoundType.STATE) } doReturn true + on { hasManualBound(LowerBoundType.BLOCK) } doReturn true + on { manualLowerBound(LowerBoundType.STATE) } doReturn LowerBoundData(50L, 1000, LowerBoundType.STATE) + on { manualLowerBound(LowerBoundType.BLOCK) } doReturn LowerBoundData(80L, 1000, LowerBoundType.BLOCK) + } + + StepVerifier.withVirtualTime { detector.detectLowerBound(manualBoundService) } + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(15)) + .expectNextSequence( + setOf( + LowerBoundData(50L, 1000, LowerBoundType.STATE), + LowerBoundData(80L, 1000, LowerBoundType.BLOCK), + ), + ) + .thenCancel() + .verify(Duration.ofSeconds(3)) + } + + @Test + fun `receive one manual and one calculated bounds`() { + val detector = TestLowerBoundDetector() + val manualBoundService = mock { + on { manualBoundTypes() } doReturn setOf(LowerBoundType.STATE) + on { hasManualBound(LowerBoundType.STATE) } doReturn true + on { hasManualBound(LowerBoundType.BLOCK) } doReturn false + on { manualLowerBound(LowerBoundType.STATE) } doReturn LowerBoundData(50L, 1000, LowerBoundType.STATE) + on { manualLowerBound(LowerBoundType.BLOCK) } doReturn null + } + + StepVerifier.withVirtualTime { detector.detectLowerBound(manualBoundService) } + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(15)) + .expectNextSequence( + setOf( + LowerBoundData(5000L, 1000, LowerBoundType.BLOCK), + LowerBoundData(50L, 1000, LowerBoundType.STATE), + ), + ) + .thenCancel() + .verify(Duration.ofSeconds(3)) + } + + @Test + fun `receive only calculated bounds`() { + val detector = TestLowerBoundDetector() + val manualBoundService = BaseManualLowerBoundService(mock(), emptyMap()) + + StepVerifier.withVirtualTime { detector.detectLowerBound(manualBoundService) } + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(15)) + .expectNextSequence( + setOf( + LowerBoundData(1000L, 1000, LowerBoundType.STATE), + LowerBoundData(5000L, 1000, LowerBoundType.BLOCK), + ), + ) + .thenCancel() + .verify(Duration.ofSeconds(3)) + } +} + +private class TestLowerBoundDetector : LowerBoundDetector(Chain.CORE__MAINNET) { + override fun period(): Long { + return 1 + } + + override fun internalDetectLowerBound(): Flux { + return Flux.just( + LowerBoundData(1000L, 1000, LowerBoundType.STATE), + LowerBoundData(5000L, 1000, LowerBoundType.BLOCK), + ) + } + + override fun types(): Set { + return setOf(LowerBoundType.STATE, LowerBoundType.BLOCK) + } +} diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/RecursiveLowerBoundServiceTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/RecursiveLowerBoundServiceTest.kt index a5fd2bc4b..bbe29ad24 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/RecursiveLowerBoundServiceTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/lowerbound/RecursiveLowerBoundServiceTest.kt @@ -13,6 +13,7 @@ import io.emeraldpay.dshackle.upstream.ethereum.ZERO_ADDRESS import io.emeraldpay.dshackle.upstream.polkadot.PolkadotLowerBoundService import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments @@ -247,6 +248,12 @@ class RecursiveLowerBoundServiceTest { private const val STATE_CHECKER_CALL_DATA = "0x1eaf190c" private const val STATE_CHECKER_BYTECODE = "0x6080604052348015600e575f5ffd5b50600436106026575f3560e01c80631eaf190c14602a575b5f5ffd5b60306044565b604051603b91906078565b60405180910390f35b5f5f73ffffffffffffffffffffffffffffffffffffffff1631905090565b5f819050919050565b6072816062565b82525050565b5f60208201905060895f830184606b565b9291505056fea2646970667358221220251f5b4d2ed1abe77f66fde198a57ada08562dc3b0afbc6bac0261d1bf516b5d64736f6c634300081e0033" + @BeforeAll + @JvmStatic + fun init() { + GoldLowerBounds.init(emptyList()) + } + @JvmStatic fun detectorsFirstBlock(): List = listOf( Arguments.of( diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/ripple/RippleChainSpecificTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/ripple/RippleChainSpecificTest.kt new file mode 100644 index 000000000..343aa84c0 --- /dev/null +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/ripple/RippleChainSpecificTest.kt @@ -0,0 +1,168 @@ +package io.emeraldpay.dshackle.upstream.ripple + +import io.emeraldpay.dshackle.Chain +import io.emeraldpay.dshackle.data.BlockId +import io.emeraldpay.dshackle.reader.ChainReader +import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.ChainResponse +import io.emeraldpay.dshackle.upstream.UpstreamAvailability +import io.emeraldpay.dshackle.upstream.ValidateUpstreamSettingsResult +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import reactor.core.publisher.Mono +import java.time.Instant + +// ledger_closed response format +val ledgerClosedResponse = """ +{ + "ledger_hash": "17ACB57A0F73B5160713E81FE72B2AC9F6064541004E272BD09F257D57C30C02", + "ledger_index": 6643099 +} +""".trimIndent() + +// ledgerClosed WebSocket subscription event +val ledgerClosedEvent = """ +{ + "type": "ledgerClosed", + "fee_base": 10, + "fee_ref": 10, + "ledger_hash": "17ACB57A0F73B5160713E81FE72B2AC9F6064541004E272BD09F257D57C30C02", + "ledger_index": 6643099, + "ledger_time": 780804221, + "reserve_base": 10000000, + "reserve_inc": 2000000, + "txn_count": 5, + "validated_ledgers": "6643000-6643099" +} +""".trimIndent() + +// server_state response for validation +val serverStateOk = """ +{ + "state": { + "build_version": "1.12.0", + "complete_ledgers": "32570-6643099", + "io_latency_ms": 1, + "jq_trans_overflow": "0", + "last_close": { + "converge_time": 2000, + "proposers": 34 + }, + "load_base": 256, + "load_factor": 256, + "load_factor_fee_escalation": 256, + "load_factor_fee_queue": 256, + "load_factor_fee_reference": 256, + "load_factor_server": 256, + "network_id": 0, + "peers": 21, + "ports": [], + "server_state": "full", + "time": "2024-Jan-01 00:00:00", + "uptime": 123456, + "validated_ledger": { + "base_fee": 10, + "close_time": 780804221, + "hash": "17ACB57A0F73B5160713E81FE72B2AC9F6064541004E272BD09F257D57C30C02", + "reserve_base": 10000000, + "reserve_inc": 2000000, + "seq": 6643099 + }, + "validation_quorum": 28 + } +} +""".trimIndent() + +val serverStateSyncing = """ +{ + "state": { + "build_version": "1.12.0", + "complete_ledgers": "32570-6643099", + "io_latency_ms": 1, + "jq_trans_overflow": "0", + "last_close": { + "converge_time": 2000, + "proposers": 34 + }, + "load_base": 256, + "load_factor": 256, + "load_factor_fee_escalation": 256, + "load_factor_fee_queue": 256, + "load_factor_fee_reference": 256, + "load_factor_server": 256, + "network_id": 0, + "peers": 21, + "ports": [], + "server_state": "connected", + "time": "2024-Jan-01 00:00:00", + "uptime": 123456, + "validated_ledger": { + "base_fee": 10, + "close_time": 780804221, + "hash": "17ACB57A0F73B5160713E81FE72B2AC9F6064541004E272BD09F257D57C30C02", + "reserve_base": 10000000, + "reserve_inc": 2000000, + "seq": 6643099 + }, + "validation_quorum": 28 + } +} +""".trimIndent() + +class RippleChainSpecificTest { + + private val dummyReader = object : ChainReader { + override fun read(key: ChainRequest): Mono = Mono.empty() + } + + @Test + fun `parseBlock with ledger_closed format`() { + val result = RippleChainSpecific.parseBlock( + ledgerClosedResponse.toByteArray(), + "test-upstream", + dummyReader, + ).block()!! + + assertThat(result.height).isEqualTo(6643099) + assertThat(result.hash) + .isEqualTo(BlockId.from("17ACB57A0F73B5160713E81FE72B2AC9F6064541004E272BD09F257D57C30C02")) + assertThat(result.upstreamId).isEqualTo("test-upstream") + assertThat(result.parentHash).isNull() + } + + @Test + fun `getFromHeader with ledgerClosed WS event`() { + val result = RippleChainSpecific.getFromHeader( + ledgerClosedEvent.toByteArray(), + "test-upstream", + dummyReader, + ).block()!! + + assertThat(result.height).isEqualTo(6643099) + assertThat(result.hash) + .isEqualTo(BlockId.from("17ACB57A0F73B5160713E81FE72B2AC9F6064541004E272BD09F257D57C30C02")) + assertThat(result.upstreamId).isEqualTo("test-upstream") + assertThat(result.parentHash).isNull() + // Ripple epoch (2000-01-01) + 780804221 seconds = expected timestamp + assertThat(result.timestamp).isEqualTo(Instant.ofEpochSecond(780804221 + 946684800L)) + } + + @Test + fun `validate returns OK for full server state`() { + val result = RippleChainSpecific.validate(serverStateOk.toByteArray()) + assertThat(result).isEqualTo(UpstreamAvailability.OK) + } + + @Test + fun `validate returns SYNCING for connected server state`() { + val result = RippleChainSpecific.validate(serverStateSyncing.toByteArray()) + assertThat(result).isEqualTo(UpstreamAvailability.SYNCING) + } + + @Test + fun `validateSettings returns VALID for matching network`() { + val chain = Chain.RIPPLE__MAINNET + val result = RippleChainSpecific.validateSettings(serverStateOk.toByteArray(), chain) + assertThat(result).isEqualTo(ValidateUpstreamSettingsResult.UPSTREAM_VALID) + } +} diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaChainSpecificTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaChainSpecificTest.kt index f2eafb137..9636affa8 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaChainSpecificTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaChainSpecificTest.kt @@ -1,39 +1,216 @@ package io.emeraldpay.dshackle.upstream.solana +import io.emeraldpay.dshackle.Global import io.emeraldpay.dshackle.data.BlockId import io.emeraldpay.dshackle.reader.ChainReader -import org.assertj.core.api.Assertions +import io.emeraldpay.dshackle.upstream.ChainRequest +import io.emeraldpay.dshackle.upstream.ChainResponse +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import org.mockito.kotlin.any import org.mockito.kotlin.mock +import org.mockito.kotlin.times +import org.mockito.kotlin.verify +import reactor.core.publisher.Mono +import java.nio.ByteBuffer +import java.time.Instant +import java.time.temporal.ChronoUnit -val example = """{ - "context": { - "slot": 112301554 - }, - "value": { - "slot": 112301554, - "block": { - "previousBlockhash": "GJp125YAN4ufCSUvZJVdCyWQJ7RPWMmwxoyUQySydZA", - "blockhash": "6ojMHjctdqfB55JDpEpqfHnP96fiaHEcvzEQ2NNcxzHP", - "parentSlot": 112301553, - "blockTime": 1639926816, - "blockHeight": 101210751 - }, - "err": null - } - } -""".trimIndent() class SolanaChainSpecificTest { + @BeforeEach + fun setup() { + // Clear cache before each test + SolanaChainSpecific.clearCache() + } + + private fun epochInfoResponse(absoluteSlot: Long, blockHeight: Long): ChainResponse { + val json = """{"absoluteSlot": $absoluteSlot, "blockHeight": $blockHeight, "epoch": 100, "slotIndex": 1000, "slotsInEpoch": 432000}""" + return ChainResponse(json.toByteArray(), null) + } + + @Test + fun `parseBlock from slotSubscribe notification`() { + val reader = mock { + on { read(any()) }.thenReturn( + Mono.just(epochInfoResponse(112301554, 101210751)), + ) + } + + val beforeCall = Instant.now() + val json = """{"slot": 112301554, "parent": 112301553, "root": 112301500}""" + val result = SolanaChainSpecific.getFromHeader(json.toByteArray(), "upstream-1", reader).block()!! + val afterCall = Instant.now() + + // Uses actual data from getEpochInfo + assertThat(result.slot).isEqualTo(112301554) + assertThat(result.height).isEqualTo(101210751) + assertThat(result.upstreamId).isEqualTo("upstream-1") + + // Synthetic hash based on actualSlot from getEpochInfo + val expectedHash = BlockId.from(ByteBuffer.allocate(32).putLong(112301554).array()) + assertThat(result.hash).isEqualTo(expectedHash) + + // Synthetic parent hash based on actualSlot - 1 + val expectedParentHash = BlockId.from(ByteBuffer.allocate(32).putLong(112301553).array()) + assertThat(result.parentHash).isEqualTo(expectedParentHash) + + // Timestamp is synthetic (Instant.now() at call time) + assertThat(result.timestamp).isBetween(beforeCall, afterCall.plus(1, ChronoUnit.SECONDS)) + } + + @Test + fun `listenNewHeadsRequest returns slotSubscribe`() { + val request = SolanaChainSpecific.listenNewHeadsRequest() + + assertThat(request.method).isEqualTo("slotSubscribe") + } + + @Test + fun `unsubscribeNewHeadsRequest returns slotUnsubscribe`() { + val request = SolanaChainSpecific.unsubscribeNewHeadsRequest("sub-123") + + assertThat(request.method).isEqualTo("slotUnsubscribe") + } + + @Test + fun `throttle HTTP calls every 5 slots`() { + val reader = mock { + on { read(any()) }.thenReturn( + Mono.just(epochInfoResponse(100, 100000000)), + ) + } + + // First slot - should trigger HTTP call (no cache) + val slot1 = """{"slot": 100, "parent": 99, "root": 50}""" + SolanaChainSpecific.getFromHeader(slot1.toByteArray(), "upstream-1", reader).block() + + // Next 4 slots - should use cached height (within interval of 5) + for (i in 101..104) { + val slotN = """{"slot": $i, "parent": ${i - 1}, "root": 50}""" + SolanaChainSpecific.getFromHeader(slotN.toByteArray(), "upstream-1", reader).block() + } + + // Only 1 HTTP call so far + verify(reader, times(1)).read(any()) + + // Slot 105 - should trigger new HTTP call (interval reached) + val slot105 = """{"slot": 105, "parent": 104, "root": 50}""" + SolanaChainSpecific.getFromHeader(slot105.toByteArray(), "upstream-1", reader).block() + + // Now 2 HTTP calls + verify(reader, times(2)).read(any()) + } + + @Test + fun `synthetic hash is deterministic based on slot`() { + val slot = 12345L + val hash1 = BlockId.from(ByteBuffer.allocate(32).putLong(slot).array()) + val hash2 = BlockId.from(ByteBuffer.allocate(32).putLong(slot).array()) + + assertThat(hash1).isEqualTo(hash2) + + val differentSlot = 12346L + val hash3 = BlockId.from(ByteBuffer.allocate(32).putLong(differentSlot).array()) + + assertThat(hash1).isNotEqualTo(hash3) + } + @Test - fun parseBlock() { - val reader = mock {} + fun `SolanaSlotNotification parses correctly`() { + val json = """{"slot": 123456, "parent": 123455, "root": 123400}""" + val notification = Global.objectMapper.readValue(json, SolanaSlotNotification::class.java) + + assertThat(notification.slot).isEqualTo(123456) + assertThat(notification.parent).isEqualTo(123455) + assertThat(notification.root).isEqualTo(123400) + } + + @Test + fun `uses slot as height when cache is empty and no HTTP call`() { + val reader = mock { + on { read(any()) }.thenReturn(Mono.error(RuntimeException("Network error"))) + } + + // First slot with HTTP error - should fallback to slot as height + val json = """{"slot": 112301554, "parent": 112301553, "root": 112301500}""" + val result = SolanaChainSpecific.getFromHeader(json.toByteArray(), "upstream-1", reader).block()!! + + assertThat(result.slot).isEqualTo(112301554) + // When cache empty and HTTP fails, uses slot as height + assertThat(result.height).isEqualTo(112301554) + } + + @Test + fun `uses optimistic estimated height between throttle intervals`() { + val reader = mock { + on { read(any()) }.thenReturn( + Mono.just(epochInfoResponse(100, 100000000)), + ) + } + + // First call sets cache at slot 100 with height 100000000 + val slot1 = """{"slot": 100, "parent": 99, "root": 50}""" + val result1 = SolanaChainSpecific.getFromHeader(slot1.toByteArray(), "upstream-1", reader).block()!! + assertThat(result1.height).isEqualTo(100000000) + + // Second call uses optimistic estimated height: 100000000 + (101 - 100) = 100000001 + val slot2 = """{"slot": 101, "parent": 100, "root": 50}""" + val result2 = SolanaChainSpecific.getFromHeader(slot2.toByteArray(), "upstream-1", reader).block()!! + + assertThat(result2.slot).isEqualTo(101) + assertThat(result2.height).isEqualTo(100000001) // optimistic estimated height + + // Third call: 100000000 + (103 - 100) = 100000003 + val slot3 = """{"slot": 103, "parent": 102, "root": 50}""" + val result3 = SolanaChainSpecific.getFromHeader(slot3.toByteArray(), "upstream-1", reader).block()!! + + assertThat(result3.slot).isEqualTo(103) + assertThat(result3.height).isEqualTo(100000003) // optimistic estimated height + } + + @Test + fun `height estimation resets after RPC check`() { + val reader = mock { + on { read(any()) } + .thenReturn(Mono.just(epochInfoResponse(100, 100000000))) + .thenReturn(Mono.just(epochInfoResponse(105, 100000004))) + } + + // First call at slot 100 - sets cache with height 100000000 + val slot1 = """{"slot": 100, "parent": 99, "root": 50}""" + SolanaChainSpecific.getFromHeader(slot1.toByteArray(), "upstream-1", reader).block() + + // Slot 105 triggers RPC check - gets actual slot 105 and height 100000004 (1 slot was skipped) + val slot105 = """{"slot": 105, "parent": 104, "root": 50}""" + val result105 = SolanaChainSpecific.getFromHeader(slot105.toByteArray(), "upstream-1", reader).block()!! + assertThat(result105.slot).isEqualTo(105) + assertThat(result105.height).isEqualTo(100000004) + + // Slot 107 uses new baseline: 100000004 + (107 - 105) = 100000006 + val slot107 = """{"slot": 107, "parent": 106, "root": 50}""" + val result107 = SolanaChainSpecific.getFromHeader(slot107.toByteArray(), "upstream-1", reader).block()!! + assertThat(result107.height).isEqualTo(100000006) + } + + @Test + fun `uses estimated height on RPC error when estimation available`() { + val reader = mock { + on { read(any()) } + .thenReturn(Mono.just(epochInfoResponse(100, 100000000))) + .thenReturn(Mono.error(RuntimeException("Network error"))) + } + + // First call succeeds at slot 100 + val slot1 = """{"slot": 100, "parent": 99, "root": 50}""" + SolanaChainSpecific.getFromHeader(slot1.toByteArray(), "upstream-1", reader).block() - val result = SolanaChainSpecific.getFromHeader(example.toByteArray(), "1", reader).block()!! + // Slot 105 triggers RPC check but fails - should use estimated height: 100000000 + (105 - 100) = 100000005 + val slot105 = """{"slot": 105, "parent": 104, "root": 50}""" + val result = SolanaChainSpecific.getFromHeader(slot105.toByteArray(), "upstream-1", reader).block()!! - Assertions.assertThat(result.height).isEqualTo(101210751) - Assertions.assertThat(result.hash).isEqualTo(BlockId.fromBase64("6ojMHjctdqfB55JDpEpqfHnP96fiaHEcvzEQ2NNcxzHP")) - Assertions.assertThat(result.upstreamId).isEqualTo("1") - Assertions.assertThat(result.parentHash).isEqualTo(BlockId.fromBase64("GJp125YAN4ufCSUvZJVdCyWQJ7RPWMmwxoyUQySydZA")) + assertThat(result.slot).isEqualTo(105) + assertThat(result.height).isEqualTo(100000005) // estimated height used as fallback } } diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaLowerBoundServiceTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaLowerBoundServiceTest.kt index 7d0be676b..d0d48809f 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaLowerBoundServiceTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/solana/SolanaLowerBoundServiceTest.kt @@ -2,12 +2,14 @@ package io.emeraldpay.dshackle.upstream.solana import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.Global +import io.emeraldpay.dshackle.config.UpstreamsConfig import io.emeraldpay.dshackle.reader.ChainReader import io.emeraldpay.dshackle.upstream.ChainRequest import io.emeraldpay.dshackle.upstream.ChainResponse import io.emeraldpay.dshackle.upstream.Upstream import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType +import io.emeraldpay.dshackle.upstream.lowerbound.ManualLowerBoundType import io.emeraldpay.dshackle.upstream.rpcclient.ListParams import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test @@ -76,4 +78,68 @@ class SolanaLowerBoundServiceTest { ), ) } + + @Test + fun `get solana lower block and slot with manual bounds`() { + val reader = mock { + on { read(ChainRequest("getFirstAvailableBlock", ListParams())) } doReturn + Mono.just(ChainResponse("25000000".toByteArray(), null)) + on { + read( + ChainRequest( + "getBlock", + ListParams( + 25000000L, + mapOf( + "showRewards" to false, + "transactionDetails" to "none", + "maxSupportedTransactionVersion" to 0, + ), + ), + ), + ) + } doReturn Mono.just( + ChainResponse( + Global.objectMapper.writeValueAsBytes( + mapOf( + "blockHeight" to 21000000, + "blockTime" to 111, + "blockhash" to "22", + "previousBlockhash" to "33", + ), + ), + null, + ), + ) + } + val settings = UpstreamsConfig.AdditionalSettings( + mapOf( + LowerBoundType.SLOT to UpstreamsConfig.ManualBoundSetting(ManualLowerBoundType.FIXED, 54L), + ), + ) + val upstream = mock { + on { getIngressReader() } doReturn reader + on { getChain() } doReturn Chain.UNSPECIFIED + on { getAdditionalSettings() } doReturn settings + } + + val detector = SolanaLowerBoundService(Chain.UNSPECIFIED, upstream) + + StepVerifier.withVirtualTime { detector.detectLowerBounds() } + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(15)) + .expectNextMatches { it.lowerBound == 21000000L && it.type == LowerBoundType.STATE } + .expectNextMatches { it.lowerBound == 54L && it.type == LowerBoundType.SLOT } + .thenCancel() + .verify(Duration.ofSeconds(3)) + + assertThat(detector.getLowerBounds().toList()) + .usingRecursiveFieldByFieldElementComparatorIgnoringFields("timestamp") + .hasSameElementsAs( + listOf( + LowerBoundData(21000000L, LowerBoundType.STATE), + LowerBoundData(54L, LowerBoundType.SLOT), + ), + ) + } } diff --git a/src/test/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetLowerBoundStateDetectorTest.kt b/src/test/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetLowerBoundStateDetectorTest.kt index 473aefbae..79820bedc 100644 --- a/src/test/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetLowerBoundStateDetectorTest.kt +++ b/src/test/kotlin/io/emeraldpay/dshackle/upstream/starknet/StarknetLowerBoundStateDetectorTest.kt @@ -3,6 +3,7 @@ package io.emeraldpay.dshackle.upstream.starknet import io.emeraldpay.dshackle.Chain import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundData import io.emeraldpay.dshackle.upstream.lowerbound.LowerBoundType +import io.emeraldpay.dshackle.upstream.lowerbound.NoopManualLowerBoundService import org.junit.jupiter.api.Test import reactor.test.StepVerifier import java.time.Duration @@ -13,7 +14,7 @@ class StarknetLowerBoundStateDetectorTest { fun `starknet lower block is 1`() { val detector = StarknetLowerBoundStateDetector(Chain.STARKNET__MAINNET) - StepVerifier.withVirtualTime { detector.detectLowerBound() } + StepVerifier.withVirtualTime { detector.detectLowerBound(NoopManualLowerBoundService()) } .expectSubscription() .expectNoEvent(Duration.ofSeconds(15)) .expectNext(LowerBoundData(1, LowerBoundType.STATE)) diff --git a/src/test/resources/configs/upstreams-bearer-auth.yaml b/src/test/resources/configs/upstreams-bearer-auth.yaml new file mode 100644 index 000000000..b9ad19d2d --- /dev/null +++ b/src/test/resources/configs/upstreams-bearer-auth.yaml @@ -0,0 +1,27 @@ +version: v1 + +upstreams: + - id: local + chain: ethereum + connection: + ethereum: + rpc: + url: "http://localhost:8545" + bearer-auth: + token: 9c199ad8f281f20154fc258fe41a6814 + ws: + url: "ws://localhost:8546" + origin: "http://localhost" + bearer-auth: + token: 258fe4149c199ad8f2811a68f20154fc + - id: both-auth + chain: ethereum + connection: + ethereum: + rpc: + url: "http://localhost:8545" + basic-auth: + username: 4fc258fe41a68149c199ad8f281f2015 + password: 1a68f20154fc258fe4149c199ad8f281 + bearer-auth: + token: 9c199ad8f281f20154fc258fe41a6814