Feature/datanode profiling v2

.github/workflows/sandbox-check.yml

@@ -1,7 +1,11 @@
 name: Sandbox Check
 on:
   push:
+    paths:
+      - 'sandbox/**'
   pull_request:
+    paths:
+      - 'sandbox/**'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
@@ -32,6 +36,15 @@ jobs:
         uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
       - name: Install protobuf compiler
         run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+      - name: Cache Cargo registry
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+          key: ${{ runner.os }}-cargo-${{ hashFiles('sandbox/**/Cargo.lock') }}
+      - name: Set Cargo build jobs
+        run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV
       - name: Run sandbox check
         run: ./gradlew check -p sandbox -Dsandbox.enabled=true -PrustDebug
       - name: Upload test results

.gitignore

@@ -72,6 +72,6 @@ testfixtures_shared/
 
 # build files generated
 doc-tools/missing-doclet/bin/
-**/Cargo.lock
+
 /sandbox/plugins/analytics-backend-datafusion/target/
 /sandbox/libs/dataformat-native/rust/target

build.gradle

@@ -55,7 +55,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "8.4.0" apply false
+  id "com.diffplug.spotless" version "8.6.0" apply false
   id "org.gradle.test-retry" version "1.6.2" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'

buildSrc/build.gradle

@@ -125,19 +125,26 @@ dependencies {
   api 'org.apache.rat:apache-rat:0.15'
   api "commons-io:commons-io:${props.getProperty('commonsio')}"
   api "net.java.dev.jna:jna:5.16.0"
-  api 'com.gradleup.shadow:shadow-gradle-plugin:9.3.1'
+  api 'com.gradleup.shadow:shadow-gradle-plugin:9.4.2'
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
   api 'de.thetaphi:forbiddenapis:3.10'
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.12'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
-  api 'org.apache.maven:maven-model:3.9.12'
+  api 'org.apache.maven:maven-model:3.9.16'
+  api 'org.codehaus.plexus:plexus-xml:3.0.1'
   api 'com.networknt:json-schema-validator:1.2.0'
   api 'org.jruby.jcodings:jcodings:1.0.58'
   api 'org.jruby.joni:joni:2.2.6'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
   api "org.ajoberstar.grgit:grgit-core:5.3.2"
 
+  constraints {
+    api('org.codehaus.plexus:plexus-utils:4.0.3') {
+      because 'fixes CVE-2025-67030'
+    }
+  }
+
   testFixturesApi "junit:junit:${props.getProperty('junit')}"
   testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
   testFixturesApi gradleApi()
@@ -160,6 +167,7 @@ dependencies {
 configurations.all {
   resolutionStrategy {
     force "com.google.guava:guava:${props.getProperty('guava')}"
+    force "org.apache.logging.log4j:log4j-core:${props.getProperty('log4j')}"
   }
 }
 

client/sniffer/licenses/jackson-core-3.1.4.jar.sha1

@@ -0,0 +1 @@
+f9fd39204bd3d4922b42f6a54b81f46bd8ed575a

distribution/src/config/opensearch.yml

@@ -121,3 +121,12 @@ ${path.logs}
 # Once there is no observed impact on performance, this feature flag can be removed.
 #
 #opensearch.experimental.optimization.datetime_formatter_caching.enabled: false
+#
+# Path to the spill directory used by the analytics-backend-datafusion plugin. When
+# unset (default), spill is disabled and queries that exceed
+# datafusion.memory_pool_limit_bytes will fail with a "DiskManager is disabled" error.
+# Set this to a directory on a dedicated filesystem to enable spill — the directory
+# should not contend with the data path. NodeScope, Final — DataFusion's DiskManager
+# is built once at runtime startup.
+#
+#datafusion.spill_directory:

gradle/libs.versions.toml

@@ -8,19 +8,19 @@ bundled_jdk = "25.0.3+9"
 # optional dependencies
 spatial4j               = "0.7"
 jts                     = "1.15.0"
-jackson_annotations     = "2.21"
-jackson                 = "2.21.3"
-jackson_databind        = "2.21.3"
-jackson3                = "3.1.3"
-jackson3_databind       = "3.1.3"
+jackson_annotations     = "2.22"
+jackson                 = "2.22.0"
+jackson_databind        = "2.22.0"
+jackson3                = "3.1.4"
+jackson3_databind       = "3.1.4"
 snakeyaml               = "2.6"
 snakeyaml_engine        = "3.0.1"
 icu4j                   = "77.1"
 supercsv                = "2.4.0"
 log4j                   = "2.25.4"
 error_prone_annotations = "2.45.0"
 slf4j                   = "2.0.17"
-asm                     = "9.10"
+asm                     = "9.10.1"
 jettison                = "1.5.4"
 woodstox                = "6.4.0"
 kotlin                  = "1.7.10"
@@ -40,7 +40,7 @@ json_smart              = "2.5.2"
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = "5.16.0"
 
-netty             = "4.2.14.Final"
+netty             = "4.2.15.Final"
 joda              = "2.12.7"
 roaringbitmap     = "1.3.0"
 
@@ -110,9 +110,6 @@ calcite               = "1.41.0"
 calcite_os_rev        = "1"
 
 # property-based testing
-jqwik                 = "1.9.2"
-junit_jupiter         = "5.11.3"
-junit_platform        = "1.11.3"
 [libraries]
 antlr4-runtime = { group = "org.antlr", name = "antlr4-runtime", version.ref = "antlr4" }
 asm-analysis = { group = "org.ow2.asm", name = "asm-analysis", version.ref = "asm" }

libs/core/licenses/jackson-core-3.1.4.jar.sha1

@@ -0,0 +1 @@
+f9fd39204bd3d4922b42f6a54b81f46bd8ed575a