From 95b0de61d172a323aead25571b8b83b4ae0b7120 Mon Sep 17 00:00:00 2001 From: George Shiqi Wu Date: Fri, 25 Aug 2023 12:50:38 -0400 Subject: [PATCH 001/258] Move some lifecycle management from doTask -> shutdown for the mm-less task runner (#14895) * save work * Add syncronized * Don't shutdown in run * Adding unit tests * Cleanup lifecycle * Fix tests * remove newline --- .../k8s/overlord/KubernetesPeonLifecycle.java | 9 +-- .../k8s/overlord/KubernetesTaskRunner.java | 24 ++++--- .../k8s/overlord/KubernetesWorkItem.java | 9 --- .../overlord/KubernetesPeonLifecycleTest.java | 14 +---- .../overlord/KubernetesTaskRunnerTest.java | 62 +++++++++---------- .../k8s/overlord/KubernetesWorkItemTest.java | 2 - 6 files changed, 52 insertions(+), 68 deletions(-) diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycle.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycle.java index f6b15f46bc58..4814d8cbb609 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycle.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycle.java @@ -137,7 +137,6 @@ protected synchronized TaskStatus run(Job job, long launchTimeout, long timeout) } catch (Exception e) { log.info("Failed to run task: %s", taskId.getOriginalTaskId()); - shutdown(); throw e; } finally { @@ -168,10 +167,9 @@ protected synchronized TaskStatus join(long timeout) throws IllegalStateExceptio finally { try { saveLogs(); - shutdown(); } catch (Exception e) { - log.warn(e, "Task [%s] cleanup failed", taskId); + log.warn(e, "Log processing failed for task [%s]", taskId); } stopTask(); @@ -188,7 +186,7 @@ protected synchronized TaskStatus join(long timeout) throws IllegalStateExceptio */ protected void shutdown() { - if (State.PENDING.equals(state.get()) || State.RUNNING.equals(state.get())) { + if (State.PENDING.equals(state.get()) || State.RUNNING.equals(state.get()) || State.STOPPED.equals(state.get())) { kubernetesClient.deletePeonJob(taskId); } } @@ -223,7 +221,7 @@ protected State getState() */ protected TaskLocation getTaskLocation() { - if (!State.RUNNING.equals(state.get())) { + if (State.PENDING.equals(state.get()) || State.NOT_STARTED.equals(state.get())) { log.debug("Can't get task location for non-running job. [%s]", taskId.getOriginalTaskId()); return TaskLocation.unknown(); } @@ -251,7 +249,6 @@ protected TaskLocation getTaskLocation() Boolean.parseBoolean(pod.getMetadata().getAnnotations().getOrDefault(DruidK8sConstants.TLS_ENABLED, "false")), pod.getMetadata() != null ? pod.getMetadata().getName() : "" ); - log.info("K8s task %s is running at location %s", taskId.getOriginalTaskId(), taskLocation); } return taskLocation; diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunner.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunner.java index 9a4e4bcca629..24e21b0b4e0e 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunner.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunner.java @@ -182,10 +182,6 @@ protected TaskStatus doTask(Task task, boolean run) KubernetesWorkItem workItem = tasks.get(task.getId()); if (workItem == null) { - throw new ISE("Task [%s] disappeared", task.getId()); - } - - if (workItem.isShutdownRequested()) { throw new ISE("Task [%s] has been shut down", task.getId()); } @@ -213,11 +209,6 @@ protected TaskStatus doTask(Task task, boolean run) log.error(e, "Task [%s] execution caught an exception", task.getId()); throw new RuntimeException(e); } - finally { - synchronized (tasks) { - tasks.remove(task.getId()); - } - } } @VisibleForTesting @@ -271,6 +262,10 @@ public void shutdown(String taskid, String reason) return; } + synchronized (tasks) { + tasks.remove(taskid); + } + workItem.shutdown(); } @@ -440,6 +435,17 @@ public Collection getPendingTasks() .collect(Collectors.toList()); } + @Override + public TaskLocation getTaskLocation(String taskId) + { + final KubernetesWorkItem workItem = tasks.get(taskId); + if (workItem == null) { + return TaskLocation.unknown(); + } else { + return workItem.getLocation(); + } + } + @Nullable @Override public RunnerTaskState getRunnerTaskState(String taskId) diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesWorkItem.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesWorkItem.java index 164a82b6ae27..94d4bbb67f63 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesWorkItem.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesWorkItem.java @@ -30,13 +30,10 @@ import org.apache.druid.java.util.common.ISE; import java.io.InputStream; -import java.util.concurrent.atomic.AtomicBoolean; public class KubernetesWorkItem extends TaskRunnerWorkItem { private final Task task; - - private final AtomicBoolean shutdownRequested = new AtomicBoolean(false); private KubernetesPeonLifecycle kubernetesPeonLifecycle = null; public KubernetesWorkItem(Task task, ListenableFuture statusFuture) @@ -53,7 +50,6 @@ protected synchronized void setKubernetesPeonLifecycle(KubernetesPeonLifecycle k protected synchronized void shutdown() { - this.shutdownRequested.set(true); if (this.kubernetesPeonLifecycle != null) { this.kubernetesPeonLifecycle.startWatchingLogs(); @@ -61,11 +57,6 @@ protected synchronized void shutdown() } } - protected boolean isShutdownRequested() - { - return shutdownRequested.get(); - } - protected boolean isPending() { return RunnerTaskState.PENDING.equals(getRunnerTaskState()); diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleTest.java index 3a017e5f74ff..084d1db62d2b 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleTest.java @@ -198,9 +198,6 @@ protected synchronized TaskStatus join(long timeout) EasyMock.anyLong(), EasyMock.eq(TimeUnit.MILLISECONDS) )).andReturn(null); - EasyMock.expect(kubernetesClient.deletePeonJob( - new K8sTaskId(ID) - )).andReturn(true); Assert.assertEquals(KubernetesPeonLifecycle.State.NOT_STARTED, peonLifecycle.getState()); stateListener.stateChanged(KubernetesPeonLifecycle.State.PENDING, ID); EasyMock.expectLastCall().once(); @@ -245,7 +242,6 @@ public void test_join_withoutJob_returnsFailedTaskStatus() throws IOException EasyMock.expectLastCall().once(); logWatch.close(); EasyMock.expectLastCall(); - EasyMock.expect(kubernetesClient.deletePeonJob(k8sTaskId)).andReturn(true); replayAll(); @@ -298,7 +294,6 @@ public void test_join() throws IOException EasyMock.expectLastCall().once(); logWatch.close(); EasyMock.expectLastCall(); - EasyMock.expect(kubernetesClient.deletePeonJob(k8sTaskId)).andReturn(true); Assert.assertEquals(KubernetesPeonLifecycle.State.NOT_STARTED, peonLifecycle.getState()); @@ -353,7 +348,6 @@ public void test_join_whenCalledMultipleTimes_raisesIllegalStateException() thro EasyMock.expectLastCall().once(); logWatch.close(); EasyMock.expectLastCall(); - EasyMock.expect(kubernetesClient.deletePeonJob(k8sTaskId)).andReturn(true); Assert.assertEquals(KubernetesPeonLifecycle.State.NOT_STARTED, peonLifecycle.getState()); @@ -408,7 +402,6 @@ public void test_join_withoutTaskStatus_returnsFailedTaskStatus() throws IOExcep EasyMock.expectLastCall().once(); logWatch.close(); EasyMock.expectLastCall(); - EasyMock.expect(kubernetesClient.deletePeonJob(k8sTaskId)).andReturn(true); Assert.assertEquals(KubernetesPeonLifecycle.State.NOT_STARTED, peonLifecycle.getState()); @@ -459,7 +452,6 @@ public void test_join_whenIOExceptionThrownWhileStreamingTaskStatus_returnsFaile EasyMock.expectLastCall().once(); logWatch.close(); EasyMock.expectLastCall(); - EasyMock.expect(kubernetesClient.deletePeonJob(k8sTaskId)).andReturn(true); Assert.assertEquals(KubernetesPeonLifecycle.State.NOT_STARTED, peonLifecycle.getState()); @@ -512,7 +504,6 @@ public void test_join_whenIOExceptionThrownWhileStreamingTaskLogs_isIgnored() th EasyMock.expectLastCall().once(); logWatch.close(); EasyMock.expectLastCall(); - EasyMock.expect(kubernetesClient.deletePeonJob(k8sTaskId)).andReturn(true); Assert.assertEquals(KubernetesPeonLifecycle.State.NOT_STARTED, peonLifecycle.getState()); @@ -554,8 +545,6 @@ public void test_join_whenRuntimeExceptionThrownWhileWaitingForKubernetesJob_thr logWatch.close(); EasyMock.expectLastCall(); - EasyMock.expect(kubernetesClient.deletePeonJob(k8sTaskId)).andReturn(true); - Assert.assertEquals(KubernetesPeonLifecycle.State.NOT_STARTED, peonLifecycle.getState()); replayAll(); @@ -908,8 +897,11 @@ public void test_getTaskLocation_withStoppedTaskState_returnsUnknown() stateListener ); setPeonLifecycleState(peonLifecycle, KubernetesPeonLifecycle.State.STOPPED); + EasyMock.expect(kubernetesClient.getPeonPod(k8sTaskId.getK8sJobName())).andReturn(Optional.absent()).once(); + replayAll(); Assert.assertEquals(TaskLocation.unknown(), peonLifecycle.getTaskLocation()); + verifyAll(); } private void setPeonLifecycleState(KubernetesPeonLifecycle peonLifecycle, KubernetesPeonLifecycle.State state) diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerTest.java index ca1fc641719e..bee3a533c7b8 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerTest.java @@ -152,8 +152,6 @@ public void test_run_withoutExistingTask() throws IOException, ExecutionExceptio Assert.assertEquals(taskStatus, future.get()); verifyAll(); - - Assert.assertFalse(runner.tasks.containsKey(task.getId())); } @Test @@ -191,8 +189,6 @@ public void test_run_whenExceptionThrown_throwsRuntimeException() throws IOExcep Assert.assertTrue(e.getCause() instanceof RuntimeException); verifyAll(); - - Assert.assertFalse(runner.tasks.containsKey(task.getId())); } @Test @@ -208,8 +204,6 @@ public void test_join_withoutExistingTask() throws ExecutionException, Interrupt Assert.assertEquals(taskStatus, future.get()); verifyAll(); - - Assert.assertFalse(runner.tasks.containsKey(task.getId())); } @Test @@ -236,28 +230,11 @@ public void test_join_whenExceptionThrown_throwsRuntimeException() Assert.assertTrue(e.getCause() instanceof RuntimeException); verifyAll(); - - Assert.assertFalse(runner.tasks.containsKey(task.getId())); - } - - @Test - public void test_doTask_withoutWorkItem_throwsRuntimeException() - { - Assert.assertThrows( - "Task [id] disappeared", - RuntimeException.class, - () -> runner.doTask(task, true) - ); } @Test public void test_doTask_whenShutdownRequested_throwsRuntimeException() { - KubernetesWorkItem workItem = new KubernetesWorkItem(task, null); - workItem.shutdown(); - - runner.tasks.put(task.getId(), workItem); - Assert.assertThrows( "Task [id] has been shut down", RuntimeException.class, @@ -266,13 +243,7 @@ public void test_doTask_whenShutdownRequested_throwsRuntimeException() } @Test - public void test_shutdown_withoutExistingTask() - { - runner.shutdown(task.getId(), ""); - } - - @Test - public void test_shutdown_withExistingTask() + public void test_shutdown_withExistingTask_removesTaskFromMap() { KubernetesWorkItem workItem = new KubernetesWorkItem(task, null) { @Override @@ -282,7 +253,13 @@ protected synchronized void shutdown() }; runner.tasks.put(task.getId(), workItem); + runner.shutdown(task.getId(), ""); + Assert.assertTrue(runner.tasks.isEmpty()); + } + @Test + public void test_shutdown_withoutExistingTask() + { runner.shutdown(task.getId(), ""); } @@ -629,6 +606,30 @@ public TaskLocation getLocation() verifyAll(); } + @Test + public void test_getTaskLocation_withExistingTask() + { + KubernetesWorkItem workItem = new KubernetesWorkItem(task, null) { + @Override + public TaskLocation getLocation() + { + return TaskLocation.create("host", 0, 1, false); + } + }; + + runner.tasks.put(task.getId(), workItem); + + TaskLocation taskLocation = runner.getTaskLocation(task.getId()); + Assert.assertEquals(TaskLocation.create("host", 0, 1, false), taskLocation); + } + + @Test + public void test_getTaskLocation_noTaskFound() + { + TaskLocation taskLocation = runner.getTaskLocation(task.getId()); + Assert.assertEquals(TaskLocation.unknown(), taskLocation); + } + @Test public void test_getTotalCapacity() { @@ -644,6 +645,5 @@ public void test_getUsedCapacity() Assert.assertEquals(1, runner.getUsedCapacity()); runner.tasks.remove(task.getId()); Assert.assertEquals(0, runner.getUsedCapacity()); - } } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesWorkItemTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesWorkItemTest.java index 5f951770480f..d5cf2ea7252d 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesWorkItemTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesWorkItemTest.java @@ -75,7 +75,6 @@ public void test_setKubernetesPeonLifecycleTwice_throwsIllegalStateException() public void test_shutdown_withoutKubernetesPeonLifecycle() { workItem.shutdown(); - Assert.assertTrue(workItem.isShutdownRequested()); } @Test @@ -91,7 +90,6 @@ public void test_shutdown_withKubernetesPeonLifecycle() workItem.shutdown(); verifyAll(); - Assert.assertTrue(workItem.isShutdownRequested()); } @Test From 9142f4b8d7acf8942f6a0df20052b32eb85d91e2 Mon Sep 17 00:00:00 2001 From: Victoria Lim Date: Fri, 25 Aug 2023 14:14:29 -0700 Subject: [PATCH 002/258] docs: update note in automatic compaction doc (#14908) --- docs/data-management/automatic-compaction.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/data-management/automatic-compaction.md b/docs/data-management/automatic-compaction.md index 1a271769860f..8d696a86d4ef 100644 --- a/docs/data-management/automatic-compaction.md +++ b/docs/data-management/automatic-compaction.md @@ -23,6 +23,9 @@ title: "Automatic compaction" --> In Apache Druid, compaction is a special type of ingestion task that reads data from a Druid datasource and writes it back into the same datasource. A common use case for this is to [optimally size segments](../operations/segment-optimization.md) after ingestion to improve query performance. Automatic compaction, or auto-compaction, refers to the system for automatic execution of compaction tasks managed by the [Druid Coordinator](../design/coordinator.md). +This topic guides you through setting up automatic compaction for your Druid cluster. See the [examples](#examples) for common use cases for automatic compaction. + +## How Druid manages automatic compaction The Coordinator [indexing period](../configuration/index.md#coordinator-operation), `druid.coordinator.period.indexingPeriod`, controls the frequency of compaction tasks. The default indexing period is 30 minutes, meaning that the Coordinator first checks for segments to compact at most 30 minutes from when auto-compaction is enabled. @@ -33,9 +36,12 @@ At every invocation of auto-compaction, the Coordinator initiates a [segment sea When there are eligible segments to compact, the Coordinator issues compaction tasks based on available worker capacity. If a compaction task takes longer than the indexing period, the Coordinator waits for it to finish before resuming the period for segment search. +:::info + Auto-compaction skips datasources that have a segment granularity of `ALL`. +::: + As a best practice, you should set up auto-compaction for all Druid datasources. You can run compaction tasks manually for cases where you want to allocate more system resources. For example, you may choose to run multiple compaction tasks in parallel to compact an existing datasource for the first time. See [Compaction](compaction.md) for additional details and use cases. -This topic guides you through setting up automatic compaction for your Druid cluster. See the [examples](#examples) for common use cases for automatic compaction. ## Enable automatic compaction @@ -174,10 +180,6 @@ The following auto-compaction configuration compacts existing `HOUR` segments in } ``` -:::info - Auto-compaction skips datasources containing ALL granularity segments when the target granularity is different. -::: - ### Update partitioning scheme For your `wikipedia` datasource, you want to optimize segment access when regularly ingesting data without compromising compute time when querying the data. Your ingestion spec for batch append uses [dynamic partitioning](../ingestion/native-batch.md#dynamic-partitioning) to optimize for write-time operations, while your stream ingestion partitioning is configured by the stream service. You want to implement auto-compaction to reorganize the data with a suitable read-time partitioning using [multi-dimension range partitioning](../ingestion/native-batch.md#multi-dimension-range-partitioning). Based on the dimensions frequently accessed in queries, you wish to partition on the following dimensions: `channel`, `countryName`, `namespace`. From 30c49c4cfc39bea1745eff1c6a1f195587f64842 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Fri, 25 Aug 2023 15:18:37 -0700 Subject: [PATCH 003/258] Web console: misc fixes and SQL query re-formatting (#14906) * better dialog formatting * use CSS to render triangle * can flatten in kafka also * better formatting * better format * fill in empty values in line chart * more fp * add show others --- licenses.yaml | 28 +- licenses/bin/@babel-helper-string-parser.MIT | 22 + licenses/bin/commander.MIT | 22 + licenses/bin/d3-dsv.BSD3 | 27 + licenses/bin/iconv-lite.MIT | 21 + licenses/bin/rw.BSD3 | 26 + licenses/bin/safer-buffer.MIT | 21 + web-console/package-lock.json | 2264 ++++++----------- web-console/package.json | 6 +- .../fancy-numeric-input.tsx | 9 +- .../supervisor-reset-offsets-dialog.scss | 32 + .../supervisor-reset-offsets-dialog.tsx | 38 +- .../async-query/async-query.mock.ts | 16 +- .../ingest-query-pattern.spec.ts | 16 +- .../input-format/input-format.tsx | 17 +- .../workbench-query/workbench-query.ts | 20 +- web-console/src/utils/sample-query.spec.tsx | 8 +- web-console/src/utils/sql.spec.ts | 88 +- .../modules/time-chart-echarts-module.ts | 53 +- .../flexible-query-input.scss | 46 +- 20 files changed, 1149 insertions(+), 1631 deletions(-) create mode 100644 licenses/bin/@babel-helper-string-parser.MIT create mode 100644 licenses/bin/commander.MIT create mode 100644 licenses/bin/d3-dsv.BSD3 create mode 100644 licenses/bin/iconv-lite.MIT create mode 100644 licenses/bin/rw.BSD3 create mode 100644 licenses/bin/safer-buffer.MIT create mode 100644 web-console/src/dialogs/supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog.scss diff --git a/licenses.yaml b/licenses.yaml index feb85c522dd2..4ddc9107c825 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -4874,8 +4874,8 @@ name: "@babel/code-frame" license_category: binary module: web-console license_name: MIT License -copyright: Sebastian McKenzie -version: 7.12.11 +copyright: The Babel Team +version: 7.22.10 license_file_path: licenses/bin/@babel-code-frame.MIT --- @@ -4884,18 +4884,28 @@ name: "@babel/helper-module-imports" license_category: binary module: web-console license_name: MIT License -copyright: Logan Smyth -version: 7.13.12 +copyright: The Babel Team +version: 7.22.5 license_file_path: licenses/bin/@babel-helper-module-imports.MIT --- +name: "@babel/helper-string-parser" +license_category: binary +module: web-console +license_name: MIT License +copyright: The Babel Team +version: 7.22.5 +license_file_path: licenses/bin/@babel-helper-string-parser.MIT + +--- + name: "@babel/helper-validator-identifier" license_category: binary module: web-console license_name: MIT License copyright: The Babel Team -version: 7.19.1 +version: 7.22.5 license_file_path: licenses/bin/@babel-helper-validator-identifier.MIT --- @@ -4905,7 +4915,7 @@ license_category: binary module: web-console license_name: MIT License copyright: The Babel Team -version: 7.18.6 +version: 7.22.10 license_file_path: licenses/bin/@babel-highlight.MIT --- @@ -4924,8 +4934,8 @@ name: "@babel/types" license_category: binary module: web-console license_name: MIT License -copyright: Sebastian McKenzie -version: 7.14.4 +copyright: The Babel Team +version: 7.22.11 license_file_path: licenses/bin/@babel-types.MIT --- @@ -4998,7 +5008,7 @@ license_category: binary module: web-console license_name: Apache License version 2.0 copyright: Imply Data -version: 0.20.5 +version: 0.21.1 --- diff --git a/licenses/bin/@babel-helper-string-parser.MIT b/licenses/bin/@babel-helper-string-parser.MIT new file mode 100644 index 000000000000..f31575ec773b --- /dev/null +++ b/licenses/bin/@babel-helper-string-parser.MIT @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2014-present Sebastian McKenzie and other contributors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses/bin/commander.MIT b/licenses/bin/commander.MIT new file mode 100644 index 000000000000..10f997ab1045 --- /dev/null +++ b/licenses/bin/commander.MIT @@ -0,0 +1,22 @@ +(The MIT License) + +Copyright (c) 2011 TJ Holowaychuk + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses/bin/d3-dsv.BSD3 b/licenses/bin/d3-dsv.BSD3 new file mode 100644 index 000000000000..3d0802c3bd11 --- /dev/null +++ b/licenses/bin/d3-dsv.BSD3 @@ -0,0 +1,27 @@ +Copyright 2013-2016 Mike Bostock +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the author nor the names of contributors may be used to + endorse or promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses/bin/iconv-lite.MIT b/licenses/bin/iconv-lite.MIT new file mode 100644 index 000000000000..d518d8376af9 --- /dev/null +++ b/licenses/bin/iconv-lite.MIT @@ -0,0 +1,21 @@ +Copyright (c) 2011 Alexander Shtuchkin + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/licenses/bin/rw.BSD3 b/licenses/bin/rw.BSD3 new file mode 100644 index 000000000000..da8230d35203 --- /dev/null +++ b/licenses/bin/rw.BSD3 @@ -0,0 +1,26 @@ +Copyright (c) 2014-2016, Michael Bostock +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* The name Michael Bostock may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses/bin/safer-buffer.MIT b/licenses/bin/safer-buffer.MIT new file mode 100644 index 000000000000..4fe9e6f10036 --- /dev/null +++ b/licenses/bin/safer-buffer.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Nikita Skovoroda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/web-console/package-lock.json b/web-console/package-lock.json index f8fb8af83d94..3e396e22fcdc 100644 --- a/web-console/package-lock.json +++ b/web-console/package-lock.json @@ -14,7 +14,7 @@ "@blueprintjs/datetime2": "^0.9.35", "@blueprintjs/icons": "^4.16.0", "@blueprintjs/popover2": "^1.14.9", - "@druid-toolkit/query": "^0.20.5", + "@druid-toolkit/query": "^0.21.1", "@druid-toolkit/visuals-core": "^0.3.3", "@druid-toolkit/visuals-react": "^0.3.3", "ace-builds": "~1.4.14", @@ -55,7 +55,7 @@ "@awesome-code-style/eslint-config": "^4.1.0", "@awesome-code-style/prettier-config": "^4.0.0", "@awesome-code-style/stylelint-config": "^4.0.0", - "@babel/core": "^7.14.3", + "@babel/core": "^7.18.6", "@babel/preset-env": "^7.14.4", "@testing-library/react": "^14.0.0", "@types/classnames": "^2.2.9", @@ -118,7 +118,7 @@ "ts-node": "^10.9.1", "typescript": "^4.9.5", "uuid": "^7.0.2", - "webpack": "^5.33.2", + "webpack": "^5.76.0", "webpack-bundle-analyzer": "^4.4.1", "webpack-cli": "^4.6.0", "webpack-dev-server": "^3.11.2" @@ -127,6 +127,19 @@ "node": ">=16" } }, + "node_modules/@ampproject/remapping": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.1.tgz", + "integrity": "sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==", + "dev": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.0", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/@awesome-code-style/eslint-config": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/@awesome-code-style/eslint-config/-/eslint-config-4.1.0.tgz", @@ -177,40 +190,47 @@ } }, "node_modules/@babel/code-frame": { - "version": "7.12.11", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.11.tgz", - "integrity": "sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw==", + "version": "7.22.10", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.10.tgz", + "integrity": "sha512-/KKIMG4UEL35WmI9OlvMhurwtytjvXoFcGNrOvyG9zIzA8YmPjVtIZUf7b05+TPO7G7/GEmLHDaoCgACHl9hhA==", "dependencies": { - "@babel/highlight": "^7.10.4" + "@babel/highlight": "^7.22.10", + "chalk": "^2.4.2" + }, + "engines": { + "node": ">=6.9.0" } }, "node_modules/@babel/compat-data": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.14.4.tgz", - "integrity": "sha512-i2wXrWQNkH6JplJQGn3Rd2I4Pij8GdHkXwHMxm+zV5YG/Jci+bCNrWZEWC4o+umiDkRrRs4dVzH3X4GP7vyjQQ==", - "dev": true + "version": "7.22.9", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.22.9.tgz", + "integrity": "sha512-5UamI7xkUcJ3i9qVDS+KFDEK8/7oJ55/sJMB1Ge7IEapr7KfdfV/HErR+koZwOfd+SgtFKOKRhRakdg++DcJpQ==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } }, "node_modules/@babel/core": { - "version": "7.14.3", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.14.3.tgz", - "integrity": "sha512-jB5AmTKOCSJIZ72sd78ECEhuPiDMKlQdDI/4QRI6lzYATx5SSogS1oQA2AoPecRCknm30gHi2l+QVvNUu3wZAg==", - "dev": true, - "dependencies": { - "@babel/code-frame": "^7.12.13", - "@babel/generator": "^7.14.3", - "@babel/helper-compilation-targets": "^7.13.16", - "@babel/helper-module-transforms": "^7.14.2", - "@babel/helpers": "^7.14.0", - "@babel/parser": "^7.14.3", - "@babel/template": "^7.12.13", - "@babel/traverse": "^7.14.2", - "@babel/types": "^7.14.2", + "version": "7.22.11", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.22.11.tgz", + "integrity": "sha512-lh7RJrtPdhibbxndr6/xx0w8+CVlY5FJZiaSz908Fpy+G0xkBFTvwLcKJFF4PJxVfGhVWNebikpWGnOoC71juQ==", + "dev": true, + "dependencies": { + "@ampproject/remapping": "^2.2.0", + "@babel/code-frame": "^7.22.10", + "@babel/generator": "^7.22.10", + "@babel/helper-compilation-targets": "^7.22.10", + "@babel/helper-module-transforms": "^7.22.9", + "@babel/helpers": "^7.22.11", + "@babel/parser": "^7.22.11", + "@babel/template": "^7.22.5", + "@babel/traverse": "^7.22.11", + "@babel/types": "^7.22.11", "convert-source-map": "^1.7.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", - "json5": "^2.1.2", - "semver": "^6.3.0", - "source-map": "^0.5.0" + "json5": "^2.2.3", + "semver": "^6.3.1" }, "engines": { "node": ">=6.9.0" @@ -220,25 +240,6 @@ "url": "https://opencollective.com/babel" } }, - "node_modules/@babel/core/node_modules/@babel/code-frame": { - "version": "7.12.13", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.13.tgz", - "integrity": "sha512-HV1Cm0Q3ZrpCR93tkWOYiuYIgLxZXZFVG2VgK+MBWjUqZTundupbfx2aXarXuw5Ko5aMcjtJgbSs4vUGBS5v6g==", - "dev": true, - "dependencies": { - "@babel/highlight": "^7.12.13" - } - }, - "node_modules/@babel/core/node_modules/@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - }, "node_modules/@babel/core/node_modules/convert-source-map": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.7.0.tgz", @@ -266,13 +267,10 @@ } }, "node_modules/@babel/core/node_modules/json5": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.0.tgz", - "integrity": "sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA==", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", "dev": true, - "dependencies": { - "minimist": "^1.2.5" - }, "bin": { "json5": "lib/cli.js" }, @@ -287,9 +285,9 @@ "dev": true }, "node_modules/@babel/core/node_modules/semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "bin": { "semver": "bin/semver.js" @@ -332,30 +330,28 @@ } }, "node_modules/@babel/generator": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.17.0.tgz", - "integrity": "sha512-I3Omiv6FGOC29dtlZhkfXO6pgkmukJSlT26QjVvS1DGZe/NzSVCPG41X0tS21oZkJYlovfj9qDWgKP+Cn4bXxw==", + "version": "7.22.10", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.22.10.tgz", + "integrity": "sha512-79KIf7YiWjjdZ81JnLujDRApWtl7BxTqWD88+FFdQEIOG8LJ0etDOM7CXuIgGJa55sGOwZVwuEsaLEm0PJ5/+A==", "dev": true, "dependencies": { - "@babel/types": "^7.17.0", - "jsesc": "^2.5.1", - "source-map": "^0.5.0" + "@babel/types": "^7.22.10", + "@jridgewell/gen-mapping": "^0.3.2", + "@jridgewell/trace-mapping": "^0.3.17", + "jsesc": "^2.5.1" }, "engines": { "node": ">=6.9.0" } }, - "node_modules/@babel/generator/node_modules/@babel/types": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.17.0.tgz", - "integrity": "sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==", + "node_modules/@babel/generator/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.19", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz", + "integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==", "dev": true, "dependencies": { - "@babel/helper-validator-identifier": "^7.16.7", - "to-fast-properties": "^2.0.0" - }, - "engines": { - "node": ">=6.9.0" + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" } }, "node_modules/@babel/helper-annotate-as-pure": { @@ -367,16 +363,6 @@ "@babel/types": "^7.12.13" } }, - "node_modules/@babel/helper-annotate-as-pure/node_modules/@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - }, "node_modules/@babel/helper-builder-binary-assignment-operator-visitor": { "version": "7.12.13", "resolved": "https://registry.npmjs.org/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.12.13.tgz", @@ -387,40 +373,46 @@ "@babel/types": "^7.12.13" } }, - "node_modules/@babel/helper-builder-binary-assignment-operator-visitor/node_modules/@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", + "node_modules/@babel/helper-compilation-targets": { + "version": "7.22.10", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.10.tgz", + "integrity": "sha512-JMSwHD4J7SLod0idLq5PKgI+6g/hLD/iuWBq08ZX49xE14VpVEojJ5rHWptpirV2j020MvypRLAXAO50igCJ5Q==", "dev": true, "dependencies": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" + "@babel/compat-data": "^7.22.9", + "@babel/helper-validator-option": "^7.22.5", + "browserslist": "^4.21.9", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" } }, - "node_modules/@babel/helper-compilation-targets": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.14.4.tgz", - "integrity": "sha512-JgdzOYZ/qGaKTVkn5qEDV/SXAh8KcyUVkCoSWGN8T3bwrgd6m+/dJa2kVGi6RJYJgEYPBdZ84BZp9dUjNWkBaA==", + "node_modules/@babel/helper-compilation-targets/node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", "dev": true, "dependencies": { - "@babel/compat-data": "^7.14.4", - "@babel/helper-validator-option": "^7.12.17", - "browserslist": "^4.16.6", - "semver": "^6.3.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" + "yallist": "^3.0.2" } }, "node_modules/@babel/helper-compilation-targets/node_modules/semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "bin": { "semver": "bin/semver.js" } }, + "node_modules/@babel/helper-compilation-targets/node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true + }, "node_modules/@babel/helper-create-class-features-plugin": { "version": "7.14.4", "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.14.4.tgz", @@ -509,26 +501,10 @@ } }, "node_modules/@babel/helper-environment-visitor": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.16.7.tgz", - "integrity": "sha512-SLLb0AAn6PkUeAfKJCCOl9e1R53pQlGAfc4y4XuMRZfqeMYLE0dM1LMhqbGAlGQY0lfw5/ohoYWAe9V1yibRag==", + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.5.tgz", + "integrity": "sha512-XGmhECfVA/5sAt+H+xpSg0mfrHq6FzNr9Oxh7PSEBBRUb/mL7Kz3NICXb194rCqAEdxkhPT1a88teizAFyvk8Q==", "dev": true, - "dependencies": { - "@babel/types": "^7.16.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-environment-visitor/node_modules/@babel/types": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.17.0.tgz", - "integrity": "sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.16.7", - "to-fast-properties": "^2.0.0" - }, "engines": { "node": ">=6.9.0" } @@ -542,88 +518,31 @@ "@babel/types": "^7.13.0" } }, - "node_modules/@babel/helper-explode-assignable-expression/node_modules/@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - }, "node_modules/@babel/helper-function-name": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.16.7.tgz", - "integrity": "sha512-QfDfEnIUyyBSR3HtrtGECuZ6DAyCkYFp7GHl75vFtTnn6pjKeK0T1DB5lLkFvBea8MdaiUABx3osbgLyInoejA==", + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.22.5.tgz", + "integrity": "sha512-wtHSq6jMRE3uF2otvfuD3DIvVhOsSNshQl0Qrd7qC9oQJzHvOL4qQXlQn2916+CXGywIjpGuIkoyZRRxHPiNQQ==", "dev": true, "dependencies": { - "@babel/helper-get-function-arity": "^7.16.7", - "@babel/template": "^7.16.7", - "@babel/types": "^7.16.7" + "@babel/template": "^7.22.5", + "@babel/types": "^7.22.5" }, "engines": { "node": ">=6.9.0" } }, - "node_modules/@babel/helper-function-name/node_modules/@babel/types": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.17.0.tgz", - "integrity": "sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.16.7", - "to-fast-properties": "^2.0.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-get-function-arity": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.16.7.tgz", - "integrity": "sha512-flc+RLSOBXzNzVhcLu6ujeHUrD6tANAOU5ojrRx/as+tbzf8+stUCj7+IfRRoAbEZqj/ahXEMsjhOhgeZsrnTw==", - "dev": true, - "dependencies": { - "@babel/types": "^7.16.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-get-function-arity/node_modules/@babel/types": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.17.0.tgz", - "integrity": "sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==", + "node_modules/@babel/helper-hoist-variables": { + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz", + "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==", "dev": true, "dependencies": { - "@babel/helper-validator-identifier": "^7.16.7", - "to-fast-properties": "^2.0.0" + "@babel/types": "^7.22.5" }, "engines": { "node": ">=6.9.0" } }, - "node_modules/@babel/helper-hoist-variables": { - "version": "7.13.16", - "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.13.16.tgz", - "integrity": "sha512-1eMtTrXtrwscjcAeO4BVK+vvkxaLJSPFz1w1KLawz6HLNi9bPFGBNwwDyVfiu1Tv/vRRFYfoGaKhmAQPGPn5Wg==", - "dev": true, - "dependencies": { - "@babel/traverse": "^7.13.15", - "@babel/types": "^7.13.16" - } - }, - "node_modules/@babel/helper-hoist-variables/node_modules/@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - }, "node_modules/@babel/helper-member-expression-to-functions": { "version": "7.13.12", "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.13.12.tgz", @@ -633,57 +552,34 @@ "@babel/types": "^7.13.12" } }, - "node_modules/@babel/helper-member-expression-to-functions/node_modules/@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - }, "node_modules/@babel/helper-module-imports": { - "version": "7.13.12", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.13.12.tgz", - "integrity": "sha512-4cVvR2/1B693IuOvSI20xqqa/+bl7lqAMR59R4iu39R9aOX8/JoYY1sFaNvUMyMBGnHdwvJgUrzNLoUZxXypxA==", - "dependencies": { - "@babel/types": "^7.13.12" - } - }, - "node_modules/@babel/helper-module-imports/node_modules/@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.22.5.tgz", + "integrity": "sha512-8Dl6+HD/cKifutF5qGd/8ZJi84QeAKh+CEe1sBzz8UayBBGg1dAIJrdHOcOM5b2MpzWL2yuotJTtGjETq0qjXg==", "dependencies": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" + "@babel/types": "^7.22.5" + }, + "engines": { + "node": ">=6.9.0" } }, "node_modules/@babel/helper-module-transforms": { - "version": "7.14.2", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.14.2.tgz", - "integrity": "sha512-OznJUda/soKXv0XhpvzGWDnml4Qnwp16GN+D/kZIdLsWoHj05kyu8Rm5kXmMef+rVJZ0+4pSGLkeixdqNUATDA==", - "dev": true, - "dependencies": { - "@babel/helper-module-imports": "^7.13.12", - "@babel/helper-replace-supers": "^7.13.12", - "@babel/helper-simple-access": "^7.13.12", - "@babel/helper-split-export-declaration": "^7.12.13", - "@babel/helper-validator-identifier": "^7.14.0", - "@babel/template": "^7.12.13", - "@babel/traverse": "^7.14.2", - "@babel/types": "^7.14.2" - } - }, - "node_modules/@babel/helper-module-transforms/node_modules/@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", + "version": "7.22.9", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.22.9.tgz", + "integrity": "sha512-t+WA2Xn5K+rTeGtC8jCsdAH52bjggG5TKRuRrAGNM/mjIbO4GxvlLMFOEz9wXY5I2XQ60PMFsAG2WIcG82dQMQ==", "dev": true, "dependencies": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" + "@babel/helper-environment-visitor": "^7.22.5", + "@babel/helper-module-imports": "^7.22.5", + "@babel/helper-simple-access": "^7.22.5", + "@babel/helper-split-export-declaration": "^7.22.6", + "@babel/helper-validator-identifier": "^7.22.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" } }, "node_modules/@babel/helper-optimise-call-expression": { @@ -695,16 +591,6 @@ "@babel/types": "^7.12.13" } }, - "node_modules/@babel/helper-optimise-call-expression/node_modules/@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - }, "node_modules/@babel/helper-plugin-utils": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.0.0.tgz", @@ -722,16 +608,6 @@ "@babel/types": "^7.13.0" } }, - "node_modules/@babel/helper-remap-async-to-generator/node_modules/@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - }, "node_modules/@babel/helper-replace-supers": { "version": "7.14.4", "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.14.4.tgz", @@ -744,33 +620,16 @@ "@babel/types": "^7.14.4" } }, - "node_modules/@babel/helper-replace-supers/node_modules/@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - }, "node_modules/@babel/helper-simple-access": { - "version": "7.13.12", - "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.13.12.tgz", - "integrity": "sha512-7FEjbrx5SL9cWvXioDbnlYTppcZGuCY6ow3/D5vMggb2Ywgu4dMrpTJX0JdQAIcRRUElOIxF3yEooa9gUb9ZbA==", + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.22.5.tgz", + "integrity": "sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w==", "dev": true, "dependencies": { - "@babel/types": "^7.13.12" - } - }, - "node_modules/@babel/helper-simple-access/node_modules/@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" + "@babel/types": "^7.22.5" + }, + "engines": { + "node": ">=6.9.0" } }, "node_modules/@babel/helper-skip-transparent-expression-wrappers": { @@ -782,54 +641,42 @@ "@babel/types": "^7.12.1" } }, - "node_modules/@babel/helper-skip-transparent-expression-wrappers/node_modules/@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - }, "node_modules/@babel/helper-split-export-declaration": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.16.7.tgz", - "integrity": "sha512-xbWoy/PFoxSWazIToT9Sif+jJTlrMcndIsaOKvTA6u7QEo7ilkRZpjew18/W3c7nm8fXdUDXh02VXTbZ0pGDNw==", + "version": "7.22.6", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz", + "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==", "dev": true, "dependencies": { - "@babel/types": "^7.16.7" + "@babel/types": "^7.22.5" }, "engines": { "node": ">=6.9.0" } }, - "node_modules/@babel/helper-split-export-declaration/node_modules/@babel/types": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.17.0.tgz", - "integrity": "sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.16.7", - "to-fast-properties": "^2.0.0" - }, + "node_modules/@babel/helper-string-parser": { + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz", + "integrity": "sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.19.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz", - "integrity": "sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==", + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.5.tgz", + "integrity": "sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ==", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-option": { - "version": "7.12.17", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.12.17.tgz", - "integrity": "sha512-TopkMDmLzq8ngChwRlyjR6raKD6gMSae4JdYDB8bByKreQgG0RBTuKe9LRxW3wFtUnjxOPRKBDwEH6Mg5KeDfw==", - "dev": true + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.22.5.tgz", + "integrity": "sha512-R3oB6xlIVKUnxNUxbmgq7pKjxpru24zlimpE8WK47fACIlM0II/Hm1RS8IaOI7NgCr6LNS+jl5l75m20npAziw==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } }, "node_modules/@babel/helper-wrap-function": { "version": "7.13.0", @@ -843,44 +690,27 @@ "@babel/types": "^7.13.0" } }, - "node_modules/@babel/helper-wrap-function/node_modules/@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - }, "node_modules/@babel/helpers": { - "version": "7.14.0", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.14.0.tgz", - "integrity": "sha512-+ufuXprtQ1D1iZTO/K9+EBRn+qPWMJjZSw/S0KlFrxCw4tkrzv9grgpDHkY9MeQTjTY8i2sp7Jep8DfU6tN9Mg==", - "dev": true, - "dependencies": { - "@babel/template": "^7.12.13", - "@babel/traverse": "^7.14.0", - "@babel/types": "^7.14.0" - } - }, - "node_modules/@babel/helpers/node_modules/@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", + "version": "7.22.11", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.22.11.tgz", + "integrity": "sha512-vyOXC8PBWaGc5h7GMsNx68OH33cypkEDJCHvYVVgVbbxJDROYVtexSk0gK5iCF1xNjRIN2s8ai7hwkWDq5szWg==", "dev": true, "dependencies": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" + "@babel/template": "^7.22.5", + "@babel/traverse": "^7.22.11", + "@babel/types": "^7.22.11" + }, + "engines": { + "node": ">=6.9.0" } }, "node_modules/@babel/highlight": { - "version": "7.18.6", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.18.6.tgz", - "integrity": "sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==", + "version": "7.22.10", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.10.tgz", + "integrity": "sha512-78aUtVcT7MUscr0K5mIEnkwxPE0MaxkR5RxRwuHaQ+JuU5AmTPhY+do2mdzVTnIJJpyBglql2pehuBIWHug+WQ==", "dependencies": { - "@babel/helper-validator-identifier": "^7.18.6", - "chalk": "^2.0.0", + "@babel/helper-validator-identifier": "^7.22.5", + "chalk": "^2.4.2", "js-tokens": "^4.0.0" }, "engines": { @@ -888,9 +718,9 @@ } }, "node_modules/@babel/parser": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.17.0.tgz", - "integrity": "sha512-VKXSCQx5D8S04ej+Dqsr1CzYvvWgf20jIw2D+YhQCrIlr2UZGaDds23Y0xg75/skOxpLCRpUZvk/1EAVkGoDOw==", + "version": "7.22.11", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.22.11.tgz", + "integrity": "sha512-R5zb8eJIBPJriQtbH/htEQy4k7E2dHWlD2Y2VT07JCzwYZHBxV5ZYtM0UhXSNMT74LyxuM+b1jdL7pSesXbC/g==", "dev": true, "bin": { "parser": "bin/babel-parser.js" @@ -2211,16 +2041,6 @@ "integrity": "sha512-ZPafIPSwzUlAoWT8DKs1W2VyF2gOWthGd5NGFMsBcMMol+ZhK+EQY/e6V96poa6PA/Bh+C9plWN0hXO1uB8AfQ==", "dev": true }, - "node_modules/@babel/preset-env/node_modules/@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - }, "node_modules/@babel/preset-env/node_modules/semver": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", @@ -2258,58 +2078,33 @@ } }, "node_modules/@babel/template": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.16.7.tgz", - "integrity": "sha512-I8j/x8kHUrbYRTUxXrrMbfCa7jxkE7tZre39x3kjr9hvI82cK1FfqLygotcWN5kdPGWcLdWMHpSBavse5tWw3w==", + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.5.tgz", + "integrity": "sha512-X7yV7eiwAxdj9k94NEylvbVHLiVG1nvzCV2EAowhxLTwODV1jl9UzZ48leOC0sH7OnuHrIkllaBgneUykIcZaw==", "dev": true, "dependencies": { - "@babel/code-frame": "^7.16.7", - "@babel/parser": "^7.16.7", - "@babel/types": "^7.16.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/template/node_modules/@babel/code-frame": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.16.7.tgz", - "integrity": "sha512-iAXqUn8IIeBTNd72xsFlgaXHkMBMt6y4HJp1tIaK465CWLT/fG1aqB7ykr95gHHmlBdGbFeWWfyB4NJJ0nmeIg==", - "dev": true, - "dependencies": { - "@babel/highlight": "^7.16.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/template/node_modules/@babel/types": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.17.0.tgz", - "integrity": "sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.16.7", - "to-fast-properties": "^2.0.0" + "@babel/code-frame": "^7.22.5", + "@babel/parser": "^7.22.5", + "@babel/types": "^7.22.5" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.17.0.tgz", - "integrity": "sha512-fpFIXvqD6kC7c7PUNnZ0Z8cQXlarCLtCUpt2S1Dx7PjoRtCFffvOkHHSom+m5HIxMZn5bIBVb71lhabcmjEsqg==", - "dev": true, - "dependencies": { - "@babel/code-frame": "^7.16.7", - "@babel/generator": "^7.17.0", - "@babel/helper-environment-visitor": "^7.16.7", - "@babel/helper-function-name": "^7.16.7", - "@babel/helper-hoist-variables": "^7.16.7", - "@babel/helper-split-export-declaration": "^7.16.7", - "@babel/parser": "^7.17.0", - "@babel/types": "^7.17.0", + "version": "7.22.11", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.22.11.tgz", + "integrity": "sha512-mzAenteTfomcB7mfPtyi+4oe5BZ6MXxWcn4CX+h4IRJ+OOGXBrWU6jDQavkQI9Vuc5P+donFabBfFCcmWka9lQ==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.22.10", + "@babel/generator": "^7.22.10", + "@babel/helper-environment-visitor": "^7.22.5", + "@babel/helper-function-name": "^7.22.5", + "@babel/helper-hoist-variables": "^7.22.5", + "@babel/helper-split-export-declaration": "^7.22.6", + "@babel/parser": "^7.22.11", + "@babel/types": "^7.22.11", "debug": "^4.1.0", "globals": "^11.1.0" }, @@ -2317,43 +2112,6 @@ "node": ">=6.9.0" } }, - "node_modules/@babel/traverse/node_modules/@babel/code-frame": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.16.7.tgz", - "integrity": "sha512-iAXqUn8IIeBTNd72xsFlgaXHkMBMt6y4HJp1tIaK465CWLT/fG1aqB7ykr95gHHmlBdGbFeWWfyB4NJJ0nmeIg==", - "dev": true, - "dependencies": { - "@babel/highlight": "^7.16.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse/node_modules/@babel/helper-hoist-variables": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.16.7.tgz", - "integrity": "sha512-m04d/0Op34H5v7pbZw6pSKP7weA6lsMvfiIAMeIvkY/R4xQtBSMFEigu9QTZ2qB/9l22vsxtM8a+Q8CzD255fg==", - "dev": true, - "dependencies": { - "@babel/types": "^7.16.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse/node_modules/@babel/types": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.17.0.tgz", - "integrity": "sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.16.7", - "to-fast-properties": "^2.0.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/@babel/traverse/node_modules/debug": { "version": "4.3.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", @@ -2378,14 +2136,16 @@ "dev": true }, "node_modules/@babel/types": { - "version": "7.5.5", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.5.5.tgz", - "integrity": "sha512-s63F9nJioLqOlW3UkyMd+BYhXt44YuaFm/VV0VwuteqjYwRrObkU7ra9pY4wAJR3oXi8hJrMcrcJdO/HH33vtw==", - "dev": true, + "version": "7.22.11", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.22.11.tgz", + "integrity": "sha512-siazHiGuZRz9aB9NpHy9GOs9xiQPKnMzgdr493iI1M67vRXpnEq8ZOOKzezC5q7zwuQ6sDhdSp4SD9ixKSqKZg==", "dependencies": { - "esutils": "^2.0.2", - "lodash": "^4.17.13", + "@babel/helper-string-parser": "^7.22.5", + "@babel/helper-validator-identifier": "^7.22.5", "to-fast-properties": "^2.0.0" + }, + "engines": { + "node": ">=6.9.0" } }, "node_modules/@bcoe/v8-coverage": { @@ -2584,9 +2344,9 @@ } }, "node_modules/@druid-toolkit/query": { - "version": "0.20.5", - "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.20.5.tgz", - "integrity": "sha512-EY0131z611tklnui+vyRqsoPjTBbonkF7WwsNvT0KsBQYm5qtuvX/QlXGfX66f4KQzoo5G/4dRIVmZ9JbSRgzw==", + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.21.1.tgz", + "integrity": "sha512-p9bE0mlE0Lv6w1HOOr6m/NB1l4oQy0ew6HFa3hSZ1T/qlDx8CPT5MXRRhruEmgtneZx4UrN6RqFULJNFVv+aWg==", "dependencies": { "tslib": "^2.5.2" } @@ -3823,35 +3583,6 @@ "node": ">=8" } }, - "node_modules/@jest/reporters/node_modules/jest-worker": { - "version": "27.5.0", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.0.tgz", - "integrity": "sha512-8OEHiPNOPTfaWnJ2SUHM8fmgeGq37uuGsQBvGKQJl1f+6WIy6g7G3fE2ruI5294bUKUI9FaCWt5hDvO8HSwsSg==", - "dev": true, - "dependencies": { - "@types/node": "*", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "engines": { - "node": ">= 10.13.0" - } - }, - "node_modules/@jest/reporters/node_modules/jest-worker/node_modules/supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", - "dev": true, - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, "node_modules/@jest/reporters/node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -4228,9 +3959,9 @@ } }, "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", - "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz", + "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==", "dev": true, "dependencies": { "@jridgewell/set-array": "^1.0.1", @@ -4260,9 +3991,9 @@ } }, "node_modules/@jridgewell/source-map": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.2.tgz", - "integrity": "sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw==", + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.5.tgz", + "integrity": "sha512-UTYAUj/wviwdsMfzoSJspJxbkH5o1snzwX0//0ENX1u/55kkZZkcTZP6u9bwKGkv+dkk9at4m1Cpt0uY80kcpQ==", "dev": true, "dependencies": { "@jridgewell/gen-mapping": "^0.3.0", @@ -4655,9 +4386,9 @@ } }, "node_modules/@types/eslint": { - "version": "7.2.10", - "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-7.2.10.tgz", - "integrity": "sha512-kUEPnMKrqbtpCq/KTaGFFKAcz6Ethm2EjCoKIDaCmfRBWLbFuTcOJfTlorwbnboXBzahqWLgUp1BQeKHiJzPUQ==", + "version": "8.44.2", + "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.44.2.tgz", + "integrity": "sha512-sdPRb9K6iL5XZOmBubg8yiFp5yS/JdUDQsq5e6h95km91MCYMuvp7mh1fjPEYUhvHepKpZOjnEaMBR4PxjWDzg==", "dev": true, "dependencies": { "@types/estree": "*", @@ -4665,9 +4396,9 @@ } }, "node_modules/@types/eslint-scope": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.0.tgz", - "integrity": "sha512-O/ql2+rrCUe2W2rs7wMR+GqPRcgB6UiqN5RhrR5xruFlY7l9YLMn0ZkDzjoHLeiFkR8MCQZVudUuuvQ2BLC9Qw==", + "version": "3.7.4", + "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.4.tgz", + "integrity": "sha512-9K4zoImiZc3HlIp6AVUDE4CWYx22a+lhSZMYNpbjW04+YF0KWj4pJXnEMjdnFTiQibFFmElcsasJXDbdI/EPhA==", "dev": true, "dependencies": { "@types/eslint": "*", @@ -4675,9 +4406,9 @@ } }, "node_modules/@types/estree": { - "version": "0.0.46", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.46.tgz", - "integrity": "sha512-laIjwTQaD+5DukBZaygQ79K1Z0jb1bPEMRrkXSLjtCcZm+abyp5YbrqpSLzD42FwWW6gK/aS4NYpJ804nG2brg==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.1.tgz", + "integrity": "sha512-LG4opVs2ANWZ1TJoKc937iMmNstM/d0ae1vNbnBvBhqCSezgVUOzcLCqbI5elV8Vy6WKwKjaqR+zO9VKirBBCA==", "dev": true }, "node_modules/@types/file-saver": { @@ -5290,148 +5021,148 @@ } }, "node_modules/@webassemblyjs/ast": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.0.tgz", - "integrity": "sha512-kX2W49LWsbthrmIRMbQZuQDhGtjyqXfEmmHyEi4XWnSZtPmxY0+3anPIzsnRb45VH/J55zlOfWvZuY47aJZTJg==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.6.tgz", + "integrity": "sha512-IN1xI7PwOvLPgjcf180gC1bqn3q/QaOCwYUahIOhbYUu8KA/3tw2RT/T0Gidi1l7Hhj5D/INhJxiICObqpMu4Q==", "dev": true, "dependencies": { - "@webassemblyjs/helper-numbers": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0" + "@webassemblyjs/helper-numbers": "1.11.6", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6" } }, "node_modules/@webassemblyjs/floating-point-hex-parser": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.0.tgz", - "integrity": "sha512-Q/aVYs/VnPDVYvsCBL/gSgwmfjeCb4LW8+TMrO3cSzJImgv8lxxEPM2JA5jMrivE7LSz3V+PFqtMbls3m1exDA==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz", + "integrity": "sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw==", "dev": true }, "node_modules/@webassemblyjs/helper-api-error": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.0.tgz", - "integrity": "sha512-baT/va95eXiXb2QflSx95QGT5ClzWpGaa8L7JnJbgzoYeaA27FCvuBXU758l+KXWRndEmUXjP0Q5fibhavIn8w==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz", + "integrity": "sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q==", "dev": true }, "node_modules/@webassemblyjs/helper-buffer": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.0.tgz", - "integrity": "sha512-u9HPBEl4DS+vA8qLQdEQ6N/eJQ7gT7aNvMIo8AAWvAl/xMrcOSiI2M0MAnMCy3jIFke7bEee/JwdX1nUpCtdyA==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.6.tgz", + "integrity": "sha512-z3nFzdcp1mb8nEOFFk8DrYLpHvhKC3grJD2ardfKOzmbmJvEf/tPIqCY+sNcwZIY8ZD7IkB2l7/pqhUhqm7hLA==", "dev": true }, "node_modules/@webassemblyjs/helper-numbers": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.0.tgz", - "integrity": "sha512-DhRQKelIj01s5IgdsOJMKLppI+4zpmcMQ3XboFPLwCpSNH6Hqo1ritgHgD0nqHeSYqofA6aBN/NmXuGjM1jEfQ==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz", + "integrity": "sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==", "dev": true, "dependencies": { - "@webassemblyjs/floating-point-hex-parser": "1.11.0", - "@webassemblyjs/helper-api-error": "1.11.0", + "@webassemblyjs/floating-point-hex-parser": "1.11.6", + "@webassemblyjs/helper-api-error": "1.11.6", "@xtuc/long": "4.2.2" } }, "node_modules/@webassemblyjs/helper-wasm-bytecode": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.0.tgz", - "integrity": "sha512-MbmhvxXExm542tWREgSFnOVo07fDpsBJg3sIl6fSp9xuu75eGz5lz31q7wTLffwL3Za7XNRCMZy210+tnsUSEA==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz", + "integrity": "sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA==", "dev": true }, "node_modules/@webassemblyjs/helper-wasm-section": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.0.tgz", - "integrity": "sha512-3Eb88hcbfY/FCukrg6i3EH8H2UsD7x8Vy47iVJrP967A9JGqgBVL9aH71SETPx1JrGsOUVLo0c7vMCN22ytJew==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.6.tgz", + "integrity": "sha512-LPpZbSOwTpEC2cgn4hTydySy1Ke+XEu+ETXuoyvuyezHO3Kjdu90KK95Sh9xTbmjrCsUwvWwCOQQNta37VrS9g==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0" + "@webassemblyjs/ast": "1.11.6", + "@webassemblyjs/helper-buffer": "1.11.6", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6", + "@webassemblyjs/wasm-gen": "1.11.6" } }, "node_modules/@webassemblyjs/ieee754": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.0.tgz", - "integrity": "sha512-KXzOqpcYQwAfeQ6WbF6HXo+0udBNmw0iXDmEK5sFlmQdmND+tr773Ti8/5T/M6Tl/413ArSJErATd8In3B+WBA==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.6.tgz", + "integrity": "sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg==", "dev": true, "dependencies": { "@xtuc/ieee754": "^1.2.0" } }, "node_modules/@webassemblyjs/leb128": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.0.tgz", - "integrity": "sha512-aqbsHa1mSQAbeeNcl38un6qVY++hh8OpCOzxhixSYgbRfNWcxJNJQwe2rezK9XEcssJbbWIkblaJRwGMS9zp+g==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.6.tgz", + "integrity": "sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ==", "dev": true, "dependencies": { "@xtuc/long": "4.2.2" } }, "node_modules/@webassemblyjs/utf8": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.0.tgz", - "integrity": "sha512-A/lclGxH6SpSLSyFowMzO/+aDEPU4hvEiooCMXQPcQFPPJaYcPQNKGOCLUySJsYJ4trbpr+Fs08n4jelkVTGVw==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.6.tgz", + "integrity": "sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA==", "dev": true }, "node_modules/@webassemblyjs/wasm-edit": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.0.tgz", - "integrity": "sha512-JHQ0damXy0G6J9ucyKVXO2j08JVJ2ntkdJlq1UTiUrIgfGMmA7Ik5VdC/L8hBK46kVJgujkBIoMtT8yVr+yVOQ==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.6.tgz", + "integrity": "sha512-Ybn2I6fnfIGuCR+Faaz7YcvtBKxvoLV3Lebn1tM4o/IAJzmi9AWYIPWpyBfU8cC+JxAO57bk4+zdsTjJR+VTOw==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/helper-wasm-section": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0", - "@webassemblyjs/wasm-opt": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0", - "@webassemblyjs/wast-printer": "1.11.0" + "@webassemblyjs/ast": "1.11.6", + "@webassemblyjs/helper-buffer": "1.11.6", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6", + "@webassemblyjs/helper-wasm-section": "1.11.6", + "@webassemblyjs/wasm-gen": "1.11.6", + "@webassemblyjs/wasm-opt": "1.11.6", + "@webassemblyjs/wasm-parser": "1.11.6", + "@webassemblyjs/wast-printer": "1.11.6" } }, "node_modules/@webassemblyjs/wasm-gen": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.0.tgz", - "integrity": "sha512-BEUv1aj0WptCZ9kIS30th5ILASUnAPEvE3tVMTrItnZRT9tXCLW2LEXT8ezLw59rqPP9klh9LPmpU+WmRQmCPQ==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.6.tgz", + "integrity": "sha512-3XOqkZP/y6B4F0PBAXvI1/bky7GryoogUtfwExeP/v7Nzwo1QLcq5oQmpKlftZLbT+ERUOAZVQjuNVak6UXjPA==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/ieee754": "1.11.0", - "@webassemblyjs/leb128": "1.11.0", - "@webassemblyjs/utf8": "1.11.0" + "@webassemblyjs/ast": "1.11.6", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6", + "@webassemblyjs/ieee754": "1.11.6", + "@webassemblyjs/leb128": "1.11.6", + "@webassemblyjs/utf8": "1.11.6" } }, "node_modules/@webassemblyjs/wasm-opt": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.0.tgz", - "integrity": "sha512-tHUSP5F4ywyh3hZ0+fDQuWxKx3mJiPeFufg+9gwTpYp324mPCQgnuVKwzLTZVqj0duRDovnPaZqDwoyhIO8kYg==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.6.tgz", + "integrity": "sha512-cOrKuLRE7PCe6AsOVl7WasYf3wbSo4CeOk6PkrjS7g57MFfVUF9u6ysQBBODX0LdgSvQqRiGz3CXvIDKcPNy4g==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0" + "@webassemblyjs/ast": "1.11.6", + "@webassemblyjs/helper-buffer": "1.11.6", + "@webassemblyjs/wasm-gen": "1.11.6", + "@webassemblyjs/wasm-parser": "1.11.6" } }, "node_modules/@webassemblyjs/wasm-parser": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.0.tgz", - "integrity": "sha512-6L285Sgu9gphrcpDXINvm0M9BskznnzJTE7gYkjDbxET28shDqp27wpruyx3C2S/dvEwiigBwLA1cz7lNUi0kw==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.6.tgz", + "integrity": "sha512-6ZwPeGzMJM3Dqp3hCsLgESxBGtT/OeCvCZ4TA1JUPYgmhAx38tTPR9JaKy0S5H3evQpO/h2uWs2j6Yc/fjkpTQ==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-api-error": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/ieee754": "1.11.0", - "@webassemblyjs/leb128": "1.11.0", - "@webassemblyjs/utf8": "1.11.0" + "@webassemblyjs/ast": "1.11.6", + "@webassemblyjs/helper-api-error": "1.11.6", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6", + "@webassemblyjs/ieee754": "1.11.6", + "@webassemblyjs/leb128": "1.11.6", + "@webassemblyjs/utf8": "1.11.6" } }, "node_modules/@webassemblyjs/wast-printer": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.0.tgz", - "integrity": "sha512-Fg5OX46pRdTgB7rKIUojkh9vXaVN6sGYCnEiJN1GYkb0RPwShZXp6KTDqmoMdQPKhcroOXh3fEzmkWmCYaKYhQ==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.6.tgz", + "integrity": "sha512-JM7AhRcE+yW2GWYaKeHL5vt4xqee5N2WcezptmgyhNS+ScggqcT1OtXykhAb13Sn5Yas0j2uv9tHgrjwvzAP4A==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", + "@webassemblyjs/ast": "1.11.6", "@xtuc/long": "4.2.2" } }, @@ -5514,9 +5245,9 @@ "integrity": "sha512-NBOQlm9+7RBqRqZwimpgquaLeTJFayqb9UEPtTkpC3TkkwDnlsT/TwsCC0svjt9kEZ6G9mH5AEOHSz6Q/HrzQQ==" }, "node_modules/acorn": { - "version": "8.8.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz", - "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==", + "version": "8.10.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz", + "integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==", "dev": true, "bin": { "acorn": "bin/acorn" @@ -5556,6 +5287,15 @@ "node": ">=0.4.0" } }, + "node_modules/acorn-import-assertions": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz", + "integrity": "sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==", + "dev": true, + "peerDependencies": { + "acorn": "^8" + } + }, "node_modules/acorn-jsx": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", @@ -6602,9 +6342,9 @@ "dev": true }, "node_modules/browserslist": { - "version": "4.21.5", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.5.tgz", - "integrity": "sha512-tUkiguQGW7S3IhB7N+c2MV/HZPSCPAAiYBZXLsBhFB/PCy6ZKKsZrmBayHV9fdGV/ARIfJ14NkxKzRDjvp7L6w==", + "version": "4.21.10", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.10.tgz", + "integrity": "sha512-bipEBdZfVH5/pwrvqc+Ub0kUPVfGUhlKxbvfD+z1BDnPEO/X98ruXGA1WP5ASpAFKan7Qr6j736IacbZQuAlKQ==", "dev": true, "funding": [ { @@ -6614,13 +6354,17 @@ { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" } ], "dependencies": { - "caniuse-lite": "^1.0.30001449", - "electron-to-chromium": "^1.4.284", - "node-releases": "^2.0.8", - "update-browserslist-db": "^1.0.10" + "caniuse-lite": "^1.0.30001517", + "electron-to-chromium": "^1.4.477", + "node-releases": "^2.0.13", + "update-browserslist-db": "^1.0.11" }, "bin": { "browserslist": "cli.js" @@ -6771,9 +6515,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001466", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001466.tgz", - "integrity": "sha512-ewtFBSfWjEmxUgNBSZItFSmVtvk9zkwkl1OfRZlKA8slltRN+/C/tuGVrF9styXkN36Yu3+SeJ1qkXxDEyNZ5w==", + "version": "1.0.30001522", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001522.tgz", + "integrity": "sha512-TKiyTVZxJGhsTszLuzb+6vUZSjVOAhClszBr2Ta2k9IwtNBT/4dzmL6aywt0HCgEZlmwJzXJd8yNiob6HgwTRg==", "dev": true, "funding": [ { @@ -6783,6 +6527,10 @@ { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" } ] }, @@ -8205,9 +7953,9 @@ "dev": true }, "node_modules/electron-to-chromium": { - "version": "1.4.330", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.330.tgz", - "integrity": "sha512-PqyefhybrVdjAJ45HaPLtuVaehiSw7C3ya0aad+rvmV53IVyXmYRk3pwIOb2TxTDTnmgQdn46NjMMaysx79/6Q==", + "version": "1.4.501", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.501.tgz", + "integrity": "sha512-NCF5hZUg73MEP0guvIM+BjPs9W07UeAuc5XCNqRZZTKJxLjE0ZS/Zo5UsV8bbs2y/jeKRPFPzdWdBfOGEZTXKg==", "dev": true }, "node_modules/emittery": { @@ -8265,9 +8013,9 @@ } }, "node_modules/enhanced-resolve": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.12.0.tgz", - "integrity": "sha512-QHTXI/sZQmko1cbDoNAa3mJ5qhWUUNAq3vR0/YiD379fWQrcfuoX1+HW2S0MTt7XmoPLapdaDKUtelUSPic7hQ==", + "version": "5.15.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.15.0.tgz", + "integrity": "sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg==", "dev": true, "dependencies": { "graceful-fs": "^4.2.4", @@ -8402,9 +8150,9 @@ "dev": true }, "node_modules/es-module-lexer": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.4.1.tgz", - "integrity": "sha512-ooYciCUtfw6/d2w56UVeqHPcoCFAiJdz5XOkYpv/Txl1HMUozpXjz/2RIQgqwKdXNDPSF1W7mJCFse3G+HDyAA==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.3.0.tgz", + "integrity": "sha512-vZK7T0N2CBmBOixhmjdqx2gWVbFZ4DXZ/NyRMZVlJXPa7CyFS+/a4QQsDGDQy9ZfEzxFuNEsMLeQJnKP2p5/JA==", "dev": true }, "node_modules/es-set-tostringtag": { @@ -13410,35 +13158,6 @@ "node": ">=0.12.0" } }, - "node_modules/jest-haste-map/node_modules/jest-worker": { - "version": "27.5.0", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.0.tgz", - "integrity": "sha512-8OEHiPNOPTfaWnJ2SUHM8fmgeGq37uuGsQBvGKQJl1f+6WIy6g7G3fE2ruI5294bUKUI9FaCWt5hDvO8HSwsSg==", - "dev": true, - "dependencies": { - "@types/node": "*", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "engines": { - "node": ">= 10.13.0" - } - }, - "node_modules/jest-haste-map/node_modules/jest-worker/node_modules/supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", - "dev": true, - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, "node_modules/jest-haste-map/node_modules/micromatch": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.4.tgz", @@ -13726,18 +13445,6 @@ "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" } }, - "node_modules/jest-message-util/node_modules/@babel/code-frame": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.16.7.tgz", - "integrity": "sha512-iAXqUn8IIeBTNd72xsFlgaXHkMBMt6y4HJp1tIaK465CWLT/fG1aqB7ykr95gHHmlBdGbFeWWfyB4NJJ0nmeIg==", - "dev": true, - "dependencies": { - "@babel/highlight": "^7.16.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/jest-message-util/node_modules/@jest/types": { "version": "27.5.0", "resolved": "https://registry.npmjs.org/@jest/types/-/types-27.5.0.tgz", @@ -14410,35 +14117,6 @@ "node": ">=8" } }, - "node_modules/jest-runner/node_modules/jest-worker": { - "version": "27.5.0", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.0.tgz", - "integrity": "sha512-8OEHiPNOPTfaWnJ2SUHM8fmgeGq37uuGsQBvGKQJl1f+6WIy6g7G3fE2ruI5294bUKUI9FaCWt5hDvO8HSwsSg==", - "dev": true, - "dependencies": { - "@types/node": "*", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "engines": { - "node": ">= 10.13.0" - } - }, - "node_modules/jest-runner/node_modules/jest-worker/node_modules/supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", - "dev": true, - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, "node_modules/jest-runner/node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -15257,14 +14935,14 @@ } }, "node_modules/jest-worker": { - "version": "26.6.2", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-26.6.2.tgz", - "integrity": "sha512-KWYVV1c4i+jbMpaBC+U++4Va0cp8OisU185o73T1vo99hqi7w8tSJfUXYswwqqrjzwxa6KpRK54WhPvwf5w6PQ==", + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz", + "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", "dev": true, "dependencies": { "@types/node": "*", "merge-stream": "^2.0.0", - "supports-color": "^7.0.0" + "supports-color": "^8.0.0" }, "engines": { "node": ">= 10.13.0" @@ -15280,15 +14958,18 @@ } }, "node_modules/jest-worker/node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", "dev": true, "dependencies": { "has-flag": "^4.0.0" }, "engines": { - "node": ">=8" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" } }, "node_modules/jest/node_modules/@jest/types": { @@ -16647,9 +16328,9 @@ "dev": true }, "node_modules/node-releases": { - "version": "2.0.10", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.10.tgz", - "integrity": "sha512-5GFldHPXVG/YZmFzJvKK2zDSzPKhEp0+ZR5SVaoSag9fsL5YgHbUHDfnG5494ISANDcK4KwPXAx2xqVEydmd7w==", + "version": "2.0.13", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.13.tgz", + "integrity": "sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==", "dev": true }, "node_modules/nopt": { @@ -20617,9 +20298,9 @@ } }, "node_modules/serialize-javascript": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", - "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.1.tgz", + "integrity": "sha512-owoXEFjWRllis8/M1Q+Cw5k8ZH40e3zhp/ovX+Xr/vi1qj6QesbyXXViFbpNvWvPNAD62SutwEXavefrLJWj7w==", "dev": true, "dependencies": { "randombytes": "^2.1.0" @@ -21094,12 +20775,6 @@ "uuid": "bin/uuid" } }, - "node_modules/source-list-map": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz", - "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw==", - "dev": true - }, "node_modules/source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", @@ -22275,13 +21950,13 @@ } }, "node_modules/terser": { - "version": "5.15.0", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.15.0.tgz", - "integrity": "sha512-L1BJiXVmheAQQy+as0oF3Pwtlo4s3Wi1X2zNZ2NxOB4wx9bdS9Vk67XQENLFdLYGCK/Z2di53mTj/hBafR+dTA==", + "version": "5.19.2", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.19.2.tgz", + "integrity": "sha512-qC5+dmecKJA4cpYxRa5aVkKehYsQKc+AHeKl0Oe62aYjBL8ZA33tTljktDHJSaxxMnbI5ZYw+o/S2DxxLu8OfA==", "dev": true, "dependencies": { - "@jridgewell/source-map": "^0.3.2", - "acorn": "^8.5.0", + "@jridgewell/source-map": "^0.3.3", + "acorn": "^8.8.2", "commander": "^2.20.0", "source-map-support": "~0.5.20" }, @@ -22293,17 +21968,16 @@ } }, "node_modules/terser-webpack-plugin": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.1.1.tgz", - "integrity": "sha512-5XNNXZiR8YO6X6KhSGXfY0QrGrCRlSwAEjIIrlRQR4W8nP69TaJUlh3bkuac6zzgspiGPfKEHcY295MMVExl5Q==", + "version": "5.3.9", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.9.tgz", + "integrity": "sha512-ZuXsqE07EcggTWQjXUj+Aot/OMcD0bMKGgF63f7UxYcu5/AJF53aIpK1YoP5xR9l6s/Hy2b+t1AM0bLNPRuhwA==", "dev": true, "dependencies": { - "jest-worker": "^26.6.2", - "p-limit": "^3.1.0", - "schema-utils": "^3.0.0", - "serialize-javascript": "^5.0.1", - "source-map": "^0.6.1", - "terser": "^5.5.1" + "@jridgewell/trace-mapping": "^0.3.17", + "jest-worker": "^27.4.5", + "schema-utils": "^3.1.1", + "serialize-javascript": "^6.0.1", + "terser": "^5.16.8" }, "engines": { "node": ">= 10.13.0" @@ -22314,30 +21988,36 @@ }, "peerDependencies": { "webpack": "^5.1.0" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "uglify-js": { + "optional": true + } } }, - "node_modules/terser-webpack-plugin/node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "node_modules/terser-webpack-plugin/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.19", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz", + "integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==", "dev": true, "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" } }, "node_modules/terser-webpack-plugin/node_modules/schema-utils": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.0.0.tgz", - "integrity": "sha512-6D82/xSzO094ajanoOSbe4YvXWMfn2A//8Y1+MUqFAJul5Bs+yn36xbK9OtNDcRVSBJ9jjeoXftM6CfztsjOAA==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", + "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==", "dev": true, "dependencies": { - "@types/json-schema": "^7.0.6", + "@types/json-schema": "^7.0.8", "ajv": "^6.12.5", "ajv-keywords": "^3.5.2" }, @@ -22349,15 +22029,6 @@ "url": "https://opencollective.com/webpack" } }, - "node_modules/terser-webpack-plugin/node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -23185,9 +22856,9 @@ } }, "node_modules/update-browserslist-db": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz", - "integrity": "sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==", + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz", + "integrity": "sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA==", "dev": true, "funding": [ { @@ -23197,6 +22868,10 @@ { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" } ], "dependencies": { @@ -23204,7 +22879,7 @@ "picocolors": "^1.0.0" }, "bin": { - "browserslist-lint": "cli.js" + "update-browserslist-db": "cli.js" }, "peerDependencies": { "browserslist": ">= 4.21.0" @@ -23467,9 +23142,9 @@ } }, "node_modules/watchpack": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.1.1.tgz", - "integrity": "sha512-Oo7LXCmc1eE1AjyuSBmtC3+Wy4HcV8PxWh2kP6fOl8yTlNS7r0K9l1ao2lrrUza7V39Y3D/BbJgY8VeSlc5JKw==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz", + "integrity": "sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==", "dev": true, "dependencies": { "glob-to-regexp": "^0.4.1", @@ -23498,34 +23173,35 @@ } }, "node_modules/webpack": { - "version": "5.33.2", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.33.2.tgz", - "integrity": "sha512-X4b7F1sYBmJx8mlh2B7mV5szEkE0jYNJ2y3akgAP0ERi0vLCG1VvdsIxt8lFd4st6SUy0lf7W0CCQS566MBpJg==", - "dev": true, - "dependencies": { - "@types/eslint-scope": "^3.7.0", - "@types/estree": "^0.0.46", - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/wasm-edit": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0", - "acorn": "^8.0.4", + "version": "5.88.2", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.88.2.tgz", + "integrity": "sha512-JmcgNZ1iKj+aiR0OvTYtWQqJwq37Pf683dY9bVORwVbUrDhLhdn/PlO2sHsFHPkj7sHNQF3JwaAkp49V+Sq1tQ==", + "dev": true, + "dependencies": { + "@types/eslint-scope": "^3.7.3", + "@types/estree": "^1.0.0", + "@webassemblyjs/ast": "^1.11.5", + "@webassemblyjs/wasm-edit": "^1.11.5", + "@webassemblyjs/wasm-parser": "^1.11.5", + "acorn": "^8.7.1", + "acorn-import-assertions": "^1.9.0", "browserslist": "^4.14.5", "chrome-trace-event": "^1.0.2", - "enhanced-resolve": "^5.7.0", - "es-module-lexer": "^0.4.0", - "eslint-scope": "^5.1.1", + "enhanced-resolve": "^5.15.0", + "es-module-lexer": "^1.2.1", + "eslint-scope": "5.1.1", "events": "^3.2.0", "glob-to-regexp": "^0.4.1", - "graceful-fs": "^4.2.4", - "json-parse-better-errors": "^1.0.2", + "graceful-fs": "^4.2.9", + "json-parse-even-better-errors": "^2.3.1", "loader-runner": "^4.2.0", "mime-types": "^2.1.27", "neo-async": "^2.6.2", - "schema-utils": "^3.0.0", + "schema-utils": "^3.2.0", "tapable": "^2.1.1", - "terser-webpack-plugin": "^5.1.1", - "watchpack": "^2.0.0", - "webpack-sources": "^2.1.1" + "terser-webpack-plugin": "^5.3.7", + "watchpack": "^2.4.0", + "webpack-sources": "^3.2.3" }, "bin": { "webpack": "bin/webpack.js" @@ -24090,27 +23766,14 @@ } }, "node_modules/webpack-sources": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-2.2.0.tgz", - "integrity": "sha512-bQsA24JLwcnWGArOKUxYKhX3Mz/nK1Xf6hxullKERyktjNMC4x8koOeaDNTA2fEJ09BdWLbM/iTW0ithREUP0w==", + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz", + "integrity": "sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==", "dev": true, - "dependencies": { - "source-list-map": "^2.0.1", - "source-map": "^0.6.1" - }, "engines": { "node": ">=10.13.0" } }, - "node_modules/webpack-sources/node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/webpack/node_modules/mime-db": { "version": "1.47.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz", @@ -24133,12 +23796,12 @@ } }, "node_modules/webpack/node_modules/schema-utils": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.0.0.tgz", - "integrity": "sha512-6D82/xSzO094ajanoOSbe4YvXWMfn2A//8Y1+MUqFAJul5Bs+yn36xbK9OtNDcRVSBJ9jjeoXftM6CfztsjOAA==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", + "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==", "dev": true, "dependencies": { - "@types/json-schema": "^7.0.6", + "@types/json-schema": "^7.0.8", "ajv": "^6.12.5", "ajv-keywords": "^3.5.2" }, @@ -24532,6 +24195,16 @@ } }, "dependencies": { + "@ampproject/remapping": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.1.tgz", + "integrity": "sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==", + "dev": true, + "requires": { + "@jridgewell/gen-mapping": "^0.3.0", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, "@awesome-code-style/eslint-config": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/@awesome-code-style/eslint-config/-/eslint-config-4.1.0.tgz", @@ -24558,61 +24231,43 @@ } }, "@babel/code-frame": { - "version": "7.12.11", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.11.tgz", - "integrity": "sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw==", + "version": "7.22.10", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.10.tgz", + "integrity": "sha512-/KKIMG4UEL35WmI9OlvMhurwtytjvXoFcGNrOvyG9zIzA8YmPjVtIZUf7b05+TPO7G7/GEmLHDaoCgACHl9hhA==", "requires": { - "@babel/highlight": "^7.10.4" + "@babel/highlight": "^7.22.10", + "chalk": "^2.4.2" } }, "@babel/compat-data": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.14.4.tgz", - "integrity": "sha512-i2wXrWQNkH6JplJQGn3Rd2I4Pij8GdHkXwHMxm+zV5YG/Jci+bCNrWZEWC4o+umiDkRrRs4dVzH3X4GP7vyjQQ==", + "version": "7.22.9", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.22.9.tgz", + "integrity": "sha512-5UamI7xkUcJ3i9qVDS+KFDEK8/7oJ55/sJMB1Ge7IEapr7KfdfV/HErR+koZwOfd+SgtFKOKRhRakdg++DcJpQ==", "dev": true }, "@babel/core": { - "version": "7.14.3", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.14.3.tgz", - "integrity": "sha512-jB5AmTKOCSJIZ72sd78ECEhuPiDMKlQdDI/4QRI6lzYATx5SSogS1oQA2AoPecRCknm30gHi2l+QVvNUu3wZAg==", - "dev": true, - "requires": { - "@babel/code-frame": "^7.12.13", - "@babel/generator": "^7.14.3", - "@babel/helper-compilation-targets": "^7.13.16", - "@babel/helper-module-transforms": "^7.14.2", - "@babel/helpers": "^7.14.0", - "@babel/parser": "^7.14.3", - "@babel/template": "^7.12.13", - "@babel/traverse": "^7.14.2", - "@babel/types": "^7.14.2", + "version": "7.22.11", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.22.11.tgz", + "integrity": "sha512-lh7RJrtPdhibbxndr6/xx0w8+CVlY5FJZiaSz908Fpy+G0xkBFTvwLcKJFF4PJxVfGhVWNebikpWGnOoC71juQ==", + "dev": true, + "requires": { + "@ampproject/remapping": "^2.2.0", + "@babel/code-frame": "^7.22.10", + "@babel/generator": "^7.22.10", + "@babel/helper-compilation-targets": "^7.22.10", + "@babel/helper-module-transforms": "^7.22.9", + "@babel/helpers": "^7.22.11", + "@babel/parser": "^7.22.11", + "@babel/template": "^7.22.5", + "@babel/traverse": "^7.22.11", + "@babel/types": "^7.22.11", "convert-source-map": "^1.7.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", - "json5": "^2.1.2", - "semver": "^6.3.0", - "source-map": "^0.5.0" + "json5": "^2.2.3", + "semver": "^6.3.1" }, "dependencies": { - "@babel/code-frame": { - "version": "7.12.13", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.13.tgz", - "integrity": "sha512-HV1Cm0Q3ZrpCR93tkWOYiuYIgLxZXZFVG2VgK+MBWjUqZTundupbfx2aXarXuw5Ko5aMcjtJgbSs4vUGBS5v6g==", - "dev": true, - "requires": { - "@babel/highlight": "^7.12.13" - } - }, - "@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - }, "convert-source-map": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.7.0.tgz", @@ -24632,13 +24287,10 @@ } }, "json5": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.0.tgz", - "integrity": "sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA==", - "dev": true, - "requires": { - "minimist": "^1.2.5" - } + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true }, "ms": { "version": "2.1.2", @@ -24647,9 +24299,9 @@ "dev": true }, "semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true } } @@ -24680,24 +24332,25 @@ } }, "@babel/generator": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.17.0.tgz", - "integrity": "sha512-I3Omiv6FGOC29dtlZhkfXO6pgkmukJSlT26QjVvS1DGZe/NzSVCPG41X0tS21oZkJYlovfj9qDWgKP+Cn4bXxw==", + "version": "7.22.10", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.22.10.tgz", + "integrity": "sha512-79KIf7YiWjjdZ81JnLujDRApWtl7BxTqWD88+FFdQEIOG8LJ0etDOM7CXuIgGJa55sGOwZVwuEsaLEm0PJ5/+A==", "dev": true, "requires": { - "@babel/types": "^7.17.0", - "jsesc": "^2.5.1", - "source-map": "^0.5.0" + "@babel/types": "^7.22.10", + "@jridgewell/gen-mapping": "^0.3.2", + "@jridgewell/trace-mapping": "^0.3.17", + "jsesc": "^2.5.1" }, "dependencies": { - "@babel/types": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.17.0.tgz", - "integrity": "sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==", + "@jridgewell/trace-mapping": { + "version": "0.3.19", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz", + "integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==", "dev": true, "requires": { - "@babel/helper-validator-identifier": "^7.16.7", - "to-fast-properties": "^2.0.0" + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" } } } @@ -24709,18 +24362,6 @@ "dev": true, "requires": { "@babel/types": "^7.12.13" - }, - "dependencies": { - "@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - } } }, "@babel/helper-builder-binary-assignment-operator-visitor": { @@ -24731,36 +24372,40 @@ "requires": { "@babel/helper-explode-assignable-expression": "^7.12.13", "@babel/types": "^7.12.13" - }, - "dependencies": { - "@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - } } }, "@babel/helper-compilation-targets": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.14.4.tgz", - "integrity": "sha512-JgdzOYZ/qGaKTVkn5qEDV/SXAh8KcyUVkCoSWGN8T3bwrgd6m+/dJa2kVGi6RJYJgEYPBdZ84BZp9dUjNWkBaA==", + "version": "7.22.10", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.10.tgz", + "integrity": "sha512-JMSwHD4J7SLod0idLq5PKgI+6g/hLD/iuWBq08ZX49xE14VpVEojJ5rHWptpirV2j020MvypRLAXAO50igCJ5Q==", "dev": true, "requires": { - "@babel/compat-data": "^7.14.4", - "@babel/helper-validator-option": "^7.12.17", - "browserslist": "^4.16.6", - "semver": "^6.3.0" + "@babel/compat-data": "^7.22.9", + "@babel/helper-validator-option": "^7.22.5", + "browserslist": "^4.21.9", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" }, "dependencies": { + "lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "requires": { + "yallist": "^3.0.2" + } + }, "semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true + }, + "yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", "dev": true } } @@ -24835,25 +24480,10 @@ } }, "@babel/helper-environment-visitor": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.16.7.tgz", - "integrity": "sha512-SLLb0AAn6PkUeAfKJCCOl9e1R53pQlGAfc4y4XuMRZfqeMYLE0dM1LMhqbGAlGQY0lfw5/ohoYWAe9V1yibRag==", - "dev": true, - "requires": { - "@babel/types": "^7.16.7" - }, - "dependencies": { - "@babel/types": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.17.0.tgz", - "integrity": "sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.16.7", - "to-fast-properties": "^2.0.0" - } - } - } + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.5.tgz", + "integrity": "sha512-XGmhECfVA/5sAt+H+xpSg0mfrHq6FzNr9Oxh7PSEBBRUb/mL7Kz3NICXb194rCqAEdxkhPT1a88teizAFyvk8Q==", + "dev": true }, "@babel/helper-explode-assignable-expression": { "version": "7.13.0", @@ -24862,84 +24492,25 @@ "dev": true, "requires": { "@babel/types": "^7.13.0" - }, - "dependencies": { - "@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - } } }, "@babel/helper-function-name": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.16.7.tgz", - "integrity": "sha512-QfDfEnIUyyBSR3HtrtGECuZ6DAyCkYFp7GHl75vFtTnn6pjKeK0T1DB5lLkFvBea8MdaiUABx3osbgLyInoejA==", + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.22.5.tgz", + "integrity": "sha512-wtHSq6jMRE3uF2otvfuD3DIvVhOsSNshQl0Qrd7qC9oQJzHvOL4qQXlQn2916+CXGywIjpGuIkoyZRRxHPiNQQ==", "dev": true, "requires": { - "@babel/helper-get-function-arity": "^7.16.7", - "@babel/template": "^7.16.7", - "@babel/types": "^7.16.7" - }, - "dependencies": { - "@babel/types": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.17.0.tgz", - "integrity": "sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.16.7", - "to-fast-properties": "^2.0.0" - } - } - } - }, - "@babel/helper-get-function-arity": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.16.7.tgz", - "integrity": "sha512-flc+RLSOBXzNzVhcLu6ujeHUrD6tANAOU5ojrRx/as+tbzf8+stUCj7+IfRRoAbEZqj/ahXEMsjhOhgeZsrnTw==", - "dev": true, - "requires": { - "@babel/types": "^7.16.7" - }, - "dependencies": { - "@babel/types": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.17.0.tgz", - "integrity": "sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.16.7", - "to-fast-properties": "^2.0.0" - } - } + "@babel/template": "^7.22.5", + "@babel/types": "^7.22.5" } }, "@babel/helper-hoist-variables": { - "version": "7.13.16", - "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.13.16.tgz", - "integrity": "sha512-1eMtTrXtrwscjcAeO4BVK+vvkxaLJSPFz1w1KLawz6HLNi9bPFGBNwwDyVfiu1Tv/vRRFYfoGaKhmAQPGPn5Wg==", + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz", + "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==", "dev": true, "requires": { - "@babel/traverse": "^7.13.15", - "@babel/types": "^7.13.16" - }, - "dependencies": { - "@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - } + "@babel/types": "^7.22.5" } }, "@babel/helper-member-expression-to-functions": { @@ -24949,65 +24520,27 @@ "dev": true, "requires": { "@babel/types": "^7.13.12" - }, - "dependencies": { - "@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - } } }, "@babel/helper-module-imports": { - "version": "7.13.12", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.13.12.tgz", - "integrity": "sha512-4cVvR2/1B693IuOvSI20xqqa/+bl7lqAMR59R4iu39R9aOX8/JoYY1sFaNvUMyMBGnHdwvJgUrzNLoUZxXypxA==", + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.22.5.tgz", + "integrity": "sha512-8Dl6+HD/cKifutF5qGd/8ZJi84QeAKh+CEe1sBzz8UayBBGg1dAIJrdHOcOM5b2MpzWL2yuotJTtGjETq0qjXg==", "requires": { - "@babel/types": "^7.13.12" - }, - "dependencies": { - "@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "requires": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - } + "@babel/types": "^7.22.5" } }, "@babel/helper-module-transforms": { - "version": "7.14.2", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.14.2.tgz", - "integrity": "sha512-OznJUda/soKXv0XhpvzGWDnml4Qnwp16GN+D/kZIdLsWoHj05kyu8Rm5kXmMef+rVJZ0+4pSGLkeixdqNUATDA==", + "version": "7.22.9", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.22.9.tgz", + "integrity": "sha512-t+WA2Xn5K+rTeGtC8jCsdAH52bjggG5TKRuRrAGNM/mjIbO4GxvlLMFOEz9wXY5I2XQ60PMFsAG2WIcG82dQMQ==", "dev": true, "requires": { - "@babel/helper-module-imports": "^7.13.12", - "@babel/helper-replace-supers": "^7.13.12", - "@babel/helper-simple-access": "^7.13.12", - "@babel/helper-split-export-declaration": "^7.12.13", - "@babel/helper-validator-identifier": "^7.14.0", - "@babel/template": "^7.12.13", - "@babel/traverse": "^7.14.2", - "@babel/types": "^7.14.2" - }, - "dependencies": { - "@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - } + "@babel/helper-environment-visitor": "^7.22.5", + "@babel/helper-module-imports": "^7.22.5", + "@babel/helper-simple-access": "^7.22.5", + "@babel/helper-split-export-declaration": "^7.22.6", + "@babel/helper-validator-identifier": "^7.22.5" } }, "@babel/helper-optimise-call-expression": { @@ -25017,18 +24550,6 @@ "dev": true, "requires": { "@babel/types": "^7.12.13" - }, - "dependencies": { - "@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - } } }, "@babel/helper-plugin-utils": { @@ -25046,18 +24567,6 @@ "@babel/helper-annotate-as-pure": "^7.12.13", "@babel/helper-wrap-function": "^7.13.0", "@babel/types": "^7.13.0" - }, - "dependencies": { - "@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - } } }, "@babel/helper-replace-supers": { @@ -25070,39 +24579,15 @@ "@babel/helper-optimise-call-expression": "^7.12.13", "@babel/traverse": "^7.14.2", "@babel/types": "^7.14.4" - }, - "dependencies": { - "@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - } } }, "@babel/helper-simple-access": { - "version": "7.13.12", - "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.13.12.tgz", - "integrity": "sha512-7FEjbrx5SL9cWvXioDbnlYTppcZGuCY6ow3/D5vMggb2Ywgu4dMrpTJX0JdQAIcRRUElOIxF3yEooa9gUb9ZbA==", + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.22.5.tgz", + "integrity": "sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w==", "dev": true, "requires": { - "@babel/types": "^7.13.12" - }, - "dependencies": { - "@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - } + "@babel/types": "^7.22.5" } }, "@babel/helper-skip-transparent-expression-wrappers": { @@ -25112,50 +24597,31 @@ "dev": true, "requires": { "@babel/types": "^7.12.1" - }, - "dependencies": { - "@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - } } }, "@babel/helper-split-export-declaration": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.16.7.tgz", - "integrity": "sha512-xbWoy/PFoxSWazIToT9Sif+jJTlrMcndIsaOKvTA6u7QEo7ilkRZpjew18/W3c7nm8fXdUDXh02VXTbZ0pGDNw==", + "version": "7.22.6", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz", + "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==", "dev": true, "requires": { - "@babel/types": "^7.16.7" - }, - "dependencies": { - "@babel/types": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.17.0.tgz", - "integrity": "sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.16.7", - "to-fast-properties": "^2.0.0" - } - } + "@babel/types": "^7.22.5" } }, + "@babel/helper-string-parser": { + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz", + "integrity": "sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==" + }, "@babel/helper-validator-identifier": { - "version": "7.19.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz", - "integrity": "sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==" + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.5.tgz", + "integrity": "sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ==" }, "@babel/helper-validator-option": { - "version": "7.12.17", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.12.17.tgz", - "integrity": "sha512-TopkMDmLzq8ngChwRlyjR6raKD6gMSae4JdYDB8bByKreQgG0RBTuKe9LRxW3wFtUnjxOPRKBDwEH6Mg5KeDfw==", + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.22.5.tgz", + "integrity": "sha512-R3oB6xlIVKUnxNUxbmgq7pKjxpru24zlimpE8WK47fACIlM0II/Hm1RS8IaOI7NgCr6LNS+jl5l75m20npAziw==", "dev": true }, "@babel/helper-wrap-function": { @@ -25168,57 +24634,33 @@ "@babel/template": "^7.12.13", "@babel/traverse": "^7.13.0", "@babel/types": "^7.13.0" - }, - "dependencies": { - "@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - } } }, "@babel/helpers": { - "version": "7.14.0", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.14.0.tgz", - "integrity": "sha512-+ufuXprtQ1D1iZTO/K9+EBRn+qPWMJjZSw/S0KlFrxCw4tkrzv9grgpDHkY9MeQTjTY8i2sp7Jep8DfU6tN9Mg==", + "version": "7.22.11", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.22.11.tgz", + "integrity": "sha512-vyOXC8PBWaGc5h7GMsNx68OH33cypkEDJCHvYVVgVbbxJDROYVtexSk0gK5iCF1xNjRIN2s8ai7hwkWDq5szWg==", "dev": true, "requires": { - "@babel/template": "^7.12.13", - "@babel/traverse": "^7.14.0", - "@babel/types": "^7.14.0" - }, - "dependencies": { - "@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - } + "@babel/template": "^7.22.5", + "@babel/traverse": "^7.22.11", + "@babel/types": "^7.22.11" } }, "@babel/highlight": { - "version": "7.18.6", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.18.6.tgz", - "integrity": "sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==", + "version": "7.22.10", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.10.tgz", + "integrity": "sha512-78aUtVcT7MUscr0K5mIEnkwxPE0MaxkR5RxRwuHaQ+JuU5AmTPhY+do2mdzVTnIJJpyBglql2pehuBIWHug+WQ==", "requires": { - "@babel/helper-validator-identifier": "^7.18.6", - "chalk": "^2.0.0", + "@babel/helper-validator-identifier": "^7.22.5", + "chalk": "^2.4.2", "js-tokens": "^4.0.0" } }, "@babel/parser": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.17.0.tgz", - "integrity": "sha512-VKXSCQx5D8S04ej+Dqsr1CzYvvWgf20jIw2D+YhQCrIlr2UZGaDds23Y0xg75/skOxpLCRpUZvk/1EAVkGoDOw==", + "version": "7.22.11", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.22.11.tgz", + "integrity": "sha512-R5zb8eJIBPJriQtbH/htEQy4k7E2dHWlD2Y2VT07JCzwYZHBxV5ZYtM0UhXSNMT74LyxuM+b1jdL7pSesXbC/g==", "dev": true }, "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": { @@ -26446,16 +25888,6 @@ "integrity": "sha512-ZPafIPSwzUlAoWT8DKs1W2VyF2gOWthGd5NGFMsBcMMol+ZhK+EQY/e6V96poa6PA/Bh+C9plWN0hXO1uB8AfQ==", "dev": true }, - "@babel/types": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz", - "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.14.0", - "to-fast-properties": "^2.0.0" - } - }, "semver": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", @@ -26486,83 +25918,34 @@ } }, "@babel/template": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.16.7.tgz", - "integrity": "sha512-I8j/x8kHUrbYRTUxXrrMbfCa7jxkE7tZre39x3kjr9hvI82cK1FfqLygotcWN5kdPGWcLdWMHpSBavse5tWw3w==", + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.5.tgz", + "integrity": "sha512-X7yV7eiwAxdj9k94NEylvbVHLiVG1nvzCV2EAowhxLTwODV1jl9UzZ48leOC0sH7OnuHrIkllaBgneUykIcZaw==", "dev": true, "requires": { - "@babel/code-frame": "^7.16.7", - "@babel/parser": "^7.16.7", - "@babel/types": "^7.16.7" - }, - "dependencies": { - "@babel/code-frame": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.16.7.tgz", - "integrity": "sha512-iAXqUn8IIeBTNd72xsFlgaXHkMBMt6y4HJp1tIaK465CWLT/fG1aqB7ykr95gHHmlBdGbFeWWfyB4NJJ0nmeIg==", - "dev": true, - "requires": { - "@babel/highlight": "^7.16.7" - } - }, - "@babel/types": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.17.0.tgz", - "integrity": "sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.16.7", - "to-fast-properties": "^2.0.0" - } - } + "@babel/code-frame": "^7.22.5", + "@babel/parser": "^7.22.5", + "@babel/types": "^7.22.5" } }, "@babel/traverse": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.17.0.tgz", - "integrity": "sha512-fpFIXvqD6kC7c7PUNnZ0Z8cQXlarCLtCUpt2S1Dx7PjoRtCFffvOkHHSom+m5HIxMZn5bIBVb71lhabcmjEsqg==", - "dev": true, - "requires": { - "@babel/code-frame": "^7.16.7", - "@babel/generator": "^7.17.0", - "@babel/helper-environment-visitor": "^7.16.7", - "@babel/helper-function-name": "^7.16.7", - "@babel/helper-hoist-variables": "^7.16.7", - "@babel/helper-split-export-declaration": "^7.16.7", - "@babel/parser": "^7.17.0", - "@babel/types": "^7.17.0", + "version": "7.22.11", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.22.11.tgz", + "integrity": "sha512-mzAenteTfomcB7mfPtyi+4oe5BZ6MXxWcn4CX+h4IRJ+OOGXBrWU6jDQavkQI9Vuc5P+donFabBfFCcmWka9lQ==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.22.10", + "@babel/generator": "^7.22.10", + "@babel/helper-environment-visitor": "^7.22.5", + "@babel/helper-function-name": "^7.22.5", + "@babel/helper-hoist-variables": "^7.22.5", + "@babel/helper-split-export-declaration": "^7.22.6", + "@babel/parser": "^7.22.11", + "@babel/types": "^7.22.11", "debug": "^4.1.0", "globals": "^11.1.0" }, "dependencies": { - "@babel/code-frame": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.16.7.tgz", - "integrity": "sha512-iAXqUn8IIeBTNd72xsFlgaXHkMBMt6y4HJp1tIaK465CWLT/fG1aqB7ykr95gHHmlBdGbFeWWfyB4NJJ0nmeIg==", - "dev": true, - "requires": { - "@babel/highlight": "^7.16.7" - } - }, - "@babel/helper-hoist-variables": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.16.7.tgz", - "integrity": "sha512-m04d/0Op34H5v7pbZw6pSKP7weA6lsMvfiIAMeIvkY/R4xQtBSMFEigu9QTZ2qB/9l22vsxtM8a+Q8CzD255fg==", - "dev": true, - "requires": { - "@babel/types": "^7.16.7" - } - }, - "@babel/types": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.17.0.tgz", - "integrity": "sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.16.7", - "to-fast-properties": "^2.0.0" - } - }, "debug": { "version": "4.3.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", @@ -26581,13 +25964,12 @@ } }, "@babel/types": { - "version": "7.5.5", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.5.5.tgz", - "integrity": "sha512-s63F9nJioLqOlW3UkyMd+BYhXt44YuaFm/VV0VwuteqjYwRrObkU7ra9pY4wAJR3oXi8hJrMcrcJdO/HH33vtw==", - "dev": true, + "version": "7.22.11", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.22.11.tgz", + "integrity": "sha512-siazHiGuZRz9aB9NpHy9GOs9xiQPKnMzgdr493iI1M67vRXpnEq8ZOOKzezC5q7zwuQ6sDhdSp4SD9ixKSqKZg==", "requires": { - "esutils": "^2.0.2", - "lodash": "^4.17.13", + "@babel/helper-string-parser": "^7.22.5", + "@babel/helper-validator-identifier": "^7.22.5", "to-fast-properties": "^2.0.0" } }, @@ -26723,9 +26105,9 @@ "dev": true }, "@druid-toolkit/query": { - "version": "0.20.5", - "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.20.5.tgz", - "integrity": "sha512-EY0131z611tklnui+vyRqsoPjTBbonkF7WwsNvT0KsBQYm5qtuvX/QlXGfX66f4KQzoo5G/4dRIVmZ9JbSRgzw==", + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.21.1.tgz", + "integrity": "sha512-p9bE0mlE0Lv6w1HOOr6m/NB1l4oQy0ew6HFa3hSZ1T/qlDx8CPT5MXRRhruEmgtneZx4UrN6RqFULJNFVv+aWg==", "requires": { "tslib": "^2.5.2" } @@ -27688,28 +27070,6 @@ "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true }, - "jest-worker": { - "version": "27.5.0", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.0.tgz", - "integrity": "sha512-8OEHiPNOPTfaWnJ2SUHM8fmgeGq37uuGsQBvGKQJl1f+6WIy6g7G3fE2ruI5294bUKUI9FaCWt5hDvO8HSwsSg==", - "dev": true, - "requires": { - "@types/node": "*", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "dependencies": { - "supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - } - } - }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -28007,9 +27367,9 @@ } }, "@jridgewell/gen-mapping": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", - "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz", + "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==", "dev": true, "requires": { "@jridgewell/set-array": "^1.0.1", @@ -28030,9 +27390,9 @@ "dev": true }, "@jridgewell/source-map": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.2.tgz", - "integrity": "sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw==", + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.5.tgz", + "integrity": "sha512-UTYAUj/wviwdsMfzoSJspJxbkH5o1snzwX0//0ENX1u/55kkZZkcTZP6u9bwKGkv+dkk9at4m1Cpt0uY80kcpQ==", "dev": true, "requires": { "@jridgewell/gen-mapping": "^0.3.0", @@ -28371,9 +27731,9 @@ } }, "@types/eslint": { - "version": "7.2.10", - "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-7.2.10.tgz", - "integrity": "sha512-kUEPnMKrqbtpCq/KTaGFFKAcz6Ethm2EjCoKIDaCmfRBWLbFuTcOJfTlorwbnboXBzahqWLgUp1BQeKHiJzPUQ==", + "version": "8.44.2", + "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.44.2.tgz", + "integrity": "sha512-sdPRb9K6iL5XZOmBubg8yiFp5yS/JdUDQsq5e6h95km91MCYMuvp7mh1fjPEYUhvHepKpZOjnEaMBR4PxjWDzg==", "dev": true, "requires": { "@types/estree": "*", @@ -28381,9 +27741,9 @@ } }, "@types/eslint-scope": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.0.tgz", - "integrity": "sha512-O/ql2+rrCUe2W2rs7wMR+GqPRcgB6UiqN5RhrR5xruFlY7l9YLMn0ZkDzjoHLeiFkR8MCQZVudUuuvQ2BLC9Qw==", + "version": "3.7.4", + "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.4.tgz", + "integrity": "sha512-9K4zoImiZc3HlIp6AVUDE4CWYx22a+lhSZMYNpbjW04+YF0KWj4pJXnEMjdnFTiQibFFmElcsasJXDbdI/EPhA==", "dev": true, "requires": { "@types/eslint": "*", @@ -28391,9 +27751,9 @@ } }, "@types/estree": { - "version": "0.0.46", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.46.tgz", - "integrity": "sha512-laIjwTQaD+5DukBZaygQ79K1Z0jb1bPEMRrkXSLjtCcZm+abyp5YbrqpSLzD42FwWW6gK/aS4NYpJ804nG2brg==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.1.tgz", + "integrity": "sha512-LG4opVs2ANWZ1TJoKc937iMmNstM/d0ae1vNbnBvBhqCSezgVUOzcLCqbI5elV8Vy6WKwKjaqR+zO9VKirBBCA==", "dev": true }, "@types/file-saver": { @@ -28868,148 +28228,148 @@ } }, "@webassemblyjs/ast": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.0.tgz", - "integrity": "sha512-kX2W49LWsbthrmIRMbQZuQDhGtjyqXfEmmHyEi4XWnSZtPmxY0+3anPIzsnRb45VH/J55zlOfWvZuY47aJZTJg==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.6.tgz", + "integrity": "sha512-IN1xI7PwOvLPgjcf180gC1bqn3q/QaOCwYUahIOhbYUu8KA/3tw2RT/T0Gidi1l7Hhj5D/INhJxiICObqpMu4Q==", "dev": true, "requires": { - "@webassemblyjs/helper-numbers": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0" + "@webassemblyjs/helper-numbers": "1.11.6", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6" } }, "@webassemblyjs/floating-point-hex-parser": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.0.tgz", - "integrity": "sha512-Q/aVYs/VnPDVYvsCBL/gSgwmfjeCb4LW8+TMrO3cSzJImgv8lxxEPM2JA5jMrivE7LSz3V+PFqtMbls3m1exDA==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz", + "integrity": "sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw==", "dev": true }, "@webassemblyjs/helper-api-error": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.0.tgz", - "integrity": "sha512-baT/va95eXiXb2QflSx95QGT5ClzWpGaa8L7JnJbgzoYeaA27FCvuBXU758l+KXWRndEmUXjP0Q5fibhavIn8w==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz", + "integrity": "sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q==", "dev": true }, "@webassemblyjs/helper-buffer": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.0.tgz", - "integrity": "sha512-u9HPBEl4DS+vA8qLQdEQ6N/eJQ7gT7aNvMIo8AAWvAl/xMrcOSiI2M0MAnMCy3jIFke7bEee/JwdX1nUpCtdyA==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.6.tgz", + "integrity": "sha512-z3nFzdcp1mb8nEOFFk8DrYLpHvhKC3grJD2ardfKOzmbmJvEf/tPIqCY+sNcwZIY8ZD7IkB2l7/pqhUhqm7hLA==", "dev": true }, "@webassemblyjs/helper-numbers": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.0.tgz", - "integrity": "sha512-DhRQKelIj01s5IgdsOJMKLppI+4zpmcMQ3XboFPLwCpSNH6Hqo1ritgHgD0nqHeSYqofA6aBN/NmXuGjM1jEfQ==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz", + "integrity": "sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==", "dev": true, "requires": { - "@webassemblyjs/floating-point-hex-parser": "1.11.0", - "@webassemblyjs/helper-api-error": "1.11.0", + "@webassemblyjs/floating-point-hex-parser": "1.11.6", + "@webassemblyjs/helper-api-error": "1.11.6", "@xtuc/long": "4.2.2" } }, "@webassemblyjs/helper-wasm-bytecode": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.0.tgz", - "integrity": "sha512-MbmhvxXExm542tWREgSFnOVo07fDpsBJg3sIl6fSp9xuu75eGz5lz31q7wTLffwL3Za7XNRCMZy210+tnsUSEA==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz", + "integrity": "sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA==", "dev": true }, "@webassemblyjs/helper-wasm-section": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.0.tgz", - "integrity": "sha512-3Eb88hcbfY/FCukrg6i3EH8H2UsD7x8Vy47iVJrP967A9JGqgBVL9aH71SETPx1JrGsOUVLo0c7vMCN22ytJew==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.6.tgz", + "integrity": "sha512-LPpZbSOwTpEC2cgn4hTydySy1Ke+XEu+ETXuoyvuyezHO3Kjdu90KK95Sh9xTbmjrCsUwvWwCOQQNta37VrS9g==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0" + "@webassemblyjs/ast": "1.11.6", + "@webassemblyjs/helper-buffer": "1.11.6", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6", + "@webassemblyjs/wasm-gen": "1.11.6" } }, "@webassemblyjs/ieee754": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.0.tgz", - "integrity": "sha512-KXzOqpcYQwAfeQ6WbF6HXo+0udBNmw0iXDmEK5sFlmQdmND+tr773Ti8/5T/M6Tl/413ArSJErATd8In3B+WBA==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.6.tgz", + "integrity": "sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg==", "dev": true, "requires": { "@xtuc/ieee754": "^1.2.0" } }, "@webassemblyjs/leb128": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.0.tgz", - "integrity": "sha512-aqbsHa1mSQAbeeNcl38un6qVY++hh8OpCOzxhixSYgbRfNWcxJNJQwe2rezK9XEcssJbbWIkblaJRwGMS9zp+g==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.6.tgz", + "integrity": "sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ==", "dev": true, "requires": { "@xtuc/long": "4.2.2" } }, "@webassemblyjs/utf8": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.0.tgz", - "integrity": "sha512-A/lclGxH6SpSLSyFowMzO/+aDEPU4hvEiooCMXQPcQFPPJaYcPQNKGOCLUySJsYJ4trbpr+Fs08n4jelkVTGVw==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.6.tgz", + "integrity": "sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA==", "dev": true }, "@webassemblyjs/wasm-edit": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.0.tgz", - "integrity": "sha512-JHQ0damXy0G6J9ucyKVXO2j08JVJ2ntkdJlq1UTiUrIgfGMmA7Ik5VdC/L8hBK46kVJgujkBIoMtT8yVr+yVOQ==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.6.tgz", + "integrity": "sha512-Ybn2I6fnfIGuCR+Faaz7YcvtBKxvoLV3Lebn1tM4o/IAJzmi9AWYIPWpyBfU8cC+JxAO57bk4+zdsTjJR+VTOw==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/helper-wasm-section": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0", - "@webassemblyjs/wasm-opt": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0", - "@webassemblyjs/wast-printer": "1.11.0" + "@webassemblyjs/ast": "1.11.6", + "@webassemblyjs/helper-buffer": "1.11.6", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6", + "@webassemblyjs/helper-wasm-section": "1.11.6", + "@webassemblyjs/wasm-gen": "1.11.6", + "@webassemblyjs/wasm-opt": "1.11.6", + "@webassemblyjs/wasm-parser": "1.11.6", + "@webassemblyjs/wast-printer": "1.11.6" } }, "@webassemblyjs/wasm-gen": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.0.tgz", - "integrity": "sha512-BEUv1aj0WptCZ9kIS30th5ILASUnAPEvE3tVMTrItnZRT9tXCLW2LEXT8ezLw59rqPP9klh9LPmpU+WmRQmCPQ==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.6.tgz", + "integrity": "sha512-3XOqkZP/y6B4F0PBAXvI1/bky7GryoogUtfwExeP/v7Nzwo1QLcq5oQmpKlftZLbT+ERUOAZVQjuNVak6UXjPA==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/ieee754": "1.11.0", - "@webassemblyjs/leb128": "1.11.0", - "@webassemblyjs/utf8": "1.11.0" + "@webassemblyjs/ast": "1.11.6", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6", + "@webassemblyjs/ieee754": "1.11.6", + "@webassemblyjs/leb128": "1.11.6", + "@webassemblyjs/utf8": "1.11.6" } }, "@webassemblyjs/wasm-opt": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.0.tgz", - "integrity": "sha512-tHUSP5F4ywyh3hZ0+fDQuWxKx3mJiPeFufg+9gwTpYp324mPCQgnuVKwzLTZVqj0duRDovnPaZqDwoyhIO8kYg==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.6.tgz", + "integrity": "sha512-cOrKuLRE7PCe6AsOVl7WasYf3wbSo4CeOk6PkrjS7g57MFfVUF9u6ysQBBODX0LdgSvQqRiGz3CXvIDKcPNy4g==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0" + "@webassemblyjs/ast": "1.11.6", + "@webassemblyjs/helper-buffer": "1.11.6", + "@webassemblyjs/wasm-gen": "1.11.6", + "@webassemblyjs/wasm-parser": "1.11.6" } }, "@webassemblyjs/wasm-parser": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.0.tgz", - "integrity": "sha512-6L285Sgu9gphrcpDXINvm0M9BskznnzJTE7gYkjDbxET28shDqp27wpruyx3C2S/dvEwiigBwLA1cz7lNUi0kw==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.6.tgz", + "integrity": "sha512-6ZwPeGzMJM3Dqp3hCsLgESxBGtT/OeCvCZ4TA1JUPYgmhAx38tTPR9JaKy0S5H3evQpO/h2uWs2j6Yc/fjkpTQ==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-api-error": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/ieee754": "1.11.0", - "@webassemblyjs/leb128": "1.11.0", - "@webassemblyjs/utf8": "1.11.0" + "@webassemblyjs/ast": "1.11.6", + "@webassemblyjs/helper-api-error": "1.11.6", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6", + "@webassemblyjs/ieee754": "1.11.6", + "@webassemblyjs/leb128": "1.11.6", + "@webassemblyjs/utf8": "1.11.6" } }, "@webassemblyjs/wast-printer": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.0.tgz", - "integrity": "sha512-Fg5OX46pRdTgB7rKIUojkh9vXaVN6sGYCnEiJN1GYkb0RPwShZXp6KTDqmoMdQPKhcroOXh3fEzmkWmCYaKYhQ==", + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.6.tgz", + "integrity": "sha512-JM7AhRcE+yW2GWYaKeHL5vt4xqee5N2WcezptmgyhNS+ScggqcT1OtXykhAb13Sn5Yas0j2uv9tHgrjwvzAP4A==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", + "@webassemblyjs/ast": "1.11.6", "@xtuc/long": "4.2.2" } }, @@ -29074,9 +28434,9 @@ "integrity": "sha512-NBOQlm9+7RBqRqZwimpgquaLeTJFayqb9UEPtTkpC3TkkwDnlsT/TwsCC0svjt9kEZ6G9mH5AEOHSz6Q/HrzQQ==" }, "acorn": { - "version": "8.8.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz", - "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==", + "version": "8.10.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz", + "integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==", "dev": true }, "acorn-globals": { @@ -29103,6 +28463,12 @@ } } }, + "acorn-import-assertions": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz", + "integrity": "sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==", + "dev": true + }, "acorn-jsx": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", @@ -29925,15 +29291,15 @@ "dev": true }, "browserslist": { - "version": "4.21.5", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.5.tgz", - "integrity": "sha512-tUkiguQGW7S3IhB7N+c2MV/HZPSCPAAiYBZXLsBhFB/PCy6ZKKsZrmBayHV9fdGV/ARIfJ14NkxKzRDjvp7L6w==", + "version": "4.21.10", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.10.tgz", + "integrity": "sha512-bipEBdZfVH5/pwrvqc+Ub0kUPVfGUhlKxbvfD+z1BDnPEO/X98ruXGA1WP5ASpAFKan7Qr6j736IacbZQuAlKQ==", "dev": true, "requires": { - "caniuse-lite": "^1.0.30001449", - "electron-to-chromium": "^1.4.284", - "node-releases": "^2.0.8", - "update-browserslist-db": "^1.0.10" + "caniuse-lite": "^1.0.30001517", + "electron-to-chromium": "^1.4.477", + "node-releases": "^2.0.13", + "update-browserslist-db": "^1.0.11" } }, "bs-logger": { @@ -30044,9 +29410,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001466", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001466.tgz", - "integrity": "sha512-ewtFBSfWjEmxUgNBSZItFSmVtvk9zkwkl1OfRZlKA8slltRN+/C/tuGVrF9styXkN36Yu3+SeJ1qkXxDEyNZ5w==", + "version": "1.0.30001522", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001522.tgz", + "integrity": "sha512-TKiyTVZxJGhsTszLuzb+6vUZSjVOAhClszBr2Ta2k9IwtNBT/4dzmL6aywt0HCgEZlmwJzXJd8yNiob6HgwTRg==", "dev": true }, "capital-case": { @@ -31194,9 +30560,9 @@ "dev": true }, "electron-to-chromium": { - "version": "1.4.330", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.330.tgz", - "integrity": "sha512-PqyefhybrVdjAJ45HaPLtuVaehiSw7C3ya0aad+rvmV53IVyXmYRk3pwIOb2TxTDTnmgQdn46NjMMaysx79/6Q==", + "version": "1.4.501", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.501.tgz", + "integrity": "sha512-NCF5hZUg73MEP0guvIM+BjPs9W07UeAuc5XCNqRZZTKJxLjE0ZS/Zo5UsV8bbs2y/jeKRPFPzdWdBfOGEZTXKg==", "dev": true }, "emittery": { @@ -31242,9 +30608,9 @@ } }, "enhanced-resolve": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.12.0.tgz", - "integrity": "sha512-QHTXI/sZQmko1cbDoNAa3mJ5qhWUUNAq3vR0/YiD379fWQrcfuoX1+HW2S0MTt7XmoPLapdaDKUtelUSPic7hQ==", + "version": "5.15.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.15.0.tgz", + "integrity": "sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg==", "dev": true, "requires": { "graceful-fs": "^4.2.4", @@ -31357,9 +30723,9 @@ } }, "es-module-lexer": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.4.1.tgz", - "integrity": "sha512-ooYciCUtfw6/d2w56UVeqHPcoCFAiJdz5XOkYpv/Txl1HMUozpXjz/2RIQgqwKdXNDPSF1W7mJCFse3G+HDyAA==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.3.0.tgz", + "integrity": "sha512-vZK7T0N2CBmBOixhmjdqx2gWVbFZ4DXZ/NyRMZVlJXPa7CyFS+/a4QQsDGDQy9ZfEzxFuNEsMLeQJnKP2p5/JA==", "dev": true }, "es-set-tostringtag": { @@ -35300,28 +34666,6 @@ "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true }, - "jest-worker": { - "version": "27.5.0", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.0.tgz", - "integrity": "sha512-8OEHiPNOPTfaWnJ2SUHM8fmgeGq37uuGsQBvGKQJl1f+6WIy6g7G3fE2ruI5294bUKUI9FaCWt5hDvO8HSwsSg==", - "dev": true, - "requires": { - "@types/node": "*", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "dependencies": { - "supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - } - } - }, "micromatch": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.4.tgz", @@ -35549,15 +34893,6 @@ "stack-utils": "^2.0.3" }, "dependencies": { - "@babel/code-frame": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.16.7.tgz", - "integrity": "sha512-iAXqUn8IIeBTNd72xsFlgaXHkMBMt6y4HJp1tIaK465CWLT/fG1aqB7ykr95gHHmlBdGbFeWWfyB4NJJ0nmeIg==", - "dev": true, - "requires": { - "@babel/highlight": "^7.16.7" - } - }, "@jest/types": { "version": "27.5.0", "resolved": "https://registry.npmjs.org/@jest/types/-/types-27.5.0.tgz", @@ -36082,28 +35417,6 @@ "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true }, - "jest-worker": { - "version": "27.5.0", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.0.tgz", - "integrity": "sha512-8OEHiPNOPTfaWnJ2SUHM8fmgeGq37uuGsQBvGKQJl1f+6WIy6g7G3fE2ruI5294bUKUI9FaCWt5hDvO8HSwsSg==", - "dev": true, - "requires": { - "@types/node": "*", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "dependencies": { - "supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - } - } - }, "supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -36739,14 +36052,14 @@ } }, "jest-worker": { - "version": "26.6.2", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-26.6.2.tgz", - "integrity": "sha512-KWYVV1c4i+jbMpaBC+U++4Va0cp8OisU185o73T1vo99hqi7w8tSJfUXYswwqqrjzwxa6KpRK54WhPvwf5w6PQ==", + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz", + "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", "dev": true, "requires": { "@types/node": "*", "merge-stream": "^2.0.0", - "supports-color": "^7.0.0" + "supports-color": "^8.0.0" }, "dependencies": { "has-flag": { @@ -36756,9 +36069,9 @@ "dev": true }, "supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", "dev": true, "requires": { "has-flag": "^4.0.0" @@ -37658,9 +36971,9 @@ "dev": true }, "node-releases": { - "version": "2.0.10", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.10.tgz", - "integrity": "sha512-5GFldHPXVG/YZmFzJvKK2zDSzPKhEp0+ZR5SVaoSag9fsL5YgHbUHDfnG5494ISANDcK4KwPXAx2xqVEydmd7w==", + "version": "2.0.13", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.13.tgz", + "integrity": "sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==", "dev": true }, "nopt": { @@ -40607,9 +39920,9 @@ } }, "serialize-javascript": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", - "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.1.tgz", + "integrity": "sha512-owoXEFjWRllis8/M1Q+Cw5k8ZH40e3zhp/ovX+Xr/vi1qj6QesbyXXViFbpNvWvPNAD62SutwEXavefrLJWj7w==", "dev": true, "requires": { "randombytes": "^2.1.0" @@ -41012,12 +40325,6 @@ } } }, - "source-list-map": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz", - "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw==", - "dev": true - }, "source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", @@ -41923,56 +41230,50 @@ } }, "terser": { - "version": "5.15.0", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.15.0.tgz", - "integrity": "sha512-L1BJiXVmheAQQy+as0oF3Pwtlo4s3Wi1X2zNZ2NxOB4wx9bdS9Vk67XQENLFdLYGCK/Z2di53mTj/hBafR+dTA==", + "version": "5.19.2", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.19.2.tgz", + "integrity": "sha512-qC5+dmecKJA4cpYxRa5aVkKehYsQKc+AHeKl0Oe62aYjBL8ZA33tTljktDHJSaxxMnbI5ZYw+o/S2DxxLu8OfA==", "dev": true, "requires": { - "@jridgewell/source-map": "^0.3.2", - "acorn": "^8.5.0", + "@jridgewell/source-map": "^0.3.3", + "acorn": "^8.8.2", "commander": "^2.20.0", "source-map-support": "~0.5.20" } }, "terser-webpack-plugin": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.1.1.tgz", - "integrity": "sha512-5XNNXZiR8YO6X6KhSGXfY0QrGrCRlSwAEjIIrlRQR4W8nP69TaJUlh3bkuac6zzgspiGPfKEHcY295MMVExl5Q==", + "version": "5.3.9", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.9.tgz", + "integrity": "sha512-ZuXsqE07EcggTWQjXUj+Aot/OMcD0bMKGgF63f7UxYcu5/AJF53aIpK1YoP5xR9l6s/Hy2b+t1AM0bLNPRuhwA==", "dev": true, "requires": { - "jest-worker": "^26.6.2", - "p-limit": "^3.1.0", - "schema-utils": "^3.0.0", - "serialize-javascript": "^5.0.1", - "source-map": "^0.6.1", - "terser": "^5.5.1" + "@jridgewell/trace-mapping": "^0.3.17", + "jest-worker": "^27.4.5", + "schema-utils": "^3.1.1", + "serialize-javascript": "^6.0.1", + "terser": "^5.16.8" }, "dependencies": { - "p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "@jridgewell/trace-mapping": { + "version": "0.3.19", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz", + "integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==", "dev": true, "requires": { - "yocto-queue": "^0.1.0" + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" } }, "schema-utils": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.0.0.tgz", - "integrity": "sha512-6D82/xSzO094ajanoOSbe4YvXWMfn2A//8Y1+MUqFAJul5Bs+yn36xbK9OtNDcRVSBJ9jjeoXftM6CfztsjOAA==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", + "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==", "dev": true, "requires": { - "@types/json-schema": "^7.0.6", + "@types/json-schema": "^7.0.8", "ajv": "^6.12.5", "ajv-keywords": "^3.5.2" } - }, - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true } } }, @@ -42573,9 +41874,9 @@ "dev": true }, "update-browserslist-db": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz", - "integrity": "sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==", + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz", + "integrity": "sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA==", "dev": true, "requires": { "escalade": "^3.1.1", @@ -42808,9 +42109,9 @@ } }, "watchpack": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.1.1.tgz", - "integrity": "sha512-Oo7LXCmc1eE1AjyuSBmtC3+Wy4HcV8PxWh2kP6fOl8yTlNS7r0K9l1ao2lrrUza7V39Y3D/BbJgY8VeSlc5JKw==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz", + "integrity": "sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==", "dev": true, "requires": { "glob-to-regexp": "^0.4.1", @@ -42833,34 +42134,35 @@ "dev": true }, "webpack": { - "version": "5.33.2", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.33.2.tgz", - "integrity": "sha512-X4b7F1sYBmJx8mlh2B7mV5szEkE0jYNJ2y3akgAP0ERi0vLCG1VvdsIxt8lFd4st6SUy0lf7W0CCQS566MBpJg==", - "dev": true, - "requires": { - "@types/eslint-scope": "^3.7.0", - "@types/estree": "^0.0.46", - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/wasm-edit": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0", - "acorn": "^8.0.4", + "version": "5.88.2", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.88.2.tgz", + "integrity": "sha512-JmcgNZ1iKj+aiR0OvTYtWQqJwq37Pf683dY9bVORwVbUrDhLhdn/PlO2sHsFHPkj7sHNQF3JwaAkp49V+Sq1tQ==", + "dev": true, + "requires": { + "@types/eslint-scope": "^3.7.3", + "@types/estree": "^1.0.0", + "@webassemblyjs/ast": "^1.11.5", + "@webassemblyjs/wasm-edit": "^1.11.5", + "@webassemblyjs/wasm-parser": "^1.11.5", + "acorn": "^8.7.1", + "acorn-import-assertions": "^1.9.0", "browserslist": "^4.14.5", "chrome-trace-event": "^1.0.2", - "enhanced-resolve": "^5.7.0", - "es-module-lexer": "^0.4.0", - "eslint-scope": "^5.1.1", + "enhanced-resolve": "^5.15.0", + "es-module-lexer": "^1.2.1", + "eslint-scope": "5.1.1", "events": "^3.2.0", "glob-to-regexp": "^0.4.1", - "graceful-fs": "^4.2.4", - "json-parse-better-errors": "^1.0.2", + "graceful-fs": "^4.2.9", + "json-parse-even-better-errors": "^2.3.1", "loader-runner": "^4.2.0", "mime-types": "^2.1.27", "neo-async": "^2.6.2", - "schema-utils": "^3.0.0", + "schema-utils": "^3.2.0", "tapable": "^2.1.1", - "terser-webpack-plugin": "^5.1.1", - "watchpack": "^2.0.0", - "webpack-sources": "^2.1.1" + "terser-webpack-plugin": "^5.3.7", + "watchpack": "^2.4.0", + "webpack-sources": "^3.2.3" }, "dependencies": { "mime-db": { @@ -42879,12 +42181,12 @@ } }, "schema-utils": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.0.0.tgz", - "integrity": "sha512-6D82/xSzO094ajanoOSbe4YvXWMfn2A//8Y1+MUqFAJul5Bs+yn36xbK9OtNDcRVSBJ9jjeoXftM6CfztsjOAA==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", + "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==", "dev": true, "requires": { - "@types/json-schema": "^7.0.6", + "@types/json-schema": "^7.0.8", "ajv": "^6.12.5", "ajv-keywords": "^3.5.2" } @@ -43291,22 +42593,10 @@ } }, "webpack-sources": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-2.2.0.tgz", - "integrity": "sha512-bQsA24JLwcnWGArOKUxYKhX3Mz/nK1Xf6hxullKERyktjNMC4x8koOeaDNTA2fEJ09BdWLbM/iTW0ithREUP0w==", - "dev": true, - "requires": { - "source-list-map": "^2.0.1", - "source-map": "^0.6.1" - }, - "dependencies": { - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true - } - } + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz", + "integrity": "sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==", + "dev": true }, "websocket-driver": { "version": "0.7.4", diff --git a/web-console/package.json b/web-console/package.json index 75ef5b1c2368..f8f0310bb530 100644 --- a/web-console/package.json +++ b/web-console/package.json @@ -68,7 +68,7 @@ "@blueprintjs/datetime2": "^0.9.35", "@blueprintjs/icons": "^4.16.0", "@blueprintjs/popover2": "^1.14.9", - "@druid-toolkit/query": "^0.20.5", + "@druid-toolkit/query": "^0.21.1", "@druid-toolkit/visuals-core": "^0.3.3", "@druid-toolkit/visuals-react": "^0.3.3", "ace-builds": "~1.4.14", @@ -109,7 +109,7 @@ "@awesome-code-style/eslint-config": "^4.1.0", "@awesome-code-style/prettier-config": "^4.0.0", "@awesome-code-style/stylelint-config": "^4.0.0", - "@babel/core": "^7.14.3", + "@babel/core": "^7.18.6", "@babel/preset-env": "^7.14.4", "@testing-library/react": "^14.0.0", "@types/classnames": "^2.2.9", @@ -172,7 +172,7 @@ "ts-node": "^10.9.1", "typescript": "^4.9.5", "uuid": "^7.0.2", - "webpack": "^5.33.2", + "webpack": "^5.76.0", "webpack-bundle-analyzer": "^4.4.1", "webpack-cli": "^4.6.0", "webpack-dev-server": "^3.11.2" diff --git a/web-console/src/components/fancy-numeric-input/fancy-numeric-input.tsx b/web-console/src/components/fancy-numeric-input/fancy-numeric-input.tsx index 573366cf6e57..ad38196caed9 100644 --- a/web-console/src/components/fancy-numeric-input/fancy-numeric-input.tsx +++ b/web-console/src/components/fancy-numeric-input/fancy-numeric-input.tsx @@ -73,6 +73,7 @@ export interface FancyNumericInputProps { value: number | undefined; defaultValue?: number; onValueChange(value: number): void; + onValueEmpty?: () => void; min?: number; max?: number; @@ -98,6 +99,7 @@ export const FancyNumericInput = React.memo(function FancyNumericInput( value, defaultValue, onValueChange, + onValueEmpty, min, max, @@ -139,8 +141,8 @@ export const FancyNumericInput = React.memo(function FancyNumericInput( } function increment(delta: number): void { - if (typeof shownNumberRaw !== 'number') return; - changeValue(shownNumberRaw + delta); + if (typeof shownNumberRaw !== 'number' && shownValue !== '') return; + changeValue((shownNumberRaw ?? 0) + delta); } function getIncrementSize(isShiftKeyPressed: boolean, isAltKeyPressed: boolean): number { @@ -171,6 +173,9 @@ export const FancyNumericInput = React.memo(function FancyNumericInput( if (typeof shownNumber === 'number') { changeValue(shownNumber); } + if (valueAsString === '' && onValueEmpty) { + onValueEmpty(); + } }} onBlur={e => { setShownValue(numberToShown(effectiveValue)); diff --git a/web-console/src/dialogs/supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog.scss b/web-console/src/dialogs/supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog.scss new file mode 100644 index 000000000000..23015cf0bb0b --- /dev/null +++ b/web-console/src/dialogs/supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog.scss @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import '../../variables'; + +.supervisor-reset-offsets-dialog { + .#{$bp-ns}-dialog-body { + position: relative; + min-height: 50vh; + overflow: auto; + max-height: 80vh; + } + + .label-button { + pointer-events: none; + } +} diff --git a/web-console/src/dialogs/supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog.tsx b/web-console/src/dialogs/supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog.tsx index 8fd8fdba5641..d9247b809f11 100644 --- a/web-console/src/dialogs/supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog.tsx +++ b/web-console/src/dialogs/supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog.tsx @@ -19,10 +19,13 @@ import { Button, Classes, Code, ControlGroup, Dialog, FormGroup, Intent } from '@blueprintjs/core'; import React, { useState } from 'react'; +import { Loader } from '../../components'; import { FancyNumericInput } from '../../components/fancy-numeric-input/fancy-numeric-input'; import { useQueryManager } from '../../hooks'; import { Api, AppToaster } from '../../singletons'; -import { deepGet, getDruidErrorMessage } from '../../utils'; +import { deepDelete, deepGet, getDruidErrorMessage } from '../../utils'; + +import './supervisor-reset-offsets-dialog.scss'; type OffsetMap = Record; @@ -50,6 +53,7 @@ export const SupervisorResetOffsetsDialog = React.memo(function SupervisorResetO const stream = deepGet(statusResp.data || {}, 'payload.stream'); const latestOffsets = deepGet(statusResp.data || {}, 'payload.latestOffsets'); + const latestOffsetsEntries = latestOffsets ? Object.entries(latestOffsets) : undefined; async function onSave() { if (!stream) return; @@ -89,29 +93,39 @@ export const SupervisorResetOffsetsDialog = React.memo(function SupervisorResetO onClose={onClose} title={`Set supervisor offsets: ${supervisorId}`} > -
-
-

- Set {supervisorId} to specific offsets -

- {latestOffsets && - Object.entries(latestOffsets).map(([key, latestOffset]) => ( - +
+ {statusResp.loading && } + {latestOffsetsEntries && ( + <> +

+ Set {supervisorId} to specific offsets +

+ {latestOffsetsEntries.map(([key, latestOffset]) => ( + -
+ {latestOffsetsEntries.length === 0 && ( +

There are no partitions currently in this supervisor.

+ )} + + )} +
+
+
+
+
+
+ + ); +}); diff --git a/web-console/src/views/workbench-view/run-panel/run-panel.tsx b/web-console/src/views/workbench-view/run-panel/run-panel.tsx index d9b5a1a34c01..ec1b95ad38a4 100644 --- a/web-console/src/views/workbench-view/run-panel/run-panel.tsx +++ b/web-console/src/views/workbench-view/run-panel/run-panel.tsx @@ -57,6 +57,7 @@ import { } from '../../../druid-models'; import { deepGet, deepSet, pluralIfNeeded, tickIcon } from '../../../utils'; import { MaxTasksButton } from '../max-tasks-button/max-tasks-button'; +import { QueryParametersDialog } from '../query-parameters-dialog/query-parameters-dialog'; import './run-panel.scss'; @@ -97,6 +98,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { const { query, onQueryChange, onRun, moreMenu, running, small, queryEngines, clusterCapacity } = props; const [editContextDialogOpen, setEditContextDialogOpen] = useState(false); + const [editParametersDialogOpen, setEditParametersDialogOpen] = useState(false); const [customTimezoneDialogOpen, setCustomTimezoneDialogOpen] = useState(false); const [indexSpecDialogSpec, setIndexSpecDialogSpec] = useState(); @@ -104,6 +106,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { const ingestMode = query.isIngestQuery(); const queryContext = query.queryContext; const numContextKeys = Object.keys(queryContext).length; + const queryParameters = query.queryParameters; const maxParseExceptions = getMaxParseExceptions(queryContext); const finalizeAggregations = getFinalizeAggregations(queryContext); @@ -238,6 +241,12 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { onClick={() => setEditContextDialogOpen(true)} label={pluralIfNeeded(numContextKeys, 'key')} /> + setEditParametersDialogOpen(true)} + label={queryParameters ? pluralIfNeeded(queryParameters.length, 'parameter') : ''} + /> {effectiveEngine !== 'native' && ( )} + {editParametersDialogOpen && ( + onQueryChange(query.changeQueryParameters(p))} + onClose={() => { + setEditParametersDialogOpen(false); + }} + /> + )} {customTimezoneDialogOpen && ( Date: Wed, 30 Aug 2023 13:36:50 -0400 Subject: [PATCH 011/258] Added brush to time-chart (#14929) --- .../modules/time-chart-echarts-module.ts | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/web-console/src/views/explore-view/modules/time-chart-echarts-module.ts b/web-console/src/views/explore-view/modules/time-chart-echarts-module.ts index 690d1c1f9e15..b4846648fdcc 100644 --- a/web-console/src/views/explore-view/modules/time-chart-echarts-module.ts +++ b/web-console/src/views/explore-view/modules/time-chart-echarts-module.ts @@ -93,7 +93,7 @@ export default typedVisualModule({ }, }, }, - module: ({ container, host }) => { + module: ({ container, host, updateWhere }) => { const myChart = echarts.init(container, 'dark'); myChart.setOption({ @@ -103,6 +103,7 @@ export default typedVisualModule({ }, tooltip: { trigger: 'axis', + transitionDuration: 0, axisPointer: { type: 'cross', label: { @@ -118,6 +119,10 @@ export default typedVisualModule({ saveAsImage: {}, }, }, + brush: { + toolbox: ['lineX'], + xAxisIndex: 0, + }, grid: { left: '3%', right: '4%', @@ -148,6 +153,8 @@ export default typedVisualModule({ async update({ table, where, parameterValues }) { const { splitColumn, metric, numberToStack, showOthers, timeGranularity } = parameterValues; + myChart.off('brushend'); + const vs = splitColumn ? ( await host.sqlQuery( @@ -187,6 +194,28 @@ export default typedVisualModule({ const effectiveVs = vs && showOthers ? vs.concat(OTHERS_VALUE) : vs; const sourceData = effectiveVs ? transformData(dataset, effectiveVs) : dataset; + myChart.on('brushend', (params: any) => { + if (!params.areas.length) return; + + const [start, end] = params.areas[0].coordRange; + + updateWhere( + where.changeClauseInWhere( + SqlExpression.parse( + `TIME_IN_INTERVAL(${C('__time')}, '${new Date(start).toISOString()}/${new Date( + end, + ).toISOString()}')`, + ), + ), + ); + + myChart.dispatchAction({ + type: 'brush', + command: 'clear', + areas: [], + }); + }); + const showSymbol = sourceData.length < 2; myChart.setOption( { @@ -194,6 +223,7 @@ export default typedVisualModule({ dimensions: ['time'].concat(vs || ['met']), source: sourceData, }, + animation: false, legend: effectiveVs ? { data: effectiveVs, From 04a1153d0f55c228cadcc0036772efe554f56eb4 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 30 Aug 2023 15:59:26 -0700 Subject: [PATCH 012/258] line chart fix others not mapping correctly (#14931) --- web-console/src/console-application.scss | 2 +- .../modules/time-chart-echarts-module.ts | 29 +++++++++++-------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/web-console/src/console-application.scss b/web-console/src/console-application.scss index 3c527b3e1291..1fa3f1dd6889 100644 --- a/web-console/src/console-application.scss +++ b/web-console/src/console-application.scss @@ -37,7 +37,7 @@ } &.thinner { - padding: 5px; + padding: 8px; } .app-view { diff --git a/web-console/src/views/explore-view/modules/time-chart-echarts-module.ts b/web-console/src/views/explore-view/modules/time-chart-echarts-module.ts index b4846648fdcc..d0f7ade6ba42 100644 --- a/web-console/src/views/explore-view/modules/time-chart-echarts-module.ts +++ b/web-console/src/views/explore-view/modules/time-chart-echarts-module.ts @@ -22,6 +22,9 @@ import * as echarts from 'echarts'; import { getInitQuery } from '../utils'; +const TIME_NAME = '__t__'; +const METRIC_NAME = '__met__'; +const STACK_NAME = '__stack__'; const OTHERS_VALUE = 'Others'; function transformData(data: any[], vs: string[]): Record[] { @@ -31,12 +34,13 @@ function transformData(data: any[], vs: string[]): Record[] { let lastDatum: Record | undefined; const ret = []; for (const d of data) { - if (d.time.valueOf() !== lastTime) { + const t = d[TIME_NAME]; + if (t.valueOf() !== lastTime) { if (lastDatum) ret.push(lastDatum); - lastTime = d.time.valueOf(); - lastDatum = { ...zeroDatum, time: d.time }; + lastTime = t.valueOf(); + lastDatum = { ...zeroDatum, [TIME_NAME]: t }; } - lastDatum![d.stack] = d.met; + lastDatum![d[STACK_NAME]] = d[METRIC_NAME]; } if (lastDatum) ret.push(lastDatum); return ret; @@ -78,7 +82,7 @@ export default typedVisualModule({ }, showOthers: { type: 'boolean', - default: false, + default: true, control: { visible: ({ params }) => Boolean(params.splitColumn), }, @@ -172,7 +176,7 @@ export default typedVisualModule({ table, splitColumn && vs && !showOthers ? where.and(splitColumn.expression.in(vs)) : where, ) - .addSelect(F.timeFloor(C('__time'), L(timeGranularity)).as('time'), { + .addSelect(F.timeFloor(C('__time'), L(timeGranularity)).as(TIME_NAME), { addToGroupBy: 'end', addToOrderBy: 'end', direction: 'ASC', @@ -183,11 +187,11 @@ export default typedVisualModule({ (showOthers ? SqlCase.ifThenElse(splitEx.in(vs!), splitEx, L(OTHERS_VALUE)) : splitEx - ).as('stack'), + ).as(STACK_NAME), { addToGroupBy: 'end' }, ); }) - .addSelect(metric.expression.as('met')), + .addSelect(metric.expression.as(METRIC_NAME)), ) ).toObjectArray(); @@ -220,7 +224,7 @@ export default typedVisualModule({ myChart.setOption( { dataset: { - dimensions: ['time'].concat(vs || ['met']), + dimensions: [TIME_NAME].concat(effectiveVs || [METRIC_NAME]), source: sourceData, }, animation: false, @@ -229,19 +233,20 @@ export default typedVisualModule({ data: effectiveVs, } : undefined, - series: (effectiveVs || ['met']).map(v => { + series: (effectiveVs || [METRIC_NAME]).map(v => { return { id: v, name: effectiveVs ? v : metric.name, type: 'line', stack: 'Total', showSymbol, - areaStyle: {}, + lineStyle: v === OTHERS_VALUE ? { color: '#ccc' } : {}, + areaStyle: v === OTHERS_VALUE ? { color: '#ccc' } : {}, emphasis: { focus: 'series', }, encode: { - x: 'time', + x: TIME_NAME, y: v, itemId: v, }, From 680669fd3a72524be71070375a11e6da637a9bec Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 30 Aug 2023 15:59:34 -0700 Subject: [PATCH 013/258] show execution dialog in task view (#14930) --- .../helpers/execution/sql-task-execution.ts | 20 +++-- .../src/views/tasks-view/tasks-view.tsx | 76 +++++++++++++------ .../execution-details-pane.tsx | 1 + .../execution-stages-pane.tsx | 2 +- 4 files changed, 65 insertions(+), 34 deletions(-) diff --git a/web-console/src/helpers/execution/sql-task-execution.ts b/web-console/src/helpers/execution/sql-task-execution.ts index 1643e8a076d0..d41370df16cf 100644 --- a/web-console/src/helpers/execution/sql-task-execution.ts +++ b/web-console/src/helpers/execution/sql-task-execution.ts @@ -224,16 +224,20 @@ export async function getTaskExecution( execution = execution.updateWithTaskPayload(taskPayload); } - // Still have to pull the destination page info from the async status + // Still have to pull the destination page info from the async status, do this in a best effort way since the statements API may have permission errors if (execution.status === 'SUCCESS' && !execution.destinationPages) { - const statusResp = await Api.instance.get( - `/druid/v2/sql/statements/${encodedId}`, - { - cancelToken, - }, - ); + try { + const statusResp = await Api.instance.get( + `/druid/v2/sql/statements/${encodedId}`, + { + cancelToken, + }, + ); - execution = execution.updateWithAsyncStatus(statusResp.data); + execution = execution.updateWithAsyncStatus(statusResp.data); + } catch (e) { + if (Api.isNetworkError(e)) throw e; + } } if (execution.hasPotentiallyStuckStage()) { diff --git a/web-console/src/views/tasks-view/tasks-view.tsx b/web-console/src/views/tasks-view/tasks-view.tsx index b9208cbed0a1..795c9908412a 100644 --- a/web-console/src/views/tasks-view/tasks-view.tsx +++ b/web-console/src/views/tasks-view/tasks-view.tsx @@ -52,6 +52,7 @@ import { QueryState, } from '../../utils'; import type { BasicAction } from '../../utils/basic-action'; +import { ExecutionDetailsDialog } from '../workbench-view/execution-details-dialog/execution-details-dialog'; import './tasks-view.scss'; @@ -99,9 +100,8 @@ export interface TasksViewState { taskSpecDialogOpen: boolean; alertErrorMsg?: string; - taskTableActionDialogId?: string; - taskTableActionDialogStatus?: string; - taskTableActionDialogActions: BasicAction[]; + taskTableActionDialogOpen?: { id: string; status: string; actions: BasicAction[] }; + executionDialogOpen?: string; visibleColumns: LocalStorageBackedVisibility; } @@ -160,8 +160,6 @@ ORDER BY taskSpecDialogOpen: Boolean(props.openTaskDialog), - taskTableActionDialogActions: [], - visibleColumns: new LocalStorageBackedVisibility( LocalStorageKeys.TASK_TABLE_COLUMN_SELECTION, ), @@ -243,10 +241,26 @@ ORDER BY datasource: string, status: string, type: string, + fromTable?: boolean, ): BasicAction[] { const { goToDatasource, goToClassicBatchDataLoader } = this.props; const actions: BasicAction[] = []; + if (fromTable) { + actions.push({ + icon: IconNames.SEARCH_TEMPLATE, + title: 'View raw details', + onAction: () => { + this.setState({ + taskTableActionDialogOpen: { + id, + status, + actions: this.getTaskActions(id, datasource, status, type), + }, + }); + }, + }); + } if (datasource && status === 'SUCCESS') { actions.push({ icon: IconNames.MULTI_SELECT, @@ -317,16 +331,19 @@ ORDER BY } private onTaskDetail(task: TaskQueryResultRow) { - this.setState({ - taskTableActionDialogId: task.task_id, - taskTableActionDialogStatus: task.status, - taskTableActionDialogActions: this.getTaskActions( - task.task_id, - task.datasource, - task.status, - task.type, - ), - }); + if (task.type === 'query_controller') { + this.setState({ + executionDialogOpen: task.task_id, + }); + } else { + this.setState({ + taskTableActionDialogOpen: { + id: task.task_id, + status: task.status, + actions: this.getTaskActions(task.task_id, task.datasource, task.status, task.type), + }, + }); + } } private renderTaskTable() { @@ -482,11 +499,10 @@ ORDER BY const id = row.value; const type = row.row.type; const { datasource, status } = row.original; - const taskActions = this.getTaskActions(id, datasource, status, type); return ( this.onTaskDetail(row.original)} - actions={taskActions} + actions={this.getTaskActions(id, datasource, status, type, true)} /> ); }, @@ -520,13 +536,13 @@ ORDER BY } render() { + const { onFiltersChange } = this.props; const { groupTasksBy, taskSpecDialogOpen, + executionDialogOpen, alertErrorMsg, - taskTableActionDialogId, - taskTableActionDialogActions, - taskTableActionDialogStatus, + taskTableActionDialogOpen, visibleColumns, } = this.state; @@ -603,12 +619,22 @@ ORDER BY >

{alertErrorMsg}

- {taskTableActionDialogId && taskTableActionDialogStatus && ( + {taskTableActionDialogOpen && ( this.setState({ taskTableActionDialogId: undefined })} + taskId={taskTableActionDialogOpen.id} + status={taskTableActionDialogOpen.status} + actions={taskTableActionDialogOpen.actions} + onClose={() => this.setState({ taskTableActionDialogOpen: undefined })} + /> + )} + {executionDialogOpen && ( + { + onFiltersChange([{ id: 'task_id', value: `=${taskId}` }]); + this.setState({ executionDialogOpen: undefined }); + }} + onClose={() => this.setState({ executionDialogOpen: undefined })} /> )}
diff --git a/web-console/src/views/workbench-view/execution-details-pane/execution-details-pane.tsx b/web-console/src/views/workbench-view/execution-details-pane/execution-details-pane.tsx index e448fd8a5078..72a6350d7ba1 100644 --- a/web-console/src/views/workbench-view/execution-details-pane/execution-details-pane.tsx +++ b/web-console/src/views/workbench-view/execution-details-pane/execution-details-pane.tsx @@ -95,6 +95,7 @@ export const ExecutionDetailsPane = React.memo(function ExecutionDetailsPane( ? String(execution.sqlQuery) : JSONBig.stringify(execution.nativeQuery, undefined, 2) } + leaveBackground /> ); diff --git a/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx b/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx index 57827534df11..1c06446eb132 100644 --- a/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx +++ b/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx @@ -221,7 +221,7 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane( accessor: d => d.index, width: 100, Cell({ value }) { - const taskId = `${execution.id}-worker${value}`; + const taskId = `${execution.id}-worker${value}_0`; return ( Date: Wed, 30 Aug 2023 22:35:08 -0700 Subject: [PATCH 014/258] cleaning DruidProcessingConfig bindings (#14927) --- .../java/org/apache/druid/guice/BrokerProcessingModule.java | 2 -- .../main/java/org/apache/druid/guice/DruidProcessingModule.java | 2 -- .../java/org/apache/druid/guice/RouterProcessingModule.java | 2 -- .../src/main/java/org/apache/druid/guice/StorageNodeModule.java | 2 +- 4 files changed, 1 insertion(+), 7 deletions(-) diff --git a/server/src/main/java/org/apache/druid/guice/BrokerProcessingModule.java b/server/src/main/java/org/apache/druid/guice/BrokerProcessingModule.java index 77572bae47a5..0fcb4785191e 100644 --- a/server/src/main/java/org/apache/druid/guice/BrokerProcessingModule.java +++ b/server/src/main/java/org/apache/druid/guice/BrokerProcessingModule.java @@ -46,7 +46,6 @@ import org.apache.druid.query.ExecutorServiceMonitor; import org.apache.druid.query.ForwardingQueryProcessingPool; import org.apache.druid.query.QueryProcessingPool; -import org.apache.druid.segment.column.ColumnConfig; import org.apache.druid.server.metrics.MetricsModule; import org.apache.druid.utils.JvmUtils; @@ -70,7 +69,6 @@ public void configure(Binder binder) { JsonConfigProvider.bind(binder, "druid.processing.merge", BrokerParallelMergeConfig.class); JsonConfigProvider.bind(binder, "druid.processing", DruidProcessingConfig.class); - binder.bind(ColumnConfig.class).to(DruidProcessingConfig.class); MetricsModule.register(binder, ExecutorServiceMonitor.class); } diff --git a/server/src/main/java/org/apache/druid/guice/DruidProcessingModule.java b/server/src/main/java/org/apache/druid/guice/DruidProcessingModule.java index 56a0fd0ede6f..76cc855cd224 100644 --- a/server/src/main/java/org/apache/druid/guice/DruidProcessingModule.java +++ b/server/src/main/java/org/apache/druid/guice/DruidProcessingModule.java @@ -46,7 +46,6 @@ import org.apache.druid.query.MetricsEmittingQueryProcessingPool; import org.apache.druid.query.PrioritizedExecutorService; import org.apache.druid.query.QueryProcessingPool; -import org.apache.druid.segment.column.ColumnConfig; import org.apache.druid.server.metrics.MetricsModule; import org.apache.druid.utils.JvmUtils; @@ -64,7 +63,6 @@ public class DruidProcessingModule implements Module public void configure(Binder binder) { JsonConfigProvider.bind(binder, "druid.processing", DruidProcessingConfig.class); - binder.bind(ColumnConfig.class).to(DruidProcessingConfig.class); MetricsModule.register(binder, ExecutorServiceMonitor.class); } diff --git a/server/src/main/java/org/apache/druid/guice/RouterProcessingModule.java b/server/src/main/java/org/apache/druid/guice/RouterProcessingModule.java index 3b68289c6dd1..dc68cbb4bffa 100644 --- a/server/src/main/java/org/apache/druid/guice/RouterProcessingModule.java +++ b/server/src/main/java/org/apache/druid/guice/RouterProcessingModule.java @@ -34,7 +34,6 @@ import org.apache.druid.query.ExecutorServiceMonitor; import org.apache.druid.query.ForwardingQueryProcessingPool; import org.apache.druid.query.QueryProcessingPool; -import org.apache.druid.segment.column.ColumnConfig; import org.apache.druid.server.metrics.MetricsModule; import java.nio.ByteBuffer; @@ -54,7 +53,6 @@ public class RouterProcessingModule implements Module public void configure(Binder binder) { JsonConfigProvider.bind(binder, "druid.processing", DruidProcessingConfig.class); - binder.bind(ColumnConfig.class).to(DruidProcessingConfig.class); MetricsModule.register(binder, ExecutorServiceMonitor.class); } diff --git a/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java b/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java index cc4e0b1e33a4..3ae5d2a3586f 100644 --- a/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java +++ b/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java @@ -57,7 +57,7 @@ public void configure(Binder binder) JsonConfigProvider.bind(binder, "druid.segmentCache", SegmentLoaderConfig.class); bindLocationSelectorStrategy(binder); binder.bind(ServerTypeConfig.class).toProvider(Providers.of(null)); - binder.bind(ColumnConfig.class).to(DruidProcessingConfig.class); + binder.bind(ColumnConfig.class).to(DruidProcessingConfig.class).in(LazySingleton.class); } @Provides From 7f26b80e2136a122952a1dea58a135edeb971177 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Fri, 1 Sep 2023 11:30:45 +0530 Subject: [PATCH 015/258] Simplify ServiceMetricEvent.Builder (#14933) Changes: - Make ServiceMetricEvent.Builder extend ServiceEventBuilder and thus convert it to a plain builder rather than a builder of builder. - Add methods setCreatedTime , setMetricAndValue to the builder --- ...asedDruidToTimelineEventConverterTest.java | 18 ++- .../graphite/WhiteListBasedConverterTest.java | 17 ++- .../emitter/influxdb/InfluxdbEmitterTest.java | 8 +- .../druid/emitter/kafka/KafkaEmitterTest.java | 2 +- .../k8s/overlord/KubernetesTaskRunner.java | 2 +- .../overlord/common/KubernetesPeonClient.java | 2 +- .../DataSourceOptimizerMonitor.java | 12 +- .../DataSourceOptimizerMonitorTest.java | 68 +++++++++++ .../OpenTelemetryEmitterTest.java | 68 +++-------- .../emitter/opentsdb/EventConverterTest.java | 18 ++- .../prometheus/PrometheusEmitterTest.java | 18 +-- .../client/cache/AbstractRedisCache.java | 4 +- .../statsd/DimensionConverterTest.java | 3 +- .../emitter/statsd/StatsDEmitterTest.java | 15 ++- .../indexing/kafka/KafkaConsumerMonitor.java | 2 +- .../namespace/cache/CacheScheduler.java | 2 +- ...ffHeapNamespaceExtractionCacheManager.java | 4 +- ...OnHeapNamespaceExtractionCacheManager.java | 4 +- .../common/actions/LocalTaskActionClient.java | 2 +- .../actions/SegmentAllocationQueue.java | 4 +- .../actions/SegmentMetadataUpdateAction.java | 2 +- .../common/actions/SegmentNukeAction.java | 2 +- .../SegmentTransactionalInsertAction.java | 6 +- .../stats/TaskRealtimeMetricsMonitor.java | 38 +++---- .../common/task/AbstractBatchIndexTask.java | 2 +- .../indexing/common/task/AbstractTask.java | 2 +- .../indexing/common/task/CompactionTask.java | 2 +- .../indexing/overlord/RemoteTaskRunner.java | 2 +- .../overlord/SingleTaskBackgroundRunner.java | 4 +- .../druid/indexing/overlord/TaskQueue.java | 2 +- .../overlord/hrtr/HttpRemoteTaskRunner.java | 2 +- .../supervisor/SeekableStreamSupervisor.java | 12 +- .../worker/shuffle/ShuffleMonitor.java | 4 +- .../util/emitter/service/ServiceEmitter.java | 2 +- .../emitter/service/ServiceMetricEvent.java | 74 ++++++------ .../java/util/metrics/CgroupCpuMonitor.java | 4 +- .../util/metrics/CgroupCpuSetMonitor.java | 8 +- .../util/metrics/CgroupMemoryMonitor.java | 4 +- .../util/metrics/CpuAcctDeltaMonitor.java | 8 +- .../util/metrics/HttpPostEmitterMonitor.java | 24 ++-- .../java/util/metrics/JvmCpuMonitor.java | 4 +- .../druid/java/util/metrics/JvmMonitor.java | 36 +++--- .../java/util/metrics/JvmThreadsMonitor.java | 10 +- .../java/util/metrics/OshiSysMonitor.java | 18 +-- .../druid/java/util/metrics/SysMonitor.java | 22 ++-- .../druid/query/DefaultQueryMetrics.java | 2 +- .../MetricsEmittingQueryProcessingPool.java | 2 +- .../core/HttpPostEmitterLoggerStressTest.java | 4 +- .../core/HttpPostEmitterStressTest.java | 6 +- .../service/ServiceMetricEventTest.java | 31 ++--- .../apache/druid/client/BrokerServerView.java | 2 +- .../druid/client/CoordinatorServerView.java | 2 +- .../druid/client/HttpServerInventoryView.java | 4 +- .../druid/client/cache/CacheMonitor.java | 24 ++-- .../druid/client/cache/CaffeineCache.java | 12 +- .../druid/client/cache/MemcachedCache.java | 4 +- .../curator/DruidConnectionStateListener.java | 4 +- .../realtime/RealtimeMetricsMonitor.java | 34 +++--- .../apache/druid/server/QueryScheduler.java | 2 +- .../druid/server/audit/SQLAuditManager.java | 2 +- .../server/coordinator/DruidCoordinator.java | 2 +- .../jetty/JettyServerModule.java | 16 +-- .../metrics/EventReceiverFirehoseMonitor.java | 4 +- .../metrics/HistoricalMetricsMonitor.java | 10 +- .../metrics/QueryCountStatsMonitor.java | 2 +- .../server/metrics/SegmentStatsMonitor.java | 4 +- .../server/metrics/ServiceStatusMonitor.java | 2 +- .../server/metrics/TaskCountStatsMonitor.java | 4 +- .../metrics/TaskSlotCountStatsMonitor.java | 2 +- .../metrics/WorkerTaskCountStatsMonitor.java | 2 +- .../realtime/RealtimeMetricsMonitorTest.java | 107 ++++++++++++++++++ .../log/ServiceMetricEventSerdeTest.java | 3 +- .../metrics/HistoricalMetricsMonitorTest.java | 69 ++++------- .../metrics/SegmentStatsMonitorTest.java | 4 +- .../druid/sql/SqlExecutionReporter.java | 6 +- .../druid/sql/avatica/AvaticaMonitor.java | 4 +- .../calcite/schema/SegmentMetadataCache.java | 6 +- 77 files changed, 529 insertions(+), 420 deletions(-) create mode 100644 extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitorTest.java create mode 100644 server/src/test/java/org/apache/druid/segment/realtime/RealtimeMetricsMonitorTest.java diff --git a/extensions-contrib/ambari-metrics-emitter/src/test/java/org/apache/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverterTest.java b/extensions-contrib/ambari-metrics-emitter/src/test/java/org/apache/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverterTest.java index 094f9f28b84d..eded587fb2ed 100644 --- a/extensions-contrib/ambari-metrics-emitter/src/test/java/org/apache/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverterTest.java +++ b/extensions-contrib/ambari-metrics-emitter/src/test/java/org/apache/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverterTest.java @@ -24,10 +24,8 @@ import org.apache.commons.io.IOUtils; import org.apache.druid.annotations.UsedByJUnitParamsRunner; import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; -import org.joda.time.DateTime; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -37,14 +35,12 @@ import java.io.IOException; import java.io.OutputStream; - @RunWith(JUnitParamsRunner.class) public class WhiteListBasedDruidToTimelineEventConverterTest { private final String prefix = "druid"; private final WhiteListBasedDruidToTimelineEventConverter defaultWhiteListBasedDruidToTimelineEventConverter = new WhiteListBasedDruidToTimelineEventConverter(prefix, "druid", null, new DefaultObjectMapper()); - private final DateTime createdTime = DateTimes.nowUtc(); private final String hostname = "testHost:8080"; private final String serviceName = "historical"; private final String defaultNamespace = prefix + "." + serviceName; @@ -77,7 +73,7 @@ public void testDefaultIsInWhiteList(String key, boolean expectedValue) ServiceMetricEvent event = ServiceMetricEvent .builder() .setFeed("metrics") - .build(createdTime, key, 10) + .setMetric(key, 10) .build(serviceName, hostname); boolean isIn = defaultWhiteListBasedDruidToTimelineEventConverter.druidEventToTimelineMetric(event) != null; @@ -118,7 +114,7 @@ public void testWhiteListedStringArrayDimension() throws IOException ServiceMetricEvent event = new ServiceMetricEvent.Builder() .setDimension("gcName", new String[] {"g1"}) - .build(createdTime, "jvm/gc/cpu", 10) + .setMetric("jvm/gc/cpu", 10) .build(serviceName, hostname); TimelineMetric metric = converter.druidEventToTimelineMetric(event); @@ -136,14 +132,14 @@ private Object[] parametersForTestGetName() .setDimension("status", "some_status") .setDimension("numDimensions", "1") .setDimension("segment", "dummy_segment") - .build(createdTime, "query/segment/time/balabla/more", 10) + .setMetric("query/segment/time/balabla/more", 10) .build(serviceName, hostname), defaultNamespace + ".query/segment/time/balabla/more" }, new Object[]{ new ServiceMetricEvent.Builder().setDimension("dataSource", "some_data_source") .setDimension("tier", "_default_tier") - .build(createdTime, "segment/max", 10) + .setMetric("segment/max", 10) .build(serviceName, hostname), null }, @@ -158,7 +154,7 @@ private Object[] parametersForTestGetName() .setDimension("remoteAddress", "194.0.90.2") .setDimension("id", "ID") .setDimension("context", "{context}") - .build(createdTime, "query/time", 10) + .setMetric("query/time", 10) .build(serviceName, hostname), defaultNamespace + ".data-source.groupBy.query/time" }, @@ -166,7 +162,7 @@ private Object[] parametersForTestGetName() new ServiceMetricEvent.Builder().setDimension("dataSource", "data-source") .setDimension("type", "groupBy") .setDimension("some_random_dim1", "random_dim_value1") - .build(createdTime, "ingest/persists/count", 10) + .setMetric("ingest/persists/count", 10) .build(serviceName, hostname), defaultNamespace + ".data-source.ingest/persists/count" }, @@ -174,7 +170,7 @@ private Object[] parametersForTestGetName() new ServiceMetricEvent.Builder().setDimension("bufferpoolName", "BufferPool") .setDimension("type", "groupBy") .setDimension("some_random_dim1", "random_dim_value1") - .build(createdTime, "jvm/bufferpool/capacity", 10) + .setMetric("jvm/bufferpool/capacity", 10) .build(serviceName, hostname), null } diff --git a/extensions-contrib/graphite-emitter/src/test/java/org/apache/druid/emitter/graphite/WhiteListBasedConverterTest.java b/extensions-contrib/graphite-emitter/src/test/java/org/apache/druid/emitter/graphite/WhiteListBasedConverterTest.java index cce8b45b3b4a..a1a4f999549d 100644 --- a/extensions-contrib/graphite-emitter/src/test/java/org/apache/druid/emitter/graphite/WhiteListBasedConverterTest.java +++ b/extensions-contrib/graphite-emitter/src/test/java/org/apache/druid/emitter/graphite/WhiteListBasedConverterTest.java @@ -24,9 +24,7 @@ import org.apache.commons.io.IOUtils; import org.apache.druid.annotations.UsedByJUnitParamsRunner; import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; -import org.joda.time.DateTime; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -49,7 +47,6 @@ public class WhiteListBasedConverterTest null, new DefaultObjectMapper() ); - private final DateTime createdTime = DateTimes.nowUtc(); private final String hostname = "testHost.yahoo.com:8080"; private final String serviceName = "historical"; private final String defaultNamespace = prefix + "." + serviceName + "." + GraphiteEmitter.sanitize(hostname); @@ -81,7 +78,7 @@ public void testDefaultIsInWhiteList(String key, boolean expectedValue) { ServiceMetricEvent event = ServiceMetricEvent .builder() - .build(createdTime, key, 10) + .setMetric(key, 10) .build(serviceName, hostname); boolean isIn = defaultWhiteListBasedConverter.druidEventToGraphite(event) != null; @@ -124,7 +121,7 @@ public void testWhiteListedStringArrayDimension() throws IOException ServiceMetricEvent event = new ServiceMetricEvent.Builder() .setDimension("gcName", new String[]{"g1"}) - .build(createdTime, "jvm/gc/cpu", 10) + .setMetric("jvm/gc/cpu", 10) .build(serviceName, hostname); GraphiteEvent graphiteEvent = converter.druidEventToGraphite(event); @@ -142,14 +139,14 @@ private Object[] parametersForTestGetPath() .setDimension("status", "some_status") .setDimension("numDimensions", "1") .setDimension("segment", "dummy_segment") - .build(createdTime, "query/segment/time/balabla/more", 10) + .setMetric("query/segment/time/balabla/more", 10) .build(serviceName, hostname), defaultNamespace + ".query/segment/time/balabla/more" }, new Object[]{ new ServiceMetricEvent.Builder().setDimension("dataSource", "some_data_source") .setDimension("tier", "_default_tier") - .build(createdTime, "segment/max", 10) + .setMetric("segment/max", 10) .build(serviceName, hostname), null }, @@ -164,7 +161,7 @@ private Object[] parametersForTestGetPath() .setDimension("remoteAddress", "194.0.90.2") .setDimension("id", "ID") .setDimension("context", "{context}") - .build(createdTime, "query/time", 10) + .setMetric("query/time", 10) .build(serviceName, hostname), defaultNamespace + ".data-source.groupBy.query/time" }, @@ -172,7 +169,7 @@ private Object[] parametersForTestGetPath() new ServiceMetricEvent.Builder().setDimension("dataSource", "data-source") .setDimension("type", "groupBy") .setDimension("some_random_dim1", "random_dim_value1") - .build(createdTime, "ingest/persists/count", 10) + .setMetric("ingest/persists/count", 10) .build(serviceName, hostname), defaultNamespace + ".ingest/persists/count" }, @@ -180,7 +177,7 @@ private Object[] parametersForTestGetPath() new ServiceMetricEvent.Builder().setDimension("bufferpoolName", "BufferPool") .setDimension("type", "groupBy") .setDimension("some_random_dim1", "random_dim_value1") - .build(createdTime, "jvm/bufferpool/capacity", 10) + .setMetric("jvm/bufferpool/capacity", 10) .build(serviceName, hostname), null } diff --git a/extensions-contrib/influxdb-emitter/src/test/java/org/apache/druid/emitter/influxdb/InfluxdbEmitterTest.java b/extensions-contrib/influxdb-emitter/src/test/java/org/apache/druid/emitter/influxdb/InfluxdbEmitterTest.java index f39549250f47..cc66cc2c5d9e 100644 --- a/extensions-contrib/influxdb-emitter/src/test/java/org/apache/druid/emitter/influxdb/InfluxdbEmitterTest.java +++ b/extensions-contrib/influxdb-emitter/src/test/java/org/apache/druid/emitter/influxdb/InfluxdbEmitterTest.java @@ -56,7 +56,7 @@ public void setUp() ServiceMetricEvent.Builder builder = ServiceMetricEvent.builder(); builder.setDimension("nonWhiteListedDim", "test"); builder.setDimension("dataSource", "test_datasource"); - ServiceEventBuilder eventBuilder = builder.build(date, metric, value); + ServiceEventBuilder eventBuilder = builder.setCreatedTime(date).setMetric(metric, value); event = (ServiceMetricEvent) eventBuilder.build(serviceDims); } @@ -106,7 +106,7 @@ public void testTransformForInfluxWithShortMetric() "0.10.0" ); ServiceMetricEvent.Builder builder = ServiceMetricEvent.builder(); - ServiceEventBuilder eventBuilder = builder.build(date, metric, value); + ServiceEventBuilder eventBuilder = builder.setCreatedTime(date).setMetric(metric, value); ServiceMetricEvent event = (ServiceMetricEvent) eventBuilder.build(serviceDims); InfluxdbEmitterConfig config = new InfluxdbEmitterConfig( "localhost", @@ -150,7 +150,7 @@ public void testMetricIsInDimensionWhitelist() "0.10.0" ); ServiceMetricEvent.Builder builder = ServiceMetricEvent.builder(); - ServiceEventBuilder eventBuilder = builder.build(date, metric, value); + ServiceEventBuilder eventBuilder = builder.setCreatedTime(date).setMetric(metric, value); builder.setDimension("dataSource", "wikipedia"); builder.setDimension("taskType", "index"); ServiceMetricEvent event = (ServiceMetricEvent) eventBuilder.build(serviceDims); @@ -196,7 +196,7 @@ public void testMetricIsInDefaultDimensionWhitelist() "0.10.0" ); ServiceMetricEvent.Builder builder = ServiceMetricEvent.builder(); - ServiceEventBuilder eventBuilder = builder.build(date, metric, value); + ServiceEventBuilder eventBuilder = builder.setCreatedTime(date).setMetric(metric, value); builder.setDimension("dataSource", "wikipedia"); builder.setDimension("taskType", "index"); ServiceMetricEvent event = (ServiceMetricEvent) eventBuilder.build(serviceDims); diff --git a/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterTest.java b/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterTest.java index 5d0df12eadfc..9e6846a5d8b8 100644 --- a/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterTest.java +++ b/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterTest.java @@ -72,7 +72,7 @@ public static Object[] data() public void testKafkaEmitter() throws InterruptedException { final List serviceMetricEvents = ImmutableList.of( - ServiceMetricEvent.builder().build("m1", 1).build("service", "host") + ServiceMetricEvent.builder().setMetric("m1", 1).build("service", "host") ); final List alertEvents = ImmutableList.of( diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunner.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunner.java index 24e21b0b4e0e..14fe54778a20 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunner.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunner.java @@ -227,7 +227,7 @@ protected void emitTaskStateMetrics(KubernetesPeonLifecycle.State state, String ServiceMetricEvent.Builder metricBuilder = new ServiceMetricEvent.Builder(); IndexTaskUtils.setTaskDimensions(metricBuilder, workItem.getTask()); emitter.emit( - metricBuilder.build( + metricBuilder.setMetric( "task/pending/time", new Duration(workItem.getCreatedTime(), DateTimes.nowUtc()).getMillis() ) diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/KubernetesPeonClient.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/KubernetesPeonClient.java index 320bffba94bf..b477be2cde28 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/KubernetesPeonClient.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/KubernetesPeonClient.java @@ -271,6 +271,6 @@ private void emitK8sPodMetrics(Task task, String metric, long durationMs) { ServiceMetricEvent.Builder metricBuilder = new ServiceMetricEvent.Builder(); IndexTaskUtils.setTaskDimensions(metricBuilder, task); - emitter.emit(metricBuilder.build(metric, durationMs)); + emitter.emit(metricBuilder.setMetric(metric, durationMs)); } } diff --git a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitor.java b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitor.java index 08b0b25ebb27..03abc99a165d 100644 --- a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitor.java +++ b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitor.java @@ -45,20 +45,20 @@ public boolean doMonitor(ServiceEmitter emitter) for (DataSourceOptimizerStats stat : stats) { final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder(); builder.setDimension("dataSource", stat.getBase()); - emitter.emit(builder.build("/materialized/view/query/totalNum", stat.getTotalcount())); - emitter.emit(builder.build("/materialized/view/query/hits", stat.getHitcount())); - emitter.emit(builder.build("/materialized/view/query/hitRate", stat.getHitRate())); - emitter.emit(builder.build("/materialized/view/select/avgCostMS", stat.getOptimizerCost())); + emitter.emit(builder.setMetric("/materialized/view/query/totalNum", stat.getTotalcount())); + emitter.emit(builder.setMetric("/materialized/view/query/hits", stat.getHitcount())); + emitter.emit(builder.setMetric("/materialized/view/query/hitRate", stat.getHitRate())); + emitter.emit(builder.setMetric("/materialized/view/select/avgCostMS", stat.getOptimizerCost())); Map derivativesStats = stat.getDerivativesHitCount(); for (Map.Entry derivative : derivativesStats.entrySet()) { builder.setDimension("derivative", derivative.getKey()); - emitter.emit(builder.build("/materialized/view/derivative/numSelected", derivative.getValue())); + emitter.emit(builder.setMetric("/materialized/view/derivative/numSelected", derivative.getValue())); } final ServiceMetricEvent.Builder builder2 = new ServiceMetricEvent.Builder(); builder2.setDimension("dataSource", stat.getBase()); for (Set fields : stat.getMissFields().keySet()) { builder2.setDimension("fields", fields.toString()); - emitter.emit(builder2.build("/materialized/view/missNum", stat.getMissFields().get(fields).get())); + emitter.emit(builder2.setMetric("/materialized/view/missNum", stat.getMissFields().get(fields).get())); } } return true; diff --git a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitorTest.java b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitorTest.java new file mode 100644 index 000000000000..4e42beb0d288 --- /dev/null +++ b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitorTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.materializedview; + +import org.apache.druid.java.util.metrics.StubServiceEmitter; +import org.easymock.EasyMock; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; + +public class DataSourceOptimizerMonitorTest +{ + + @Test + public void testDoMonitor() + { + final DataSourceOptimizerStats wikiStats + = new DataSourceOptimizerStats("wiki", 100, 200, 30, Collections.emptyMap(), Collections.emptyMap()); + final DataSourceOptimizerStats koalaStats + = new DataSourceOptimizerStats("koala", 35, 62, 13, Collections.emptyMap(), Collections.emptyMap()); + + DataSourceOptimizer optimizer = EasyMock.createMock(DataSourceOptimizer.class); + EasyMock.expect(optimizer.getAndResetStats()).andReturn( + Arrays.asList(wikiStats, koalaStats) + ).anyTimes(); + EasyMock.replay(optimizer); + + DataSourceOptimizerMonitor monitor = new DataSourceOptimizerMonitor(optimizer); + StubServiceEmitter emitter = new StubServiceEmitter("test", "localhost"); + monitor.doMonitor(emitter); + + // Verify metrics emitted for all the datasources + Map wikiFilter = Collections.singletonMap("dataSource", "wiki"); + emitter.verifyValue("/materialized/view/query/totalNum", wikiFilter, wikiStats.getTotalcount()); + emitter.verifyValue("/materialized/view/query/hits", wikiFilter, wikiStats.getHitcount()); + emitter.verifyValue("/materialized/view/query/hitRate", wikiFilter, wikiStats.getHitRate()); + emitter.verifyValue("/materialized/view/select/avgCostMS", wikiFilter, wikiStats.getOptimizerCost()); + + Map koalaFilter = Collections.singletonMap("dataSource", "koala"); + emitter.verifyValue("/materialized/view/query/totalNum", koalaFilter, koalaStats.getTotalcount()); + emitter.verifyValue("/materialized/view/query/hits", koalaFilter, koalaStats.getHitcount()); + emitter.verifyValue("/materialized/view/query/hitRate", koalaFilter, koalaStats.getHitRate()); + emitter.verifyValue("/materialized/view/select/avgCostMS", koalaFilter, koalaStats.getOptimizerCost()); + + + EasyMock.verify(optimizer); + } + +} diff --git a/extensions-contrib/opentelemetry-emitter/src/test/java/org/apache/druid/emitter/opentelemetry/OpenTelemetryEmitterTest.java b/extensions-contrib/opentelemetry-emitter/src/test/java/org/apache/druid/emitter/opentelemetry/OpenTelemetryEmitterTest.java index 58ee485f9c9c..3ff719616034 100644 --- a/extensions-contrib/opentelemetry-emitter/src/test/java/org/apache/druid/emitter/opentelemetry/OpenTelemetryEmitterTest.java +++ b/extensions-contrib/opentelemetry-emitter/src/test/java/org/apache/druid/emitter/opentelemetry/OpenTelemetryEmitterTest.java @@ -43,7 +43,6 @@ import java.util.HashMap; import java.util.Map; - public class OpenTelemetryEmitterTest { private static class NoopExporter implements SpanExporter @@ -119,15 +118,8 @@ public String getFeed() public void testNoEmitNotQueryTimeMetric() { final ServiceMetricEvent notQueryTimeMetric = - new ServiceMetricEvent.Builder().build( - TIMESTAMP, - "query/cache/total/hitRate", - 0.54 - ) - .build( - "broker", - "brokerHost1" - ); + new ServiceMetricEvent.Builder().setMetric("query/cache/total/hitRate", 0.54) + .build("broker", "brokerHost1"); emitter.emit(notQueryTimeMetric); Assert.assertNull(noopExporter.spanDataCollection); @@ -148,15 +140,9 @@ public void testTraceparentId() final ServiceMetricEvent queryTimeMetric = new ServiceMetricEvent.Builder().setDimension("context", context) - .build( - createdTime, - "query/time", - metricValue - ) - .build( - serviceName, - "host" - ); + .setCreatedTime(createdTime) + .setMetric("query/time", metricValue) + .build(serviceName, "host"); emitter.emit(queryTimeMetric); @@ -179,15 +165,8 @@ public void testAttributes() final ServiceMetricEvent queryTimeMetricWithAttributes = new ServiceMetricEvent.Builder().setDimension("context", context) - .build( - TIMESTAMP, - "query/time", - 100 - ) - .build( - "druid/broker", - "host" - ); + .setMetric("query/time", 100) + .build("druid/broker", "host"); emitter.emit(queryTimeMetricWithAttributes); @@ -207,15 +186,8 @@ public void testFilterNullValue() final ServiceMetricEvent queryTimeMetric = new ServiceMetricEvent.Builder().setDimension("context", context) - .build( - TIMESTAMP, - "query/time", - 100 - ) - .build( - "druid/broker", - "host" - ); + .setMetric("query/time", 100) + .build("druid/broker", "host"); emitter.emit(queryTimeMetric); @@ -228,15 +200,8 @@ public void testOkStatus() { final ServiceMetricEvent queryTimeMetric = new ServiceMetricEvent.Builder().setDimension("success", "true") - .build( - TIMESTAMP, - "query/time", - 100 - ) - .build( - "druid/broker", - "host" - ); + .setMetric("query/time", 100) + .build("druid/broker", "host"); emitter.emit(queryTimeMetric); @@ -249,15 +214,8 @@ public void testErrorStatus() { final ServiceMetricEvent queryTimeMetric = new ServiceMetricEvent.Builder().setDimension("success", "false") - .build( - TIMESTAMP, - "query/time", - 100 - ) - .build( - "druid/broker", - "host" - ); + .setMetric("query/time", 100) + .build("druid/broker", "host"); emitter.emit(queryTimeMetric); diff --git a/extensions-contrib/opentsdb-emitter/src/test/java/org/apache/druid/emitter/opentsdb/EventConverterTest.java b/extensions-contrib/opentsdb-emitter/src/test/java/org/apache/druid/emitter/opentsdb/EventConverterTest.java index 1380417f16d9..d9434876c88a 100644 --- a/extensions-contrib/opentsdb-emitter/src/test/java/org/apache/druid/emitter/opentsdb/EventConverterTest.java +++ b/extensions-contrib/opentsdb-emitter/src/test/java/org/apache/druid/emitter/opentsdb/EventConverterTest.java @@ -60,7 +60,8 @@ public void testConvertWithNamespacePrefix() ServiceMetricEvent configuredEvent = new ServiceMetricEvent.Builder() .setDimension("dataSource", "foo:bar") .setDimension("type", "groupBy") - .build(dateTime, "query/time", 10) + .setCreatedTime(dateTime) + .setMetric("query/time", 10) .build("druid:broker", "127.0.0.1:8080"); Map expectedTags = new HashMap<>(); @@ -78,7 +79,8 @@ public void testConvertWithNamespacePrefix() ServiceMetricEvent notConfiguredEvent = new ServiceMetricEvent.Builder() .setDimension("dataSource", "data-source") .setDimension("type", "groupBy") - .build(dateTime, "foo/bar", 10) + .setCreatedTime(dateTime) + .setMetric("foo/bar", 10) .build("broker", "brokerHost1"); Assert.assertNull(converterWithNamespacePrefix.convert(notConfiguredEvent)); } @@ -90,7 +92,8 @@ public void testConvertWithNamespacePrefixContainingSpace() ServiceMetricEvent configuredEvent = new ServiceMetricEvent.Builder() .setDimension("dataSource", "foo:bar") .setDimension("type", "groupBy") - .build(dateTime, "query/time", 10) + .setCreatedTime(dateTime) + .setMetric("query/time", 10) .build("druid:broker", "127.0.0.1:8080"); Map expectedTags = new HashMap<>(); @@ -108,7 +111,8 @@ public void testConvertWithNamespacePrefixContainingSpace() ServiceMetricEvent notConfiguredEvent = new ServiceMetricEvent.Builder() .setDimension("dataSource", "data-source") .setDimension("type", "groupBy") - .build(dateTime, "foo/bar", 10) + .setCreatedTime(dateTime) + .setMetric("foo/bar", 10) .build("broker", "brokerHost1"); Assert.assertNull(converterWithNamespacePrefixContainingSpace.convert(notConfiguredEvent)); } @@ -120,7 +124,8 @@ public void testConvertWithoutNamespacePrefix() ServiceMetricEvent configuredEvent = new ServiceMetricEvent.Builder() .setDimension("dataSource", "foo:bar") .setDimension("type", "groupBy") - .build(dateTime, "query/time", 10) + .setCreatedTime(dateTime) + .setMetric("query/time", 10) .build("druid:broker", "127.0.0.1:8080"); Map expectedTags = new HashMap<>(); @@ -138,7 +143,8 @@ public void testConvertWithoutNamespacePrefix() ServiceMetricEvent notConfiguredEvent = new ServiceMetricEvent.Builder() .setDimension("dataSource", "data-source") .setDimension("type", "groupBy") - .build(dateTime, "foo/bar", 10) + .setCreatedTime(dateTime) + .setMetric("foo/bar", 10) .build("broker", "brokerHost1"); Assert.assertNull(converterWithoutNamespacePrefix.convert(notConfiguredEvent)); } diff --git a/extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/PrometheusEmitterTest.java b/extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/PrometheusEmitterTest.java index 5428f67395f5..ca1f63ba34c2 100644 --- a/extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/PrometheusEmitterTest.java +++ b/extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/PrometheusEmitterTest.java @@ -47,7 +47,7 @@ public void testEmitterWithServiceLabel() Emitter emitter = prometheusEmitterModule.getEmitter(config); ServiceMetricEvent build = ServiceMetricEvent.builder() .setDimension("server", "druid-data01.vpc.region") - .build("segment/loadQueue/count", 10) + .setMetric("segment/loadQueue/count", 10) .build(ImmutableMap.of("service", "historical", "host", "druid.test.cn")); Assert.assertEquals("historical", build.getService()); Assert.assertEquals("druid.test.cn", build.getHost()); @@ -68,7 +68,7 @@ public void testEmitterWithServiceAndHostLabel() Emitter emitter = prometheusEmitterModule.getEmitter(config); ServiceMetricEvent build = ServiceMetricEvent.builder() .setDimension("server", "druid-data01.vpc.region") - .build("segment/loadQueue/count", 10) + .setMetric("segment/loadQueue/count", 10) .build(ImmutableMap.of("service", "historical", "host", "druid.test.cn")); Assert.assertEquals("historical", build.getService()); Assert.assertEquals("druid.test.cn", build.getHost()); @@ -91,7 +91,7 @@ public void testEmitterWithExtraLabels() Emitter emitter = prometheusEmitterModule.getEmitter(config); ServiceMetricEvent build = ServiceMetricEvent.builder() .setDimension("server", "druid-data01.vpc.region") - .build("segment/loadQueue/count", 10) + .setMetric("segment/loadQueue/count", 10) .build(ImmutableMap.of("service", "historical", "host", "druid.test.cn")); emitter.emit(build); Double count = CollectorRegistry.defaultRegistry.getSampleValue( @@ -113,7 +113,7 @@ public void testEmitterWithServiceLabelAndExtraLabel() Emitter emitter = prometheusEmitterModule.getEmitter(config); ServiceMetricEvent build = ServiceMetricEvent.builder() .setDimension("server", "druid-data01.vpc.region") - .build("segment/loadQueue/count", 10) + .setMetric("segment/loadQueue/count", 10) .build(ImmutableMap.of("service", "historical", "host", "druid.test.cn")); emitter.emit(build); Double count = CollectorRegistry.defaultRegistry.getSampleValue( @@ -134,7 +134,7 @@ public void testEmitterWithEmptyExtraLabels() Emitter emitter = prometheusEmitterModule.getEmitter(config); ServiceMetricEvent build = ServiceMetricEvent.builder() .setDimension("server", "druid-data01.vpc.region") - .build("segment/loadQueue/count", 10) + .setMetric("segment/loadQueue/count", 10) .build(ImmutableMap.of("service", "historical", "host", "druid.test.cn")); emitter.emit(build); Double count = CollectorRegistry.defaultRegistry.getSampleValue( @@ -157,7 +157,7 @@ public void testEmitterWithMultipleExtraLabels() Emitter emitter = prometheusEmitterModule.getEmitter(config); ServiceMetricEvent build = ServiceMetricEvent.builder() .setDimension("server", "druid-data01.vpc.region") - .build("segment/loadQueue/count", 10) + .setMetric("segment/loadQueue/count", 10) .build(ImmutableMap.of("service", "historical", "host", "druid.test.cn")); emitter.emit(build); Double count = CollectorRegistry.defaultRegistry.getSampleValue( @@ -180,7 +180,7 @@ public void testEmitterWithLabelCollision() Emitter emitter = prometheusEmitterModule.getEmitter(config); ServiceMetricEvent build = ServiceMetricEvent.builder() .setDimension("server", "druid-data01.vpc.region") - .build("segment/loadQueue/count", 10) + .setMetric("segment/loadQueue/count", 10) .build(ImmutableMap.of("service", "historical", "host", "druid.test.cn")); emitter.emit(build); Double count = CollectorRegistry.defaultRegistry.getSampleValue( @@ -208,7 +208,7 @@ public void testEmitterMetric() ServiceMetricEvent build = ServiceMetricEvent.builder() .setDimension("dataSource", "test") .setDimension("taskType", "index_parallel") - .build("task/run/time", 500) + .setMetric("task/run/time", 500) .build(ImmutableMap.of("service", "overlord", "host", "druid.test.cn")); emitter.emit(build); double assertEpsilon = 0.0001; @@ -247,7 +247,7 @@ public void testEmitterPush() throws IOException emitter.setPushGateway(mockPushGateway); ServiceMetricEvent build = ServiceMetricEvent.builder() .setDimension("task", "index_parallel") - .build("task/run/time", 500) + .setMetric("task/run/time", 500) .build(ImmutableMap.of("service", "peon", "host", "druid.test.cn")); emitter.emit(build); emitter.flush(); diff --git a/extensions-contrib/redis-cache/src/main/java/org/apache/druid/client/cache/AbstractRedisCache.java b/extensions-contrib/redis-cache/src/main/java/org/apache/druid/client/cache/AbstractRedisCache.java index da11b76fbd76..ce2e29ce6dc1 100644 --- a/extensions-contrib/redis-cache/src/main/java/org/apache/druid/client/cache/AbstractRedisCache.java +++ b/extensions-contrib/redis-cache/src/main/java/org/apache/druid/client/cache/AbstractRedisCache.java @@ -149,8 +149,8 @@ public void doMonitor(ServiceEmitter emitter) final long priorCount = priorRequestCount.get(); final long totalCount = totalRequestCount.get(); final ServiceMetricEvent.Builder builder = ServiceMetricEvent.builder(); - emitter.emit(builder.build("query/cache/redis/total/requests", totalCount)); - emitter.emit(builder.build("query/cache/redis/delta/requests", totalCount - priorCount)); + emitter.emit(builder.setMetric("query/cache/redis/total/requests", totalCount)); + emitter.emit(builder.setMetric("query/cache/redis/delta/requests", totalCount - priorCount)); if (!priorRequestCount.compareAndSet(priorCount, totalCount)) { log.error("Prior value changed while I was reporting! updating anyways"); priorRequestCount.set(totalCount); diff --git a/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/DimensionConverterTest.java b/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/DimensionConverterTest.java index c6bb65281bb0..433e0bf2c298 100644 --- a/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/DimensionConverterTest.java +++ b/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/DimensionConverterTest.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.junit.Assert; import org.junit.Test; @@ -43,7 +42,7 @@ public void testConvert() .setDimension("remoteAddress", "194.0.90.2") .setDimension("id", "ID") .setDimension("context", "{context}") - .build(DateTimes.nowUtc(), "query/time", 10) + .setMetric("query/time", 10) .build("broker", "brokerHost1"); ImmutableMap.Builder actual = new ImmutableMap.Builder<>(); diff --git a/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/StatsDEmitterTest.java b/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/StatsDEmitterTest.java index 0cd754a1b2b8..79d95e83f0e2 100644 --- a/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/StatsDEmitterTest.java +++ b/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/StatsDEmitterTest.java @@ -24,7 +24,6 @@ import com.google.common.collect.ImmutableMap; import com.timgroup.statsd.Event; import com.timgroup.statsd.StatsDClient; -import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.emitter.service.AlertBuilder; import org.apache.druid.java.util.emitter.service.AlertEvent; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; @@ -51,7 +50,7 @@ public void testConvertRange() client.gauge("broker.query.cache.total.hitRate", 54); emitter.emit(new ServiceMetricEvent.Builder() .setDimension("dataSource", "data-source") - .build(DateTimes.nowUtc(), "query/cache/total/hitRate", 0.54) + .setMetric("query/cache/total/hitRate", 0.54) .build("broker", "brokerHost1") ); } @@ -68,7 +67,7 @@ public void testConvertRangeWithDogstatsd() client.gauge("broker.query.cache.total.hitRate", 0.54); emitter.emit(new ServiceMetricEvent.Builder() .setDimension("dataSource", "data-source") - .build(DateTimes.nowUtc(), "query/cache/total/hitRate", 0.54) + .setMetric("query/cache/total/hitRate", 0.54) .build("broker", "brokerHost1") ); } @@ -94,7 +93,7 @@ public void testNoConvertRange() .setDimension("remoteAddress", "194.0.90.2") .setDimension("id", "ID") .setDimension("context", "{context}") - .build(DateTimes.nowUtc(), "query/time", 10) + .setMetric("query/time", 10) .build("broker", "brokerHost1") ); } @@ -120,7 +119,7 @@ public void testConfigOptions() .setDimension("remoteAddress", "194.0.90.2") .setDimension("id", "ID") .setDimension("context", "{context}") - .build(DateTimes.nowUtc(), "query/time", 10) + .setMetric("query/time", 10) .build("broker", "brokerHost1") ); } @@ -148,7 +147,7 @@ public void testDogstatsdEnabled() .setDimension("remoteAddress", "194.0.90.2") .setDimension("id", "ID") .setDimension("context", "{context}") - .build(DateTimes.nowUtc(), "query/time", 10) + .setMetric("query/time", 10) .build("broker", "brokerHost1") ); } @@ -165,7 +164,7 @@ public void testBlankHolderOptions() client.count("brokerHost1.broker.jvm.gc.count.G1-GC", 1); emitter.emit(new ServiceMetricEvent.Builder() .setDimension("gcName", "G1 GC") - .build(DateTimes.nowUtc(), "jvm/gc/count", 1) + .setMetric("jvm/gc/count", 1) .build("broker", "brokerHost1") ); } @@ -185,7 +184,7 @@ public void testServiceAsTagOption() emitter.emit(new ServiceMetricEvent.Builder() .setDimension("dataSource", "data-source") .setDimension("type", "groupBy") - .build(DateTimes.nowUtc(), "query/time", 10) + .setMetric("query/time", 10) .build("druid/broker", "brokerHost1") ); } diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/KafkaConsumerMonitor.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/KafkaConsumerMonitor.java index dd10335045f5..6a8c6c149051 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/KafkaConsumerMonitor.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/KafkaConsumerMonitor.java @@ -71,7 +71,7 @@ public boolean doMonitor(final ServiceEmitter emitter) if (newValue != priorValue) { final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder().setDimension(TOPIC_TAG, topic); - emitter.emit(builder.build(METRICS.get(metricName.name()), newValue - priorValue)); + emitter.emit(builder.setMetric(METRICS.get(metricName.name()), newValue - priorValue)); } } } diff --git a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/CacheScheduler.java b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/CacheScheduler.java index cd21738f66c8..63471afe3d17 100644 --- a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/CacheScheduler.java +++ b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/CacheScheduler.java @@ -445,7 +445,7 @@ public void run() final long tasks = updatesStarted.get(); serviceEmitter.emit( ServiceMetricEvent.builder() - .build("namespace/deltaTasksStarted", tasks - priorUpdatesStarted) + .setMetric("namespace/deltaTasksStarted", tasks - priorUpdatesStarted) ); priorUpdatesStarted = tasks; } diff --git a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/OffHeapNamespaceExtractionCacheManager.java b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/OffHeapNamespaceExtractionCacheManager.java index 419182d3bc7c..7eb1637ebaf0 100644 --- a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/OffHeapNamespaceExtractionCacheManager.java +++ b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/OffHeapNamespaceExtractionCacheManager.java @@ -270,7 +270,7 @@ int cacheCount() @Override void monitor(ServiceEmitter serviceEmitter) { - serviceEmitter.emit(ServiceMetricEvent.builder().build("namespace/cache/count", cacheCount())); - serviceEmitter.emit(ServiceMetricEvent.builder().build("namespace/cache/diskSize", tmpFile.length())); + serviceEmitter.emit(ServiceMetricEvent.builder().setMetric("namespace/cache/count", cacheCount())); + serviceEmitter.emit(ServiceMetricEvent.builder().setMetric("namespace/cache/diskSize", tmpFile.length())); } } diff --git a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/OnHeapNamespaceExtractionCacheManager.java b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/OnHeapNamespaceExtractionCacheManager.java index 1a1a1306b3a5..8cb7a3a5439f 100644 --- a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/OnHeapNamespaceExtractionCacheManager.java +++ b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/OnHeapNamespaceExtractionCacheManager.java @@ -144,7 +144,7 @@ void monitor(ServiceEmitter serviceEmitter) } } - serviceEmitter.emit(ServiceMetricEvent.builder().build("namespace/cache/numEntries", numEntries)); - serviceEmitter.emit(ServiceMetricEvent.builder().build("namespace/cache/heapSizeInBytes", heapSizeInBytes)); + serviceEmitter.emit(ServiceMetricEvent.builder().setMetric("namespace/cache/numEntries", numEntries)); + serviceEmitter.emit(ServiceMetricEvent.builder().setMetric("namespace/cache/heapSizeInBytes", heapSizeInBytes)); } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/LocalTaskActionClient.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/LocalTaskActionClient.java index a07977c6a564..bd0be9f74114 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/LocalTaskActionClient.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/LocalTaskActionClient.java @@ -107,7 +107,7 @@ private void emitTimerMetric(final String metric, final TaskAction action, fi if (actionType != null) { metricBuilder.setDimension("taskActionType", actionType); } - toolbox.getEmitter().emit(metricBuilder.build(metric, Math.max(0, time))); + toolbox.getEmitter().emit(metricBuilder.setMetric(metric, Math.max(0, time))); } @Nullable diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentAllocationQueue.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentAllocationQueue.java index 4d51350c595c..6638c2f2578e 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentAllocationQueue.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentAllocationQueue.java @@ -529,7 +529,7 @@ private void emitTaskMetric(String metric, long value, SegmentAllocateRequest re final ServiceMetricEvent.Builder metricBuilder = ServiceMetricEvent.builder(); IndexTaskUtils.setTaskDimensions(metricBuilder, request.getTask()); metricBuilder.setDimension("taskActionType", SegmentAllocateAction.TYPE); - emitter.emit(metricBuilder.build(metric, value)); + emitter.emit(metricBuilder.setMetric(metric, value)); } private void emitBatchMetric(String metric, long value, AllocateRequestKey key) @@ -538,7 +538,7 @@ private void emitBatchMetric(String metric, long value, AllocateRequestKey key) metricBuilder.setDimension("taskActionType", SegmentAllocateAction.TYPE); metricBuilder.setDimension(DruidMetrics.DATASOURCE, key.dataSource); metricBuilder.setDimension(DruidMetrics.INTERVAL, key.preferredAllocationInterval.toString()); - emitter.emit(metricBuilder.build(metric, value)); + emitter.emit(metricBuilder.setMetric(metric, value)); } /** diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentMetadataUpdateAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentMetadataUpdateAction.java index 143805c82d4b..0ca2bbae4809 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentMetadataUpdateAction.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentMetadataUpdateAction.java @@ -95,7 +95,7 @@ public Void perform(Task task, TaskActionToolbox toolbox) for (DataSegment segment : segments) { metricBuilder.setDimension(DruidMetrics.INTERVAL, segment.getInterval().toString()); - toolbox.getEmitter().emit(metricBuilder.build("segment/moved/bytes", segment.getSize())); + toolbox.getEmitter().emit(metricBuilder.setMetric("segment/moved/bytes", segment.getSize())); } return null; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentNukeAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentNukeAction.java index 562e8681c761..53651cdb523f 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentNukeAction.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentNukeAction.java @@ -97,7 +97,7 @@ public Void perform(Task task, TaskActionToolbox toolbox) for (DataSegment segment : segments) { metricBuilder.setDimension(DruidMetrics.INTERVAL, segment.getInterval().toString()); - toolbox.getEmitter().emit(metricBuilder.build("segment/nuked/bytes", segment.getSize())); + toolbox.getEmitter().emit(metricBuilder.setMetric("segment/nuked/bytes", segment.getSize())); } return null; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertAction.java index cd11e0befdca..99dc94a9828c 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertAction.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertAction.java @@ -227,9 +227,9 @@ public SegmentPublishResult perform(Task task, TaskActionToolbox toolbox) IndexTaskUtils.setTaskDimensions(metricBuilder, task); if (retVal.isSuccess()) { - toolbox.getEmitter().emit(metricBuilder.build("segment/txn/success", 1)); + toolbox.getEmitter().emit(metricBuilder.setMetric("segment/txn/success", 1)); } else { - toolbox.getEmitter().emit(metricBuilder.build("segment/txn/failure", 1)); + toolbox.getEmitter().emit(metricBuilder.setMetric("segment/txn/failure", 1)); } // getSegments() should return an empty set if announceHistoricalSegments() failed @@ -239,7 +239,7 @@ public SegmentPublishResult perform(Task task, TaskActionToolbox toolbox) DruidMetrics.PARTITIONING_TYPE, segment.getShardSpec() == null ? null : segment.getShardSpec().getType() ); - toolbox.getEmitter().emit(metricBuilder.build("segment/added/bytes", segment.getSize())); + toolbox.getEmitter().emit(metricBuilder.setMetric("segment/added/bytes", segment.getSize())); // Emit the segment related metadata using the configured emitters. // There is a possibility that some segments' metadata event might get missed if the // server crashes after commiting segment but before emitting the event. diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/stats/TaskRealtimeMetricsMonitor.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/stats/TaskRealtimeMetricsMonitor.java index 42438ebd3623..393035b03efd 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/stats/TaskRealtimeMetricsMonitor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/stats/TaskRealtimeMetricsMonitor.java @@ -86,60 +86,60 @@ public boolean doMonitor(ServiceEmitter emitter) ); } builder.setDimensionIfNotNull(DruidMetrics.TAGS, metricTags); - emitter.emit(builder.build("ingest/events/thrownAway", thrownAway)); + emitter.emit(builder.setMetric("ingest/events/thrownAway", thrownAway)); final long unparseable = rowIngestionMetersTotals.getUnparseable() - previousRowIngestionMetersTotals.getUnparseable(); if (unparseable > 0) { log.error("[%,d] unparseable events discarded. Turn on debug logging to see exception stack trace.", unparseable); } - emitter.emit(builder.build("ingest/events/unparseable", unparseable)); + emitter.emit(builder.setMetric("ingest/events/unparseable", unparseable)); final long processedWithError = rowIngestionMetersTotals.getProcessedWithError() - previousRowIngestionMetersTotals.getProcessedWithError(); if (processedWithError > 0) { log.error("[%,d] events processed with errors! Set logParseExceptions to true in the ingestion spec to log these errors.", processedWithError); } - emitter.emit(builder.build("ingest/events/processedWithError", processedWithError)); + emitter.emit(builder.setMetric("ingest/events/processedWithError", processedWithError)); - emitter.emit(builder.build("ingest/events/processed", rowIngestionMetersTotals.getProcessed() - previousRowIngestionMetersTotals.getProcessed())); + emitter.emit(builder.setMetric("ingest/events/processed", rowIngestionMetersTotals.getProcessed() - previousRowIngestionMetersTotals.getProcessed())); final long dedup = metrics.dedup() - previousFireDepartmentMetrics.dedup(); if (dedup > 0) { log.warn("[%,d] duplicate events!", dedup); } - emitter.emit(builder.build("ingest/events/duplicate", dedup)); + emitter.emit(builder.setMetric("ingest/events/duplicate", dedup)); emitter.emit( - builder.build( + builder.setMetric( "ingest/input/bytes", rowIngestionMetersTotals.getProcessedBytes() - previousRowIngestionMetersTotals.getProcessedBytes() ) ); - emitter.emit(builder.build("ingest/rows/output", metrics.rowOutput() - previousFireDepartmentMetrics.rowOutput())); - emitter.emit(builder.build("ingest/persists/count", metrics.numPersists() - previousFireDepartmentMetrics.numPersists())); - emitter.emit(builder.build("ingest/persists/time", metrics.persistTimeMillis() - previousFireDepartmentMetrics.persistTimeMillis())); - emitter.emit(builder.build("ingest/persists/cpu", metrics.persistCpuTime() - previousFireDepartmentMetrics.persistCpuTime())); + emitter.emit(builder.setMetric("ingest/rows/output", metrics.rowOutput() - previousFireDepartmentMetrics.rowOutput())); + emitter.emit(builder.setMetric("ingest/persists/count", metrics.numPersists() - previousFireDepartmentMetrics.numPersists())); + emitter.emit(builder.setMetric("ingest/persists/time", metrics.persistTimeMillis() - previousFireDepartmentMetrics.persistTimeMillis())); + emitter.emit(builder.setMetric("ingest/persists/cpu", metrics.persistCpuTime() - previousFireDepartmentMetrics.persistCpuTime())); emitter.emit( - builder.build( + builder.setMetric( "ingest/persists/backPressure", metrics.persistBackPressureMillis() - previousFireDepartmentMetrics.persistBackPressureMillis() ) ); - emitter.emit(builder.build("ingest/persists/failed", metrics.failedPersists() - previousFireDepartmentMetrics.failedPersists())); - emitter.emit(builder.build("ingest/handoff/failed", metrics.failedHandoffs() - previousFireDepartmentMetrics.failedHandoffs())); - emitter.emit(builder.build("ingest/merge/time", metrics.mergeTimeMillis() - previousFireDepartmentMetrics.mergeTimeMillis())); - emitter.emit(builder.build("ingest/merge/cpu", metrics.mergeCpuTime() - previousFireDepartmentMetrics.mergeCpuTime())); - emitter.emit(builder.build("ingest/handoff/count", metrics.handOffCount() - previousFireDepartmentMetrics.handOffCount())); - emitter.emit(builder.build("ingest/sink/count", metrics.sinkCount())); + emitter.emit(builder.setMetric("ingest/persists/failed", metrics.failedPersists() - previousFireDepartmentMetrics.failedPersists())); + emitter.emit(builder.setMetric("ingest/handoff/failed", metrics.failedHandoffs() - previousFireDepartmentMetrics.failedHandoffs())); + emitter.emit(builder.setMetric("ingest/merge/time", metrics.mergeTimeMillis() - previousFireDepartmentMetrics.mergeTimeMillis())); + emitter.emit(builder.setMetric("ingest/merge/cpu", metrics.mergeCpuTime() - previousFireDepartmentMetrics.mergeCpuTime())); + emitter.emit(builder.setMetric("ingest/handoff/count", metrics.handOffCount() - previousFireDepartmentMetrics.handOffCount())); + emitter.emit(builder.setMetric("ingest/sink/count", metrics.sinkCount())); long messageGap = metrics.messageGap(); if (messageGap >= 0) { - emitter.emit(builder.build("ingest/events/messageGap", messageGap)); + emitter.emit(builder.setMetric("ingest/events/messageGap", messageGap)); } long maxSegmentHandoffTime = metrics.maxSegmentHandoffTime(); if (maxSegmentHandoffTime >= 0) { - emitter.emit(builder.build("ingest/handoff/time", maxSegmentHandoffTime)); + emitter.emit(builder.setMetric("ingest/handoff/time", maxSegmentHandoffTime)); } previousRowIngestionMetersTotals = rowIngestionMetersTotals; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java index 11bcfb6de785..30b44ff91590 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java @@ -653,7 +653,7 @@ protected boolean waitForSegmentAvailability( .setDimension("groupId", getGroupId()) .setDimensionIfNotNull(DruidMetrics.TAGS, getContextValue(DruidMetrics.TAGS)) .setDimension("segmentAvailabilityConfirmed", segmentAvailabilityConfirmationCompleted) - .build("task/segmentAvailability/wait/time", segmentAvailabilityWaitTimeMs) + .setMetric("task/segmentAvailability/wait/time", segmentAvailabilityWaitTimeMs) ); } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractTask.java index 3ad7b452855c..680684000ff6 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractTask.java @@ -412,7 +412,7 @@ public void emitMetric( if (emitter == null || metric == null || value == null) { return; } - emitter.emit(getMetricBuilder().build(metric, value)); + emitter.emit(getMetricBuilder().setMetric(metric, value)); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java index c068aa1433fb..1a0e5c971fda 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java @@ -796,7 +796,7 @@ private static DataSchema createDataSchema( } finally { if (emitter != null) { - emitter.emit(metricBuilder.build("compact/segmentAnalyzer/fetchAndProcessMillis", clock.millis() - start)); + emitter.emit(metricBuilder.setMetric("compact/segmentAnalyzer/fetchAndProcessMillis", clock.millis() - start)); } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/RemoteTaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/RemoteTaskRunner.java index 87d1dab4eec6..a79c263ec40c 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/RemoteTaskRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/RemoteTaskRunner.java @@ -966,7 +966,7 @@ private boolean announceTask( final ServiceMetricEvent.Builder metricBuilder = new ServiceMetricEvent.Builder(); IndexTaskUtils.setTaskDimensions(metricBuilder, task); - emitter.emit(metricBuilder.build( + emitter.emit(metricBuilder.setMetric( "task/pending/time", new Duration(workItem.getQueueInsertionTime(), DateTimes.nowUtc()).getMillis()) ); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunner.java index fce76d1d1af9..26358deea3e0 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunner.java @@ -245,8 +245,8 @@ public void stop() .setDimension("graceful", "true") // for backward compatibility .setDimension("error", String.valueOf(error)); - emitter.emit(metricBuilder.build("task/interrupt/count", 1L)); - emitter.emit(metricBuilder.build("task/interrupt/elapsed", elapsed)); + emitter.emit(metricBuilder.setMetric("task/interrupt/count", 1L)); + emitter.emit(metricBuilder.setMetric("task/interrupt/elapsed", elapsed)); } // Ok, now interrupt everything. diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskQueue.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskQueue.java index dae78143de74..ae0708344db4 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskQueue.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskQueue.java @@ -761,7 +761,7 @@ private void handleStatus(final TaskStatus status) // Emit event and log, if the task is done if (status.isComplete()) { IndexTaskUtils.setTaskStatusDimensions(metricBuilder, status); - emitter.emit(metricBuilder.build("task/run/time", status.getDuration())); + emitter.emit(metricBuilder.setMetric("task/run/time", status.getDuration())); log.info( "Completed task[%s] with status[%s] in [%d]ms.", diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunner.java index bfa1c03dc0d5..4fa1d21d3d5a 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunner.java @@ -1582,7 +1582,7 @@ public void taskAddedOrUpdated(final TaskAnnouncement announcement, final Worker final ServiceMetricEvent.Builder metricBuilder = new ServiceMetricEvent.Builder(); IndexTaskUtils.setTaskDimensions(metricBuilder, taskItem.getTask()); - emitter.emit(metricBuilder.build( + emitter.emit(metricBuilder.setMetric( "task/pending/time", new Duration(taskItem.getCreatedTime(), DateTimes.nowUtc()).getMillis()) ); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index 72380436fadb..f79785b82370 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -4317,7 +4317,7 @@ protected void emitNoticeProcessTime(String noticeType, long timeInMillis) .setDimension("noticeType", noticeType) .setDimension("dataSource", dataSource) .setDimensionIfNotNull(DruidMetrics.TAGS, spec.getContextValue(DruidMetrics.TAGS)) - .build("ingest/notices/time", timeInMillis) + .setMetric("ingest/notices/time", timeInMillis) ); } catch (Exception e) { @@ -4336,7 +4336,7 @@ protected void emitNoticesQueueSize() ServiceMetricEvent.builder() .setDimension("dataSource", dataSource) .setDimensionIfNotNull(DruidMetrics.TAGS, spec.getContextValue(DruidMetrics.TAGS)) - .build("ingest/notices/queueSize", getNoticesQueueSize()) + .setMetric("ingest/notices/queueSize", getNoticesQueueSize()) ); } catch (Exception e) { @@ -4394,7 +4394,7 @@ protected void emitLag() .setDimension(DruidMetrics.STREAM, getIoConfig().getStream()) .setDimension(DruidMetrics.PARTITION, entry.getKey()) .setDimensionIfNotNull(DruidMetrics.TAGS, metricTags) - .build( + .setMetric( StringUtils.format("ingest/%s/partitionLag%s", type, suffix), entry.getValue() ) @@ -4405,21 +4405,21 @@ protected void emitLag() .setDimension(DruidMetrics.DATASOURCE, dataSource) .setDimension(DruidMetrics.STREAM, getIoConfig().getStream()) .setDimensionIfNotNull(DruidMetrics.TAGS, metricTags) - .build(StringUtils.format("ingest/%s/lag%s", type, suffix), lagStats.getTotalLag()) + .setMetric(StringUtils.format("ingest/%s/lag%s", type, suffix), lagStats.getTotalLag()) ); emitter.emit( ServiceMetricEvent.builder() .setDimension(DruidMetrics.DATASOURCE, dataSource) .setDimension(DruidMetrics.STREAM, getIoConfig().getStream()) .setDimensionIfNotNull(DruidMetrics.TAGS, metricTags) - .build(StringUtils.format("ingest/%s/maxLag%s", type, suffix), lagStats.getMaxLag()) + .setMetric(StringUtils.format("ingest/%s/maxLag%s", type, suffix), lagStats.getMaxLag()) ); emitter.emit( ServiceMetricEvent.builder() .setDimension(DruidMetrics.DATASOURCE, dataSource) .setDimension(DruidMetrics.STREAM, getIoConfig().getStream()) .setDimensionIfNotNull(DruidMetrics.TAGS, metricTags) - .build(StringUtils.format("ingest/%s/avgLag%s", type, suffix), lagStats.getAvgLag()) + .setMetric(StringUtils.format("ingest/%s/avgLag%s", type, suffix), lagStats.getAvgLag()) ); }; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/worker/shuffle/ShuffleMonitor.java b/indexing-service/src/main/java/org/apache/druid/indexing/worker/shuffle/ShuffleMonitor.java index 6157698d6271..e66041c2930f 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/worker/shuffle/ShuffleMonitor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/worker/shuffle/ShuffleMonitor.java @@ -58,8 +58,8 @@ public boolean doMonitor(ServiceEmitter emitter) final Builder metricBuilder = ServiceMetricEvent .builder() .setDimension(SUPERVISOR_TASK_ID_DIMENSION, supervisorTaskId); - emitter.emit(metricBuilder.build(SHUFFLE_BYTES_KEY, perDatasourceShuffleMetrics.getShuffleBytes())); - emitter.emit(metricBuilder.build(SHUFFLE_REQUESTS_KEY, perDatasourceShuffleMetrics.getShuffleRequests())); + emitter.emit(metricBuilder.setMetric(SHUFFLE_BYTES_KEY, perDatasourceShuffleMetrics.getShuffleBytes())); + emitter.emit(metricBuilder.setMetric(SHUFFLE_REQUESTS_KEY, perDatasourceShuffleMetrics.getShuffleRequests())); }); } return true; diff --git a/processing/src/main/java/org/apache/druid/java/util/emitter/service/ServiceEmitter.java b/processing/src/main/java/org/apache/druid/java/util/emitter/service/ServiceEmitter.java index ad007f33ec69..17b3ac59db67 100644 --- a/processing/src/main/java/org/apache/druid/java/util/emitter/service/ServiceEmitter.java +++ b/processing/src/main/java/org/apache/druid/java/util/emitter/service/ServiceEmitter.java @@ -67,7 +67,7 @@ public void emit(Event event) emitter.emit(event); } - public void emit(ServiceEventBuilder builder) + public void emit(ServiceEventBuilder builder) { emit(builder.build(serviceDimensions)); } diff --git a/processing/src/main/java/org/apache/druid/java/util/emitter/service/ServiceMetricEvent.java b/processing/src/main/java/org/apache/druid/java/util/emitter/service/ServiceMetricEvent.java index 1ba264b4e391..3bab9a3ad04d 100644 --- a/processing/src/main/java/org/apache/druid/java/util/emitter/service/ServiceMetricEvent.java +++ b/processing/src/main/java/org/apache/druid/java/util/emitter/service/ServiceMetricEvent.java @@ -20,7 +20,7 @@ package org.apache.druid.java.util.emitter.service; import com.fasterxml.jackson.annotation.JsonValue; -import com.google.common.base.Predicate; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import org.apache.druid.guice.annotations.PublicApi; @@ -119,23 +119,23 @@ public EventMap toMap() .putAll( Maps.filterEntries( userDims, - new Predicate>() - { - @Override - public boolean apply(Map.Entry input) - { - return input.getKey() != null; - } - } + input -> input.getKey() != null ) ) .build(); } - public static class Builder + /** + * Builder for a {@link ServiceMetricEvent}. This builder can be used for + * building only one event. + */ + public static class Builder extends ServiceEventBuilder { private final Map userDims = new TreeMap<>(); private String feed = "metrics"; + private String metric; + private Number value; + private DateTime createdTime; public Builder setFeed(String feed) { @@ -168,19 +168,7 @@ public Object getDimension(String dim) return userDims.get(dim); } - public ServiceEventBuilder build( - final String metric, - final Number value - ) - { - return build(null, metric, value); - } - - public ServiceEventBuilder build( - final DateTime createdTime, - final String metric, - final Number value - ) + public Builder setMetric(String metric, Number value) { if (Double.isNaN(value.doubleValue())) { throw new ISE("Value of NaN is not allowed!"); @@ -189,21 +177,31 @@ public ServiceEventBuilder build( throw new ISE("Value of Infinite is not allowed!"); } - return new ServiceEventBuilder() - { - @Override - public ServiceMetricEvent build(ImmutableMap serviceDimensions) - { - return new ServiceMetricEvent( - createdTime, - serviceDimensions, - userDims, - feed, - metric, - value - ); - } - }; + this.metric = metric; + this.value = value; + return this; + } + + public Builder setCreatedTime(DateTime createdTime) + { + this.createdTime = createdTime; + return this; + } + + @Override + public ServiceMetricEvent build(ImmutableMap serviceDimensions) + { + Preconditions.checkNotNull(metric, "Metric is not set"); + Preconditions.checkNotNull(value, "Value is not set"); + + return new ServiceMetricEvent( + createdTime, + serviceDimensions, + userDims, + feed, + metric, + value + ); } } } diff --git a/processing/src/main/java/org/apache/druid/java/util/metrics/CgroupCpuMonitor.java b/processing/src/main/java/org/apache/druid/java/util/metrics/CgroupCpuMonitor.java index ac4d5450f028..af2408a156ed 100644 --- a/processing/src/main/java/org/apache/druid/java/util/metrics/CgroupCpuMonitor.java +++ b/processing/src/main/java/org/apache/druid/java/util/metrics/CgroupCpuMonitor.java @@ -62,8 +62,8 @@ public boolean doMonitor(ServiceEmitter emitter) final ServiceMetricEvent.Builder builder = builder(); MonitorUtils.addDimensionsToBuilder(builder, dimensions); - emitter.emit(builder.build("cgroup/cpu/shares", cpuSnapshot.getShares())); - emitter.emit(builder.build( + emitter.emit(builder.setMetric("cgroup/cpu/shares", cpuSnapshot.getShares())); + emitter.emit(builder.setMetric( "cgroup/cpu/cores_quota", computeProcessorQuota(cpuSnapshot.getQuotaUs(), cpuSnapshot.getPeriodUs()) )); diff --git a/processing/src/main/java/org/apache/druid/java/util/metrics/CgroupCpuSetMonitor.java b/processing/src/main/java/org/apache/druid/java/util/metrics/CgroupCpuSetMonitor.java index 43a403ad5af4..0534e00259f3 100644 --- a/processing/src/main/java/org/apache/druid/java/util/metrics/CgroupCpuSetMonitor.java +++ b/processing/src/main/java/org/apache/druid/java/util/metrics/CgroupCpuSetMonitor.java @@ -62,19 +62,19 @@ public boolean doMonitor(ServiceEmitter emitter) final ServiceMetricEvent.Builder builder = builder(); MonitorUtils.addDimensionsToBuilder(builder, dimensions); - emitter.emit(builder.build( + emitter.emit(builder.setMetric( "cgroup/cpuset/cpu_count", cpusetSnapshot.getCpuSetCpus().length )); - emitter.emit(builder.build( + emitter.emit(builder.setMetric( "cgroup/cpuset/effective_cpu_count", cpusetSnapshot.getEffectiveCpuSetCpus().length )); - emitter.emit(builder.build( + emitter.emit(builder.setMetric( "cgroup/cpuset/mems_count", cpusetSnapshot.getCpuSetMems().length )); - emitter.emit(builder.build( + emitter.emit(builder.setMetric( "cgroup/cpuset/effective_mems_count", cpusetSnapshot.getEffectiveCpuSetMems().length )); diff --git a/processing/src/main/java/org/apache/druid/java/util/metrics/CgroupMemoryMonitor.java b/processing/src/main/java/org/apache/druid/java/util/metrics/CgroupMemoryMonitor.java index 7e7c9c52b4be..d282cf5cddd0 100644 --- a/processing/src/main/java/org/apache/druid/java/util/metrics/CgroupMemoryMonitor.java +++ b/processing/src/main/java/org/apache/druid/java/util/metrics/CgroupMemoryMonitor.java @@ -65,12 +65,12 @@ public boolean doMonitor(ServiceEmitter emitter) MonitorUtils.addDimensionsToBuilder(builder, dimensions); // See https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt // There are inconsistent units for these. Most are bytes. - emitter.emit(builder.build(StringUtils.format("cgroup/memory/%s", key), value)); + emitter.emit(builder.setMetric(StringUtils.format("cgroup/memory/%s", key), value)); }); stat.getNumaMemoryStats().forEach((key, value) -> { final ServiceMetricEvent.Builder builder = builder().setDimension("numaZone", Long.toString(key)); MonitorUtils.addDimensionsToBuilder(builder, dimensions); - value.forEach((k, v) -> emitter.emit(builder.build(StringUtils.format("cgroup/memory_numa/%s/pages", k), v))); + value.forEach((k, v) -> emitter.emit(builder.setMetric(StringUtils.format("cgroup/memory_numa/%s/pages", k), v))); }); return true; } diff --git a/processing/src/main/java/org/apache/druid/java/util/metrics/CpuAcctDeltaMonitor.java b/processing/src/main/java/org/apache/druid/java/util/metrics/CpuAcctDeltaMonitor.java index f9a30b1bffd5..cbaf39729f27 100644 --- a/processing/src/main/java/org/apache/druid/java/util/metrics/CpuAcctDeltaMonitor.java +++ b/processing/src/main/java/org/apache/druid/java/util/metrics/CpuAcctDeltaMonitor.java @@ -102,20 +102,18 @@ public boolean doMonitor(ServiceEmitter emitter) .setDimension("cpuTime", "sys"); MonitorUtils.addDimensionsToBuilder(builderUsr, dimensions); MonitorUtils.addDimensionsToBuilder(builderSys, dimensions); - emitter.emit(builderUsr.build( - dateTime, + emitter.emit(builderUsr.setCreatedTime(dateTime).setMetric( "cgroup/cpu_time_delta_ns", snapshot.usrTime(i) - priorSnapshotHolder.metric.usrTime(i) )); - emitter.emit(builderSys.build( - dateTime, + emitter.emit(builderSys.setCreatedTime(dateTime).setMetric( "cgroup/cpu_time_delta_ns", snapshot.sysTime(i) - priorSnapshotHolder.metric.sysTime(i) )); } if (snapshot.cpuCount() > 0) { // Don't bother emitting metrics if there aren't actually any cpus (usually from error) - emitter.emit(builder().build(dateTime, "cgroup/cpu_time_delta_ns_elapsed", elapsedNs)); + emitter.emit(builder().setCreatedTime(dateTime).setMetric("cgroup/cpu_time_delta_ns_elapsed", elapsedNs)); } return true; } diff --git a/processing/src/main/java/org/apache/druid/java/util/metrics/HttpPostEmitterMonitor.java b/processing/src/main/java/org/apache/druid/java/util/metrics/HttpPostEmitterMonitor.java index e925fab5a1ad..cd37908034eb 100644 --- a/processing/src/main/java/org/apache/druid/java/util/metrics/HttpPostEmitterMonitor.java +++ b/processing/src/main/java/org/apache/druid/java/util/metrics/HttpPostEmitterMonitor.java @@ -59,12 +59,12 @@ public boolean doMonitor(ServiceEmitter emitter) emitTimeCounterMetrics(emitter, httpPostEmitter.getSuccessfulSendingTimeCounter(), "emitter/successfulSending/"); emitTimeCounterMetrics(emitter, httpPostEmitter.getFailedSendingTimeCounter(), "emitter/failedSending/"); - emitter.emit(builder.build("emitter/events/emitQueue", httpPostEmitter.getEventsToEmit())); - emitter.emit(builder.build("emitter/events/large/emitQueue", httpPostEmitter.getLargeEventsToEmit())); + emitter.emit(builder.setMetric("emitter/events/emitQueue", httpPostEmitter.getEventsToEmit())); + emitter.emit(builder.setMetric("emitter/events/large/emitQueue", httpPostEmitter.getLargeEventsToEmit())); - emitter.emit(builder.build("emitter/buffers/emitQueue", httpPostEmitter.getBuffersToEmit())); + emitter.emit(builder.setMetric("emitter/buffers/emitQueue", httpPostEmitter.getBuffersToEmit())); - emitter.emit(builder.build("emitter/buffers/reuseQueue", httpPostEmitter.getBuffersToReuse())); + emitter.emit(builder.setMetric("emitter/buffers/reuseQueue", httpPostEmitter.getBuffersToReuse())); return true; } @@ -73,7 +73,7 @@ private void emitEmittedEvents(ServiceEmitter emitter) { long newTotalEmittedEvents = httpPostEmitter.getTotalEmittedEvents(); long emittedEventsDelta = newTotalEmittedEvents - lastTotalEmittedEvents; - emitter.emit(builder.build("emitter/events/emitted/delta", emittedEventsDelta)); + emitter.emit(builder.setMetric("emitter/events/emitted/delta", emittedEventsDelta)); lastTotalEmittedEvents = newTotalEmittedEvents; } @@ -81,7 +81,7 @@ private void emitDroppedBuffers(ServiceEmitter emitter) { int newTotalDroppedBuffers = httpPostEmitter.getTotalDroppedBuffers(); int droppedBuffersDelta = newTotalDroppedBuffers - lastTotalDroppedBuffers; - emitter.emit(builder.build("emitter/buffers/dropped/delta", droppedBuffersDelta)); + emitter.emit(builder.setMetric("emitter/buffers/dropped/delta", droppedBuffersDelta)); lastTotalDroppedBuffers = newTotalDroppedBuffers; } @@ -89,7 +89,7 @@ private void emitAllocatedBuffers(ServiceEmitter emitter) { int newTotalAllocatedBuffers = httpPostEmitter.getTotalAllocatedBuffers(); int allocatedBuffersDelta = newTotalAllocatedBuffers - lastTotalAllocatedBuffers; - emitter.emit(builder.build("emitter/buffers/allocated/delta", allocatedBuffersDelta)); + emitter.emit(builder.setMetric("emitter/buffers/allocated/delta", allocatedBuffersDelta)); lastTotalAllocatedBuffers = newTotalAllocatedBuffers; } @@ -97,7 +97,7 @@ private void emitFailedBuffers(ServiceEmitter emitter) { int newTotalFailedBuffers = httpPostEmitter.getTotalFailedBuffers(); int failedBuffersDelta = newTotalFailedBuffers - lastTotalFailedBuffers; - emitter.emit(builder.build("emitter/buffers/failed/delta", failedBuffersDelta)); + emitter.emit(builder.setMetric("emitter/buffers/failed/delta", failedBuffersDelta)); lastTotalFailedBuffers = newTotalFailedBuffers; } @@ -107,16 +107,16 @@ private void emitTimeCounterMetrics(ServiceEmitter emitter, ConcurrentTimeCounte int timeSum = ConcurrentTimeCounter.timeSum(timeSumAndCount); int count = ConcurrentTimeCounter.count(timeSumAndCount); if (count != 0) { - emitter.emit(builder.build(metricNameBase + "timeMsSum", timeSum)); - emitter.emit(builder.build(metricNameBase + "count", count)); + emitter.emit(builder.setMetric(metricNameBase + "timeMsSum", timeSum)); + emitter.emit(builder.setMetric(metricNameBase + "count", count)); } Integer maxTime = timeCounter.getAndResetMaxTime(); if (maxTime != null) { - emitter.emit(builder.build(metricNameBase + "maxTimeMs", maxTime)); + emitter.emit(builder.setMetric(metricNameBase + "maxTimeMs", maxTime)); } Integer minTime = timeCounter.getAndResetMinTime(); if (minTime != null) { - emitter.emit(builder.build(metricNameBase + "minTimeMs", minTime)); + emitter.emit(builder.setMetric(metricNameBase + "minTimeMs", minTime)); } } diff --git a/processing/src/main/java/org/apache/druid/java/util/metrics/JvmCpuMonitor.java b/processing/src/main/java/org/apache/druid/java/util/metrics/JvmCpuMonitor.java index eb3028c02e93..b587d26ee607 100644 --- a/processing/src/main/java/org/apache/druid/java/util/metrics/JvmCpuMonitor.java +++ b/processing/src/main/java/org/apache/druid/java/util/metrics/JvmCpuMonitor.java @@ -76,10 +76,10 @@ public boolean doMonitor(ServiceEmitter emitter) ); if (procDiff != null) { for (Map.Entry entry : procDiff.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } } - emitter.emit(builder.build("jvm/cpu/percent", procCpu.getPercent())); + emitter.emit(builder.setMetric("jvm/cpu/percent", procCpu.getPercent())); } catch (SigarException e) { log.error(e, "Failed to get ProcCpu"); diff --git a/processing/src/main/java/org/apache/druid/java/util/metrics/JvmMonitor.java b/processing/src/main/java/org/apache/druid/java/util/metrics/JvmMonitor.java index 524a8dc87f0b..39d5832db990 100644 --- a/processing/src/main/java/org/apache/druid/java/util/metrics/JvmMonitor.java +++ b/processing/src/main/java/org/apache/druid/java/util/metrics/JvmMonitor.java @@ -85,7 +85,7 @@ private void emitThreadAllocationMetrics(ServiceEmitter emitter) MonitorUtils.addDimensionsToBuilder(builder, dimensions); if (collector != null) { long delta = collector.calculateDelta(); - emitter.emit(builder.build("jvm/heapAlloc/bytes", delta)); + emitter.emit(builder.setMetric("jvm/heapAlloc/bytes", delta)); } } @@ -107,10 +107,10 @@ private void emitJvmMemMetrics(ServiceEmitter emitter) final ServiceMetricEvent.Builder builder = builder().setDimension("memKind", kind); MonitorUtils.addDimensionsToBuilder(builder, dimensions); - emitter.emit(builder.build("jvm/mem/max", usage.getMax())); - emitter.emit(builder.build("jvm/mem/committed", usage.getCommitted())); - emitter.emit(builder.build("jvm/mem/used", usage.getUsed())); - emitter.emit(builder.build("jvm/mem/init", usage.getInit())); + emitter.emit(builder.setMetric("jvm/mem/max", usage.getMax())); + emitter.emit(builder.setMetric("jvm/mem/committed", usage.getCommitted())); + emitter.emit(builder.setMetric("jvm/mem/used", usage.getUsed())); + emitter.emit(builder.setMetric("jvm/mem/init", usage.getInit())); } // jvm/pool @@ -122,10 +122,10 @@ private void emitJvmMemMetrics(ServiceEmitter emitter) .setDimension("poolName", pool.getName()); MonitorUtils.addDimensionsToBuilder(builder, dimensions); - emitter.emit(builder.build("jvm/pool/max", usage.getMax())); - emitter.emit(builder.build("jvm/pool/committed", usage.getCommitted())); - emitter.emit(builder.build("jvm/pool/used", usage.getUsed())); - emitter.emit(builder.build("jvm/pool/init", usage.getInit())); + emitter.emit(builder.setMetric("jvm/pool/max", usage.getMax())); + emitter.emit(builder.setMetric("jvm/pool/committed", usage.getCommitted())); + emitter.emit(builder.setMetric("jvm/pool/used", usage.getUsed())); + emitter.emit(builder.setMetric("jvm/pool/init", usage.getInit())); } } @@ -135,9 +135,9 @@ private void emitDirectMemMetrics(ServiceEmitter emitter) final ServiceMetricEvent.Builder builder = builder().setDimension("bufferpoolName", pool.getName()); MonitorUtils.addDimensionsToBuilder(builder, dimensions); - emitter.emit(builder.build("jvm/bufferpool/capacity", pool.getTotalCapacity())); - emitter.emit(builder.build("jvm/bufferpool/used", pool.getMemoryUsed())); - emitter.emit(builder.build("jvm/bufferpool/count", pool.getCount())); + emitter.emit(builder.setMetric("jvm/bufferpool/capacity", pool.getTotalCapacity())); + emitter.emit(builder.setMetric("jvm/bufferpool/used", pool.getMemoryUsed())); + emitter.emit(builder.setMetric("jvm/bufferpool/count", pool.getCount())); } } @@ -265,12 +265,12 @@ void emit(ServiceEmitter emitter, Map dimensions) MonitorUtils.addDimensionsToBuilder(builder, dimensionsCopy); long newInvocations = gcBean.getCollectionCount(); - emitter.emit(builder.build("jvm/gc/count", newInvocations - lastInvocations)); + emitter.emit(builder.setMetric("jvm/gc/count", newInvocations - lastInvocations)); lastInvocations = newInvocations; // getCollectionTime is in milliseconds; we report jvm/gc/cpu in nanoseconds. long newCpuMillis = gcBean.getCollectionTime(); - emitter.emit(builder.build("jvm/gc/cpu", (newCpuMillis - lastCpuMillis) * 1_000_000L)); + emitter.emit(builder.setMetric("jvm/gc/cpu", (newCpuMillis - lastCpuMillis) * 1_000_000L)); lastCpuMillis = newCpuMillis; } } @@ -311,10 +311,10 @@ void emit(ServiceEmitter emitter, Map dimensions) builder.setDimension("gcGenSpaceName", name); - emitter.emit(builder.build("jvm/gc/mem/max", memoryUsage.getMax())); - emitter.emit(builder.build("jvm/gc/mem/capacity", memoryUsage.getCommitted())); - emitter.emit(builder.build("jvm/gc/mem/used", memoryUsage.getUsed())); - emitter.emit(builder.build("jvm/gc/mem/init", memoryUsage.getInit())); + emitter.emit(builder.setMetric("jvm/gc/mem/max", memoryUsage.getMax())); + emitter.emit(builder.setMetric("jvm/gc/mem/capacity", memoryUsage.getCommitted())); + emitter.emit(builder.setMetric("jvm/gc/mem/used", memoryUsage.getUsed())); + emitter.emit(builder.setMetric("jvm/gc/mem/init", memoryUsage.getInit())); } } } diff --git a/processing/src/main/java/org/apache/druid/java/util/metrics/JvmThreadsMonitor.java b/processing/src/main/java/org/apache/druid/java/util/metrics/JvmThreadsMonitor.java index eabb3a434c17..d81fab69429e 100644 --- a/processing/src/main/java/org/apache/druid/java/util/metrics/JvmThreadsMonitor.java +++ b/processing/src/main/java/org/apache/druid/java/util/metrics/JvmThreadsMonitor.java @@ -62,12 +62,12 @@ public boolean doMonitor(ServiceEmitter emitter) long startedThreadsDiff = newStartedThreads - lastStartedThreads; - emitter.emit(builder.build("jvm/threads/started", startedThreadsDiff)); - emitter.emit(builder.build("jvm/threads/finished", lastLiveThreads + startedThreadsDiff - newLiveThreads)); - emitter.emit(builder.build("jvm/threads/live", newLiveThreads)); - emitter.emit(builder.build("jvm/threads/liveDaemon", threadBean.getDaemonThreadCount())); + emitter.emit(builder.setMetric("jvm/threads/started", startedThreadsDiff)); + emitter.emit(builder.setMetric("jvm/threads/finished", lastLiveThreads + startedThreadsDiff - newLiveThreads)); + emitter.emit(builder.setMetric("jvm/threads/live", newLiveThreads)); + emitter.emit(builder.setMetric("jvm/threads/liveDaemon", threadBean.getDaemonThreadCount())); - emitter.emit(builder.build("jvm/threads/livePeak", threadBean.getPeakThreadCount())); + emitter.emit(builder.setMetric("jvm/threads/livePeak", threadBean.getPeakThreadCount())); threadBean.resetPeakThreadCount(); lastStartedThreads = newStartedThreads; diff --git a/processing/src/main/java/org/apache/druid/java/util/metrics/OshiSysMonitor.java b/processing/src/main/java/org/apache/druid/java/util/metrics/OshiSysMonitor.java index 40d97b57c7ee..ecc1ff0d97e1 100644 --- a/processing/src/main/java/org/apache/druid/java/util/metrics/OshiSysMonitor.java +++ b/processing/src/main/java/org/apache/druid/java/util/metrics/OshiSysMonitor.java @@ -203,7 +203,7 @@ public void emit(ServiceEmitter emitter) final ServiceMetricEvent.Builder builder = builder(); MonitorUtils.addDimensionsToBuilder(builder, dimensions); for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } } } @@ -231,7 +231,7 @@ public void emit(ServiceEmitter emitter) final ServiceMetricEvent.Builder builder = builder(); MonitorUtils.addDimensionsToBuilder(builder, dimensions); for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } this.prevPageIn = currPageIn; @@ -258,7 +258,7 @@ public void emit(ServiceEmitter emitter) .setDimension("fsDirName", fs.getMount()); MonitorUtils.addDimensionsToBuilder(builder, dimensions); for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } } } @@ -292,7 +292,7 @@ public void emit(ServiceEmitter emitter) .setDimension("diskName", disk.getName()); MonitorUtils.addDimensionsToBuilder(builder, dimensions); for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } } } @@ -334,7 +334,7 @@ public void emit(ServiceEmitter emitter) .setDimension("netHwaddr", net.getMacaddr()); MonitorUtils.addDimensionsToBuilder(builder, dimensions); for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } } } @@ -387,7 +387,7 @@ public void emit(ServiceEmitter emitter) MonitorUtils.addDimensionsToBuilder(builder, dimensions); if (total != 0) { // prevent divide by 0 exception and don't emit such events - emitter.emit(builder.build("sys/cpu", entry.getValue() * 100 / total)); // [0,100] + emitter.emit(builder.setMetric("sys/cpu", entry.getValue() * 100 / total)); // [0,100] } } @@ -410,7 +410,7 @@ public void emit(ServiceEmitter emitter) "sys/uptime", uptime ); for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } CentralProcessor processor = hal.getProcessor(); double[] la = processor.getSystemLoadAverage(3); @@ -422,7 +422,7 @@ public void emit(ServiceEmitter emitter) "sys/la/15", la[2] ); for (Map.Entry entry : statsCpuLoadAverage.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } } } @@ -456,7 +456,7 @@ public void emit(ServiceEmitter emitter) ); if (stats != null) { for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } } } diff --git a/processing/src/main/java/org/apache/druid/java/util/metrics/SysMonitor.java b/processing/src/main/java/org/apache/druid/java/util/metrics/SysMonitor.java index c8ce4cfb90b7..ee3627986c9f 100644 --- a/processing/src/main/java/org/apache/druid/java/util/metrics/SysMonitor.java +++ b/processing/src/main/java/org/apache/druid/java/util/metrics/SysMonitor.java @@ -144,7 +144,7 @@ public void emit(ServiceEmitter emitter) final ServiceMetricEvent.Builder builder = builder(); MonitorUtils.addDimensionsToBuilder(builder, dimensions); for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } } } @@ -195,7 +195,7 @@ public void emit(ServiceEmitter emitter) final ServiceMetricEvent.Builder builder = builder(); MonitorUtils.addDimensionsToBuilder(builder, dimensions); for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } this.prevPageIn = currPageIn; @@ -235,7 +235,7 @@ public void emit(ServiceEmitter emitter) .setDimension("fsDirName", dir); // fsDirName because FsStats uses fsDirName MonitorUtils.addDimensionsToBuilder(builder, dimensions); for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } } } @@ -281,7 +281,7 @@ public void emit(ServiceEmitter emitter) .setDimension("fsOptions", fs.getOptions().split(",")); MonitorUtils.addDimensionsToBuilder(builder, dimensions); for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } } } else { @@ -342,7 +342,7 @@ public void emit(ServiceEmitter emitter) .setDimension("fsOptions", fs.getOptions().split(",")); MonitorUtils.addDimensionsToBuilder(builder, dimensions); for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } } } @@ -412,7 +412,7 @@ public void emit(ServiceEmitter emitter) .setDimension("netHwaddr", netconf.getHwaddr()); MonitorUtils.addDimensionsToBuilder(builder, dimensions); for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } } } @@ -464,7 +464,7 @@ public void emit(ServiceEmitter emitter) .setDimension("cpuName", name) .setDimension("cpuTime", entry.getKey()); MonitorUtils.addDimensionsToBuilder(builder, dimensions); - emitter.emit(builder.build("sys/cpu", entry.getValue() * 100 / total)); // [0,100] + emitter.emit(builder.setMetric("sys/cpu", entry.getValue() * 100 / total)); // [0,100] } } } @@ -501,7 +501,7 @@ public void emit(ServiceEmitter emitter) "sys/uptime", Double.valueOf(uptime.getUptime()).longValue() ); for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } } @@ -512,7 +512,7 @@ public void emit(ServiceEmitter emitter) "sys/la/15", la[2] ); for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } } } @@ -552,7 +552,7 @@ public void emit(ServiceEmitter emitter) ); if (stats != null) { for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } } } @@ -588,7 +588,7 @@ public void emit(ServiceEmitter emitter) .put("sys/tcp/state/bound", (long) netStat.getTcpBound()) .build(); for (Map.Entry entry : stats.entrySet()) { - emitter.emit(builder.build(entry.getKey(), entry.getValue())); + emitter.emit(builder.setMetric(entry.getKey(), entry.getValue())); } } } diff --git a/processing/src/main/java/org/apache/druid/query/DefaultQueryMetrics.java b/processing/src/main/java/org/apache/druid/query/DefaultQueryMetrics.java index 6cab0252e7f7..11eb0ccdd33b 100644 --- a/processing/src/main/java/org/apache/druid/query/DefaultQueryMetrics.java +++ b/processing/src/main/java/org/apache/druid/query/DefaultQueryMetrics.java @@ -385,7 +385,7 @@ public void emit(ServiceEmitter emitter) { checkModifiedFromOwnerThread(); for (Map.Entry metric : metrics.entrySet()) { - emitter.emit(builder.build(metric.getKey(), metric.getValue())); + emitter.emit(builder.setMetric(metric.getKey(), metric.getValue())); } metrics.clear(); } diff --git a/processing/src/main/java/org/apache/druid/query/MetricsEmittingQueryProcessingPool.java b/processing/src/main/java/org/apache/druid/query/MetricsEmittingQueryProcessingPool.java index 4047d7977f91..6c3ac7eb9f32 100644 --- a/processing/src/main/java/org/apache/druid/query/MetricsEmittingQueryProcessingPool.java +++ b/processing/src/main/java/org/apache/druid/query/MetricsEmittingQueryProcessingPool.java @@ -40,7 +40,7 @@ public MetricsEmittingQueryProcessingPool( public void emitMetrics(ServiceEmitter emitter, ServiceMetricEvent.Builder metricBuilder) { if (delegate() instanceof PrioritizedExecutorService) { - emitter.emit(metricBuilder.build("segment/scan/pending", ((PrioritizedExecutorService) delegate()).getQueueSize())); + emitter.emit(metricBuilder.setMetric("segment/scan/pending", ((PrioritizedExecutorService) delegate()).getQueueSize())); } } diff --git a/processing/src/test/java/org/apache/druid/java/util/emitter/core/HttpPostEmitterLoggerStressTest.java b/processing/src/test/java/org/apache/druid/java/util/emitter/core/HttpPostEmitterLoggerStressTest.java index 2a1515e9331f..ad4094679292 100644 --- a/processing/src/test/java/org/apache/druid/java/util/emitter/core/HttpPostEmitterLoggerStressTest.java +++ b/processing/src/test/java/org/apache/druid/java/util/emitter/core/HttpPostEmitterLoggerStressTest.java @@ -58,7 +58,7 @@ public void testBurstFollowedByQuietPeriod() throws InterruptedException, IOExce httpClient.setGoHandler(new GoHandler() { @Override - protected ListenableFuture go(Request request) throws X + protected ListenableFuture go(Request request) { return GoHandlers.immediateFuture(EmitterTest.okResponse()); } @@ -67,7 +67,7 @@ protected ListenableFuture go(Request request) t Event smallEvent = ServiceMetricEvent.builder() .setFeed("smallEvents") .setDimension("test", "hi") - .build("metric", 10) + .setMetric("metric", 10) .build("qwerty", "asdfgh"); for (int i = 0; i < 1000; i++) { diff --git a/processing/src/test/java/org/apache/druid/java/util/emitter/core/HttpPostEmitterStressTest.java b/processing/src/test/java/org/apache/druid/java/util/emitter/core/HttpPostEmitterStressTest.java index 26827735472c..5e6877f5943a 100644 --- a/processing/src/test/java/org/apache/druid/java/util/emitter/core/HttpPostEmitterStressTest.java +++ b/processing/src/test/java/org/apache/druid/java/util/emitter/core/HttpPostEmitterStressTest.java @@ -170,7 +170,7 @@ protected ListenableFuture go(Request request) t Event bigEvent = ServiceMetricEvent.builder() .setFeed("bigEvents") .setDimension("test", bigString) - .build("metric", 10) + .setMetric("metric", 10) .build("qwerty", "asdfgh"); for (int i = 0; i < 1000; i++) { @@ -210,13 +210,13 @@ protected ListenableFuture go(Request request) t Event smallEvent = ServiceMetricEvent.builder() .setFeed("smallEvents") .setDimension("test", "hi") - .build("metric", 10) + .setMetric("metric", 10) .build("qwerty", "asdfgh"); Event bigEvent = ServiceMetricEvent.builder() .setFeed("bigEvents") .setDimension("test", bigString) - .build("metric", 10) + .setMetric("metric", 10) .build("qwerty", "asdfgh"); final CountDownLatch threadsCompleted = new CountDownLatch(2); diff --git a/processing/src/test/java/org/apache/druid/java/util/emitter/service/ServiceMetricEventTest.java b/processing/src/test/java/org/apache/druid/java/util/emitter/service/ServiceMetricEventTest.java index 42f299f238dc..5fbf4dc9c9e1 100644 --- a/processing/src/test/java/org/apache/druid/java/util/emitter/service/ServiceMetricEventTest.java +++ b/processing/src/test/java/org/apache/druid/java/util/emitter/service/ServiceMetricEventTest.java @@ -48,7 +48,7 @@ public void testStupidTest() .setDimension("user8", "h") .setDimension("user9", "i") .setDimension("user10", "j") - .build("test-metric", 1234) + .setMetric("test-metric", 1234) .build("test", "localhost"); Assert.assertEquals( ImmutableMap.builder() @@ -84,7 +84,7 @@ public void testStupidTest() .setDimension("user8", "h") .setDimension("user9", "i") .setDimension("user10", "j") - .build("test-metric", 1234) + .setMetric("test-metric", 1234) .build("test", "localhost"); Assert.assertEquals( @@ -120,7 +120,7 @@ public void testStupidTest() .setDimension("user8", new String[]{"h"}) .setDimension("user9", new String[]{"i"}) .setDimension("user10", new String[]{"j"}) - .build("test-metric", 1234) + .setMetric("test-metric", 1234) .build("test", "localhost"); Assert.assertEquals( @@ -156,7 +156,7 @@ public void testStupidTest() .setDimension("user8", "h") .setDimension("user9", "i") .setDimension("user10", "j") - .build(null, "test-metric", 1234) + .setMetric("test-metric", 1234) .build("test", "localhost") .getCreatedTime() ); @@ -173,7 +173,7 @@ public void testStupidTest() .setDimension("user8", new String[]{"h"}) .setDimension("user9", new String[]{"i"}) .setDimension("user10", new String[]{"j"}) - .build("test-metric", 1234) + .setMetric("test-metric", 1234) .build("test", "localhost") .getCreatedTime() ); @@ -208,7 +208,8 @@ public void testStupidTest() .setDimension("user8", "h") .setDimension("user9", "i") .setDimension("user10", "j") - .build(DateTimes.utc(42), "test-metric", 1234) + .setCreatedTime(DateTimes.utc(42)) + .setMetric("test-metric", 1234) .build("test", "localhost") .toMap() ); @@ -243,7 +244,8 @@ public void testStupidTest() .setDimension("user8", new String[]{"h"}) .setDimension("user9", new String[]{"i"}) .setDimension("user10", new String[]{"j"}) - .build(DateTimes.utc(42), "test-metric", 1234) + .setCreatedTime(DateTimes.utc(42)) + .setMetric("test-metric", 1234) .build("test", "localhost") .toMap() ); @@ -262,7 +264,8 @@ public void testStupidTest() ServiceMetricEvent.builder() .setDimension("foo", "bar") .setDimension("baz", new String[]{"foo", "qux"}) - .build(DateTimes.utc(42), "test-metric", 1234) + .setCreatedTime(DateTimes.utc(42)) + .setMetric("test-metric", 1234) .build("test", "localhost") .toMap() ); @@ -271,26 +274,26 @@ public void testStupidTest() @Test(expected = IllegalStateException.class) public void testInfinite() { - ServiceMetricEvent.builder().build("foo", 1 / 0d); + ServiceMetricEvent.builder().setMetric("foo", 1 / 0d); } @Test(expected = IllegalStateException.class) public void testInfinite2() { - ServiceMetricEvent.builder().build("foo", 1 / 0f); + ServiceMetricEvent.builder().setMetric("foo", 1 / 0f); } @Test(expected = IllegalStateException.class) public void testNaN() { - ServiceMetricEvent.builder().build("foo", 0 / 0d); + ServiceMetricEvent.builder().setMetric("foo", 0 / 0d); } @Test(expected = IllegalStateException.class) public void testNaN2() { - ServiceMetricEvent.builder().build("foo", 0 / 0f); + ServiceMetricEvent.builder().setMetric("foo", 0 / 0f); } @Test @@ -299,7 +302,7 @@ public void testSetDimensionIfNotNullSetsNonNullDimension() Map userDimMap = ImmutableMap.of("k1", "v1", "k2", "v2"); ServiceMetricEvent target = ServiceMetricEvent.builder() .setDimensionIfNotNull("userDimMap", userDimMap) - .build("foo", 1) + .setMetric("foo", 1) .build("service", "host"); Assert.assertEquals(userDimMap, target.getUserDims().get("userDimMap")); } @@ -309,7 +312,7 @@ public void testSetDimensionIfNotNullShouldNotSetNullDimension() { ServiceMetricEvent target = ServiceMetricEvent.builder() .setDimensionIfNotNull("userDimMap", null) - .build("foo", 1) + .setMetric("foo", 1) .build("service", "host"); Assert.assertTrue(target.getUserDims().isEmpty()); Assert.assertNull(target.getUserDims().get("userDimMap")); diff --git a/server/src/main/java/org/apache/druid/client/BrokerServerView.java b/server/src/main/java/org/apache/druid/client/BrokerServerView.java index 9c35b0c90302..8990d2a8f163 100644 --- a/server/src/main/java/org/apache/druid/client/BrokerServerView.java +++ b/server/src/main/java/org/apache/druid/client/BrokerServerView.java @@ -184,7 +184,7 @@ public void start() throws InterruptedException awaitInitialization(); final long endMillis = System.currentTimeMillis(); log.info("BrokerServerView initialized in [%,d] ms.", endMillis - startMillis); - emitter.emit(ServiceMetricEvent.builder().build( + emitter.emit(ServiceMetricEvent.builder().setMetric( "serverview/init/time", endMillis - startMillis )); diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java index 56ad1a78872a..77bb90c4a5fd 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java @@ -124,7 +124,7 @@ public void start() throws InterruptedException initialized.await(); final long endMillis = System.currentTimeMillis(); log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); - emitter.emit(ServiceMetricEvent.builder().build( + emitter.emit(ServiceMetricEvent.builder().setMetric( "serverview/init/time", endMillis - startMillis )); diff --git a/server/src/main/java/org/apache/druid/client/HttpServerInventoryView.java b/server/src/main/java/org/apache/druid/client/HttpServerInventoryView.java index 378706809b9d..eeac3439e4d9 100644 --- a/server/src/main/java/org/apache/druid/client/HttpServerInventoryView.java +++ b/server/src/main/java/org/apache/druid/client/HttpServerInventoryView.java @@ -495,12 +495,12 @@ private void emitServerStatusMetrics() final boolean isSynced = serverHolder.syncer.isSyncedSuccessfully(); serviceEmitter.emit( - eventBuilder.build("serverview/sync/healthy", isSynced ? 1 : 0) + eventBuilder.setMetric("serverview/sync/healthy", isSynced ? 1 : 0) ); final long unstableTimeMillis = serverHolder.syncer.getUnstableTimeMillis(); if (unstableTimeMillis > 0) { serviceEmitter.emit( - eventBuilder.build("serverview/sync/unstableTime", unstableTimeMillis) + eventBuilder.setMetric("serverview/sync/unstableTime", unstableTimeMillis) ); } }); diff --git a/server/src/main/java/org/apache/druid/client/cache/CacheMonitor.java b/server/src/main/java/org/apache/druid/client/cache/CacheMonitor.java index 43bcfb1c0eb3..2c43d1ea7686 100644 --- a/server/src/main/java/org/apache/druid/client/cache/CacheMonitor.java +++ b/server/src/main/java/org/apache/druid/client/cache/CacheMonitor.java @@ -83,21 +83,21 @@ private void emitStats( { if (cache != null) { // Cache stats. - emitter.emit(builder.build(StringUtils.format("%s/numEntries", metricPrefix), cacheStats.getNumEntries())); - emitter.emit(builder.build(StringUtils.format("%s/sizeBytes", metricPrefix), cacheStats.getSizeInBytes())); - emitter.emit(builder.build(StringUtils.format("%s/hits", metricPrefix), cacheStats.getNumHits())); - emitter.emit(builder.build(StringUtils.format("%s/misses", metricPrefix), cacheStats.getNumMisses())); - emitter.emit(builder.build(StringUtils.format("%s/evictions", metricPrefix), cacheStats.getNumEvictions())); - emitter.emit(builder.build(StringUtils.format("%s/hitRate", metricPrefix), cacheStats.hitRate())); - emitter.emit(builder.build(StringUtils.format("%s/averageBytes", metricPrefix), cacheStats.averageBytes())); - emitter.emit(builder.build(StringUtils.format("%s/timeouts", metricPrefix), cacheStats.getNumTimeouts())); - emitter.emit(builder.build(StringUtils.format("%s/errors", metricPrefix), cacheStats.getNumErrors())); + emitter.emit(builder.setMetric(StringUtils.format("%s/numEntries", metricPrefix), cacheStats.getNumEntries())); + emitter.emit(builder.setMetric(StringUtils.format("%s/sizeBytes", metricPrefix), cacheStats.getSizeInBytes())); + emitter.emit(builder.setMetric(StringUtils.format("%s/hits", metricPrefix), cacheStats.getNumHits())); + emitter.emit(builder.setMetric(StringUtils.format("%s/misses", metricPrefix), cacheStats.getNumMisses())); + emitter.emit(builder.setMetric(StringUtils.format("%s/evictions", metricPrefix), cacheStats.getNumEvictions())); + emitter.emit(builder.setMetric(StringUtils.format("%s/hitRate", metricPrefix), cacheStats.hitRate())); + emitter.emit(builder.setMetric(StringUtils.format("%s/averageBytes", metricPrefix), cacheStats.averageBytes())); + emitter.emit(builder.setMetric(StringUtils.format("%s/timeouts", metricPrefix), cacheStats.getNumTimeouts())); + emitter.emit(builder.setMetric(StringUtils.format("%s/errors", metricPrefix), cacheStats.getNumErrors())); // Cache populator stats. - emitter.emit(builder.build(StringUtils.format("%s/put/ok", metricPrefix), cachePopulatorStats.getNumOk())); - emitter.emit(builder.build(StringUtils.format("%s/put/error", metricPrefix), cachePopulatorStats.getNumError())); + emitter.emit(builder.setMetric(StringUtils.format("%s/put/ok", metricPrefix), cachePopulatorStats.getNumOk())); + emitter.emit(builder.setMetric(StringUtils.format("%s/put/error", metricPrefix), cachePopulatorStats.getNumError())); emitter.emit( - builder.build(StringUtils.format("%s/put/oversized", metricPrefix), cachePopulatorStats.getNumOversized()) + builder.setMetric(StringUtils.format("%s/put/oversized", metricPrefix), cachePopulatorStats.getNumOversized()) ); } } diff --git a/server/src/main/java/org/apache/druid/client/cache/CaffeineCache.java b/server/src/main/java/org/apache/druid/client/cache/CaffeineCache.java index 362a387231a6..400ee677985a 100644 --- a/server/src/main/java/org/apache/druid/client/cache/CaffeineCache.java +++ b/server/src/main/java/org/apache/druid/client/cache/CaffeineCache.java @@ -156,12 +156,12 @@ public void doMonitor(ServiceEmitter emitter) final CacheStats deltaStats = newStats.minus(oldStats); final ServiceMetricEvent.Builder builder = ServiceMetricEvent.builder(); - emitter.emit(builder.build("query/cache/caffeine/delta/requests", deltaStats.requestCount())); - emitter.emit(builder.build("query/cache/caffeine/total/requests", newStats.requestCount())); - emitter.emit(builder.build("query/cache/caffeine/delta/loadTime", deltaStats.totalLoadTime())); - emitter.emit(builder.build("query/cache/caffeine/total/loadTime", newStats.totalLoadTime())); - emitter.emit(builder.build("query/cache/caffeine/delta/evictionBytes", deltaStats.evictionWeight())); - emitter.emit(builder.build("query/cache/caffeine/total/evictionBytes", newStats.evictionWeight())); + emitter.emit(builder.setMetric("query/cache/caffeine/delta/requests", deltaStats.requestCount())); + emitter.emit(builder.setMetric("query/cache/caffeine/total/requests", newStats.requestCount())); + emitter.emit(builder.setMetric("query/cache/caffeine/delta/loadTime", deltaStats.totalLoadTime())); + emitter.emit(builder.setMetric("query/cache/caffeine/total/loadTime", newStats.totalLoadTime())); + emitter.emit(builder.setMetric("query/cache/caffeine/delta/evictionBytes", deltaStats.evictionWeight())); + emitter.emit(builder.setMetric("query/cache/caffeine/total/evictionBytes", newStats.evictionWeight())); if (!priorStats.compareAndSet(oldStats, newStats)) { // ISE for stack trace log.warn( diff --git a/server/src/main/java/org/apache/druid/client/cache/MemcachedCache.java b/server/src/main/java/org/apache/druid/client/cache/MemcachedCache.java index c13ca5434420..08ba9f8706c1 100644 --- a/server/src/main/java/org/apache/druid/client/cache/MemcachedCache.java +++ b/server/src/main/java/org/apache/druid/client/cache/MemcachedCache.java @@ -115,12 +115,12 @@ public boolean doMonitor(ServiceEmitter emitter) for (Map.Entry entry : currentValues.entrySet()) { emitter.emit( builder.setDimension("memcached metric", entry.getKey()) - .build("query/cache/memcached/total", entry.getValue()) + .setMetric("query/cache/memcached/total", entry.getValue()) ); final Long prior = priorValues.get(entry.getKey()); if (prior != null) { emitter.emit( - builder.setDimension("memcached metric", entry.getKey()).build( + builder.setDimension("memcached metric", entry.getKey()).setMetric( "query/cache/memcached/delta", entry.getValue() - prior ) diff --git a/server/src/main/java/org/apache/druid/curator/DruidConnectionStateListener.java b/server/src/main/java/org/apache/druid/curator/DruidConnectionStateListener.java index b0ba274d5014..dce5c375629e 100644 --- a/server/src/main/java/org/apache/druid/curator/DruidConnectionStateListener.java +++ b/server/src/main/java/org/apache/druid/curator/DruidConnectionStateListener.java @@ -75,7 +75,7 @@ public void stateChanged(CuratorFramework curatorFramework, ConnectionState newS } if (disconnectDuration != NIL) { - emitter.emit(ServiceMetricEvent.builder().build(METRIC_RECONNECT_TIME, disconnectDuration)); + emitter.emit(ServiceMetricEvent.builder().setMetric(METRIC_RECONNECT_TIME, disconnectDuration)); } } else { synchronized (this) { @@ -97,7 +97,7 @@ public boolean isConnected() @Override public boolean doMonitor(ServiceEmitter emitter) { - emitter.emit(ServiceMetricEvent.builder().build(METRIC_IS_CONNECTED, isConnected() ? 1 : 0)); + emitter.emit(ServiceMetricEvent.builder().setMetric(METRIC_IS_CONNECTED, isConnected() ? 1 : 0)); return true; } } diff --git a/server/src/main/java/org/apache/druid/segment/realtime/RealtimeMetricsMonitor.java b/server/src/main/java/org/apache/druid/segment/realtime/RealtimeMetricsMonitor.java index 43058851a902..c923c9e7bbd5 100644 --- a/server/src/main/java/org/apache/druid/segment/realtime/RealtimeMetricsMonitor.java +++ b/server/src/main/java/org/apache/druid/segment/realtime/RealtimeMetricsMonitor.java @@ -81,7 +81,7 @@ public boolean doMonitor(ServiceEmitter emitter) thrownAway ); } - emitter.emit(builder.build("ingest/events/thrownAway", thrownAway)); + emitter.emit(builder.setMetric("ingest/events/thrownAway", thrownAway)); final long unparseable = metrics.unparseable() - previous.unparseable(); if (unparseable > 0) { log.error( @@ -89,39 +89,39 @@ public boolean doMonitor(ServiceEmitter emitter) unparseable ); } - emitter.emit(builder.build("ingest/events/unparseable", unparseable)); + emitter.emit(builder.setMetric("ingest/events/unparseable", unparseable)); final long dedup = metrics.dedup() - previous.dedup(); if (dedup > 0) { log.warn("[%,d] duplicate events!", dedup); } - emitter.emit(builder.build("ingest/events/duplicate", dedup)); + emitter.emit(builder.setMetric("ingest/events/duplicate", dedup)); - emitter.emit(builder.build("ingest/events/processed", metrics.processed() - previous.processed())); - emitter.emit(builder.build("ingest/rows/output", metrics.rowOutput() - previous.rowOutput())); - emitter.emit(builder.build("ingest/persists/count", metrics.numPersists() - previous.numPersists())); - emitter.emit(builder.build("ingest/persists/time", metrics.persistTimeMillis() - previous.persistTimeMillis())); - emitter.emit(builder.build("ingest/persists/cpu", metrics.persistCpuTime() - previous.persistCpuTime())); + emitter.emit(builder.setMetric("ingest/events/processed", metrics.processed() - previous.processed())); + emitter.emit(builder.setMetric("ingest/rows/output", metrics.rowOutput() - previous.rowOutput())); + emitter.emit(builder.setMetric("ingest/persists/count", metrics.numPersists() - previous.numPersists())); + emitter.emit(builder.setMetric("ingest/persists/time", metrics.persistTimeMillis() - previous.persistTimeMillis())); + emitter.emit(builder.setMetric("ingest/persists/cpu", metrics.persistCpuTime() - previous.persistCpuTime())); emitter.emit( - builder.build( + builder.setMetric( "ingest/persists/backPressure", metrics.persistBackPressureMillis() - previous.persistBackPressureMillis() ) ); - emitter.emit(builder.build("ingest/persists/failed", metrics.failedPersists() - previous.failedPersists())); - emitter.emit(builder.build("ingest/handoff/failed", metrics.failedHandoffs() - previous.failedHandoffs())); - emitter.emit(builder.build("ingest/merge/time", metrics.mergeTimeMillis() - previous.mergeTimeMillis())); - emitter.emit(builder.build("ingest/merge/cpu", metrics.mergeCpuTime() - previous.mergeCpuTime())); - emitter.emit(builder.build("ingest/handoff/count", metrics.handOffCount() - previous.handOffCount())); - emitter.emit(builder.build("ingest/sink/count", metrics.sinkCount())); + emitter.emit(builder.setMetric("ingest/persists/failed", metrics.failedPersists() - previous.failedPersists())); + emitter.emit(builder.setMetric("ingest/handoff/failed", metrics.failedHandoffs() - previous.failedHandoffs())); + emitter.emit(builder.setMetric("ingest/merge/time", metrics.mergeTimeMillis() - previous.mergeTimeMillis())); + emitter.emit(builder.setMetric("ingest/merge/cpu", metrics.mergeCpuTime() - previous.mergeCpuTime())); + emitter.emit(builder.setMetric("ingest/handoff/count", metrics.handOffCount() - previous.handOffCount())); + emitter.emit(builder.setMetric("ingest/sink/count", metrics.sinkCount())); long messageGap = metrics.messageGap(); if (messageGap >= 0) { - emitter.emit(builder.build("ingest/events/messageGap", messageGap)); + emitter.emit(builder.setMetric("ingest/events/messageGap", messageGap)); } long maxSegmentHandoffTime = metrics.maxSegmentHandoffTime(); if (maxSegmentHandoffTime >= 0) { - emitter.emit(builder.build("ingest/handoff/time", maxSegmentHandoffTime)); + emitter.emit(builder.setMetric("ingest/handoff/time", maxSegmentHandoffTime)); } previousValues.put(fireDepartment, metrics); diff --git a/server/src/main/java/org/apache/druid/server/QueryScheduler.java b/server/src/main/java/org/apache/druid/server/QueryScheduler.java index 4dba34a6bed1..489af47e869e 100644 --- a/server/src/main/java/org/apache/druid/server/QueryScheduler.java +++ b/server/src/main/java/org/apache/druid/server/QueryScheduler.java @@ -173,7 +173,7 @@ public Query prioritizeAndLaneQuery(QueryPlus queryPlus, Set= 0) { diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java index faa399ba737b..ccb9347ee14b 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java @@ -736,7 +736,7 @@ private void emitStat(CoordinatorStat stat, Map dimensionValu dimensionValues.forEach( (dim, dimValue) -> eventBuilder.setDimension(dim.reportedName(), dimValue) ); - emitter.emit(eventBuilder.build(stat.getMetricName(), value)); + emitter.emit(eventBuilder.setMetric(stat.getMetricName(), value)); } Duration getPeriod() diff --git a/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java b/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java index 129c3e1400e0..1e2bab200415 100644 --- a/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java +++ b/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java @@ -512,15 +512,15 @@ public boolean doMonitor(ServiceEmitter emitter) { final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder(); MonitorUtils.addDimensionsToBuilder(builder, dimensions); - emitter.emit(builder.build("jetty/numOpenConnections", ACTIVE_CONNECTIONS.get())); + emitter.emit(builder.setMetric("jetty/numOpenConnections", ACTIVE_CONNECTIONS.get())); if (jettyServerThreadPool != null) { - emitter.emit(builder.build("jetty/threadPool/total", jettyServerThreadPool.getThreads())); - emitter.emit(builder.build("jetty/threadPool/idle", jettyServerThreadPool.getIdleThreads())); - emitter.emit(builder.build("jetty/threadPool/isLowOnThreads", jettyServerThreadPool.isLowOnThreads() ? 1 : 0)); - emitter.emit(builder.build("jetty/threadPool/min", jettyServerThreadPool.getMinThreads())); - emitter.emit(builder.build("jetty/threadPool/max", jettyServerThreadPool.getMaxThreads())); - emitter.emit(builder.build("jetty/threadPool/queueSize", jettyServerThreadPool.getQueueSize())); - emitter.emit(builder.build("jetty/threadPool/busy", jettyServerThreadPool.getBusyThreads())); + emitter.emit(builder.setMetric("jetty/threadPool/total", jettyServerThreadPool.getThreads())); + emitter.emit(builder.setMetric("jetty/threadPool/idle", jettyServerThreadPool.getIdleThreads())); + emitter.emit(builder.setMetric("jetty/threadPool/isLowOnThreads", jettyServerThreadPool.isLowOnThreads() ? 1 : 0)); + emitter.emit(builder.setMetric("jetty/threadPool/min", jettyServerThreadPool.getMinThreads())); + emitter.emit(builder.setMetric("jetty/threadPool/max", jettyServerThreadPool.getMaxThreads())); + emitter.emit(builder.setMetric("jetty/threadPool/queueSize", jettyServerThreadPool.getQueueSize())); + emitter.emit(builder.setMetric("jetty/threadPool/busy", jettyServerThreadPool.getBusyThreads())); } return true; diff --git a/server/src/main/java/org/apache/druid/server/metrics/EventReceiverFirehoseMonitor.java b/server/src/main/java/org/apache/druid/server/metrics/EventReceiverFirehoseMonitor.java index 8922f29b4b35..29c1808ad515 100644 --- a/server/src/main/java/org/apache/druid/server/metrics/EventReceiverFirehoseMonitor.java +++ b/server/src/main/java/org/apache/druid/server/metrics/EventReceiverFirehoseMonitor.java @@ -64,7 +64,7 @@ public boolean doMonitor(ServiceEmitter emitter) "bufferCapacity", String.valueOf(metric.getCapacity()) ); - emitter.emit(builder.build("ingest/events/buffered", metric.getCurrentBufferSize())); + emitter.emit(builder.setMetric("ingest/events/buffered", metric.getCurrentBufferSize())); Map diff = keyedDiff.to( serviceName, ImmutableMap.of("ingest/bytes/received", metric.getBytesReceived()) @@ -72,7 +72,7 @@ public boolean doMonitor(ServiceEmitter emitter) if (diff != null) { final ServiceMetricEvent.Builder eventBuilder = createEventBuilder(serviceName); for (Map.Entry diffEntry : diff.entrySet()) { - emitter.emit(eventBuilder.build(diffEntry.getKey(), diffEntry.getValue())); + emitter.emit(eventBuilder.setMetric(diffEntry.getKey(), diffEntry.getValue())); } } } diff --git a/server/src/main/java/org/apache/druid/server/metrics/HistoricalMetricsMonitor.java b/server/src/main/java/org/apache/druid/server/metrics/HistoricalMetricsMonitor.java index 30083cbc0911..ce21a2cef910 100644 --- a/server/src/main/java/org/apache/druid/server/metrics/HistoricalMetricsMonitor.java +++ b/server/src/main/java/org/apache/druid/server/metrics/HistoricalMetricsMonitor.java @@ -54,7 +54,7 @@ public HistoricalMetricsMonitor( @Override public boolean doMonitor(ServiceEmitter emitter) { - emitter.emit(new ServiceMetricEvent.Builder().build("segment/max", serverConfig.getMaxSize())); + emitter.emit(new ServiceMetricEvent.Builder().setMetric("segment/max", serverConfig.getMaxSize())); final Object2LongOpenHashMap pendingDeleteSizes = new Object2LongOpenHashMap<>(); @@ -71,7 +71,7 @@ public boolean doMonitor(ServiceEmitter emitter) .setDimension(DruidMetrics.DATASOURCE, dataSource) .setDimension("tier", serverConfig.getTier()) .setDimension("priority", String.valueOf(serverConfig.getPriority())) - .build("segment/pendingDelete", pendingDeleteSize) + .setMetric("segment/pendingDelete", pendingDeleteSize) ); } @@ -85,9 +85,9 @@ public boolean doMonitor(ServiceEmitter emitter) .setDimension("priority", String.valueOf(serverConfig.getPriority())); - emitter.emit(builder.build("segment/used", used)); + emitter.emit(builder.setMetric("segment/used", used)); final double usedPercent = serverConfig.getMaxSize() == 0 ? 0 : used / (double) serverConfig.getMaxSize(); - emitter.emit(builder.build("segment/usedPercent", usedPercent)); + emitter.emit(builder.setMetric("segment/usedPercent", usedPercent)); } for (Map.Entry entry : segmentManager.getDataSourceCounts().entrySet()) { @@ -101,7 +101,7 @@ public boolean doMonitor(ServiceEmitter emitter) String.valueOf(serverConfig.getPriority()) ); - emitter.emit(builder.build("segment/count", count)); + emitter.emit(builder.setMetric("segment/count", count)); } return true; diff --git a/server/src/main/java/org/apache/druid/server/metrics/QueryCountStatsMonitor.java b/server/src/main/java/org/apache/druid/server/metrics/QueryCountStatsMonitor.java index c58be1e04c78..da2017dbc00a 100644 --- a/server/src/main/java/org/apache/druid/server/metrics/QueryCountStatsMonitor.java +++ b/server/src/main/java/org/apache/druid/server/metrics/QueryCountStatsMonitor.java @@ -62,7 +62,7 @@ public boolean doMonitor(ServiceEmitter emitter) ); if (diff != null) { for (Map.Entry diffEntry : diff.entrySet()) { - emitter.emit(builder.build(diffEntry.getKey(), diffEntry.getValue())); + emitter.emit(builder.setMetric(diffEntry.getKey(), diffEntry.getValue())); } } return true; diff --git a/server/src/main/java/org/apache/druid/server/metrics/SegmentStatsMonitor.java b/server/src/main/java/org/apache/druid/server/metrics/SegmentStatsMonitor.java index c72aba29575f..68b703f8951f 100644 --- a/server/src/main/java/org/apache/druid/server/metrics/SegmentStatsMonitor.java +++ b/server/src/main/java/org/apache/druid/server/metrics/SegmentStatsMonitor.java @@ -81,7 +81,7 @@ public boolean doMonitor(ServiceEmitter emitter) .setDimension(DruidMetrics.DATASOURCE, dataSource) .setDimension("tier", serverConfig.getTier()) .setDimension("priority", String.valueOf(serverConfig.getPriority())); - emitter.emit(builder.build("segment/rowCount/avg", averageSize)); + emitter.emit(builder.setMetric("segment/rowCount/avg", averageSize)); } for (Map.Entry entry : segmentLoadDropHandler.getRowCountDistributionPerDatasource() @@ -95,7 +95,7 @@ public boolean doMonitor(ServiceEmitter emitter) .setDimension("tier", serverConfig.getTier()) .setDimension("priority", String.valueOf(serverConfig.getPriority())); builder.setDimension("range", bucketDimension); - ServiceEventBuilder output = builder.build("segment/rowCount/range/count", count); + ServiceEventBuilder output = builder.setMetric("segment/rowCount/range/count", count); emitter.emit(output); }); } diff --git a/server/src/main/java/org/apache/druid/server/metrics/ServiceStatusMonitor.java b/server/src/main/java/org/apache/druid/server/metrics/ServiceStatusMonitor.java index ad87c1bcb638..e1d0d17915ca 100644 --- a/server/src/main/java/org/apache/druid/server/metrics/ServiceStatusMonitor.java +++ b/server/src/main/java/org/apache/druid/server/metrics/ServiceStatusMonitor.java @@ -50,7 +50,7 @@ public boolean doMonitor(ServiceEmitter emitter) heartbeatTagsSupplier.get().forEach(builder::setDimension); } - emitter.emit(builder.build("service/heartbeat", 1)); + emitter.emit(builder.setMetric("service/heartbeat", 1)); return true; } } diff --git a/server/src/main/java/org/apache/druid/server/metrics/TaskCountStatsMonitor.java b/server/src/main/java/org/apache/druid/server/metrics/TaskCountStatsMonitor.java index f9cdaee75cac..3e120e0ffd41 100644 --- a/server/src/main/java/org/apache/druid/server/metrics/TaskCountStatsMonitor.java +++ b/server/src/main/java/org/apache/druid/server/metrics/TaskCountStatsMonitor.java @@ -67,7 +67,7 @@ private void emit(ServiceEmitter emitter, String key, Map counts) if (counts != null) { counts.forEach((k, v) -> { builder.setDimension("dataSource", k); - emitter.emit(builder.build(key, v)); + emitter.emit(builder.setMetric(key, v)); }); } } @@ -82,7 +82,7 @@ private void emit(ServiceEmitter emitter, CoordinatorStat stat, Map eventBuilder.setDimension(dim.reportedName(), dimValue) ); - emitter.emit(eventBuilder.build(stat.getMetricName(), value)); + emitter.emit(eventBuilder.setMetric(stat.getMetricName(), value)); } } diff --git a/server/src/main/java/org/apache/druid/server/metrics/TaskSlotCountStatsMonitor.java b/server/src/main/java/org/apache/druid/server/metrics/TaskSlotCountStatsMonitor.java index 3431e1e143b2..46227a3ca452 100644 --- a/server/src/main/java/org/apache/druid/server/metrics/TaskSlotCountStatsMonitor.java +++ b/server/src/main/java/org/apache/druid/server/metrics/TaskSlotCountStatsMonitor.java @@ -55,7 +55,7 @@ private void emit(ServiceEmitter emitter, String key, Map counts) if (counts != null) { counts.forEach((k, v) -> { builder.setDimension("category", k); - emitter.emit(builder.build(key, v)); + emitter.emit(builder.setMetric(key, v)); }); } } diff --git a/server/src/main/java/org/apache/druid/server/metrics/WorkerTaskCountStatsMonitor.java b/server/src/main/java/org/apache/druid/server/metrics/WorkerTaskCountStatsMonitor.java index a464a81a81de..d6b322c2f0e9 100644 --- a/server/src/main/java/org/apache/druid/server/metrics/WorkerTaskCountStatsMonitor.java +++ b/server/src/main/java/org/apache/druid/server/metrics/WorkerTaskCountStatsMonitor.java @@ -74,7 +74,7 @@ private void emit(ServiceEmitter emitter, String metricName, Long value) final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder(); builder.setDimension(DruidMetrics.CATEGORY, workerCategory); builder.setDimension(DruidMetrics.WORKER_VERSION, workerVersion); - emitter.emit(builder.build(metricName, value)); + emitter.emit(builder.setMetric(metricName, value)); } } } diff --git a/server/src/test/java/org/apache/druid/segment/realtime/RealtimeMetricsMonitorTest.java b/server/src/test/java/org/apache/druid/segment/realtime/RealtimeMetricsMonitorTest.java new file mode 100644 index 000000000000..cca07bc9708c --- /dev/null +++ b/server/src/test/java/org/apache/druid/segment/realtime/RealtimeMetricsMonitorTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.realtime; + +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.metrics.StubServiceEmitter; +import org.apache.druid.segment.indexing.DataSchema; +import org.apache.druid.segment.indexing.RealtimeIOConfig; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; +import java.util.Random; + +public class RealtimeMetricsMonitorTest +{ + + private StubServiceEmitter emitter; + private Random random; + + @Before + public void setup() + { + random = new Random(100); + emitter = new StubServiceEmitter("test", "localhost"); + } + + @Test + public void testDoMonitor() + { + FireDepartment fireDepartment = new FireDepartment( + new DataSchema("wiki", null, null, null, null, null, null, new DefaultObjectMapper()), + new RealtimeIOConfig(null, null), + null + ); + + // Add some metrics and invoke monitoring + final FireDepartmentMetrics metrics = fireDepartment.getMetrics(); + invokeRandomTimes(metrics::incrementThrownAway); + invokeRandomTimes(metrics::incrementUnparseable); + invokeRandomTimes(metrics::incrementProcessed); + invokeRandomTimes(metrics::incrementDedup); + invokeRandomTimes(metrics::incrementFailedHandoffs); + invokeRandomTimes(metrics::incrementFailedPersists); + invokeRandomTimes(metrics::incrementHandOffCount); + invokeRandomTimes(metrics::incrementNumPersists); + + metrics.incrementPushedRows(random.nextInt()); + metrics.incrementRowOutputCount(random.nextInt()); + metrics.incrementMergedRows(random.nextInt()); + metrics.incrementMergeCpuTime(random.nextInt()); + metrics.setSinkCount(random.nextInt()); + + RealtimeMetricsMonitor monitor = new RealtimeMetricsMonitor(Collections.singletonList(fireDepartment)); + monitor.doMonitor(emitter); + + // Verify the metrics + emitter.verifyValue("ingest/events/thrownAway", metrics.thrownAway()); + emitter.verifyValue("ingest/events/unparseable", metrics.unparseable()); + + emitter.verifyValue("ingest/events/duplicate", metrics.dedup()); + emitter.verifyValue("ingest/events/processed", metrics.processed()); + emitter.verifyValue("ingest/rows/output", metrics.rowOutput()); + emitter.verifyValue("ingest/persists/count", metrics.numPersists()); + emitter.verifyValue("ingest/persists/time", metrics.persistTimeMillis()); + emitter.verifyValue("ingest/persists/cpu", metrics.persistCpuTime()); + emitter.verifyValue("ingest/persists/backPressure", metrics.persistBackPressureMillis()); + emitter.verifyValue("ingest/persists/failed", metrics.failedPersists()); + emitter.verifyValue("ingest/handoff/failed", metrics.failedHandoffs()); + emitter.verifyValue("ingest/merge/time", metrics.mergeTimeMillis()); + emitter.verifyValue("ingest/merge/cpu", metrics.mergeCpuTime()); + emitter.verifyValue("ingest/handoff/count", metrics.handOffCount()); + emitter.verifyValue("ingest/sink/count", metrics.sinkCount()); + } + + private void invokeRandomTimes(Action action) + { + int limit = random.nextInt(20); + for (int i = 0; i < limit; ++i) { + action.perform(); + } + } + + @FunctionalInterface + private interface Action + { + void perform(); + } + +} diff --git a/server/src/test/java/org/apache/druid/server/log/ServiceMetricEventSerdeTest.java b/server/src/test/java/org/apache/druid/server/log/ServiceMetricEventSerdeTest.java index 341bab57bb0b..76b1cb9e8f90 100644 --- a/server/src/test/java/org/apache/druid/server/log/ServiceMetricEventSerdeTest.java +++ b/server/src/test/java/org/apache/druid/server/log/ServiceMetricEventSerdeTest.java @@ -38,7 +38,8 @@ public void testSerializeServiceMetricEventMap() throws JsonProcessingException String timestamp = "2022-08-17T18:51:00.000Z"; Event event = ServiceMetricEvent.builder() .setFeed("my-feed") - .build(DateTimes.of(timestamp), "m1", 1) + .setCreatedTime(DateTimes.of(timestamp)) + .setMetric("m1", 1) .build("my-service", "my-host"); String actual = mapper.writeValueAsString(event.toMap()); diff --git a/server/src/test/java/org/apache/druid/server/metrics/HistoricalMetricsMonitorTest.java b/server/src/test/java/org/apache/druid/server/metrics/HistoricalMetricsMonitorTest.java index 8e658faafb97..1806903a9baf 100644 --- a/server/src/test/java/org/apache/druid/server/metrics/HistoricalMetricsMonitorTest.java +++ b/server/src/test/java/org/apache/druid/server/metrics/HistoricalMetricsMonitorTest.java @@ -19,28 +19,21 @@ package org.apache.druid.server.metrics; -import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import org.apache.druid.client.DruidServerConfig; import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.java.util.emitter.service.ServiceEventBuilder; -import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; +import org.apache.druid.java.util.emitter.core.Event; +import org.apache.druid.java.util.metrics.StubServiceEmitter; import org.apache.druid.server.SegmentManager; import org.apache.druid.server.coordination.SegmentLoadDropHandler; import org.apache.druid.timeline.DataSegment; -import org.easymock.Capture; -import org.easymock.CaptureType; import org.easymock.EasyMock; import org.easymock.EasyMockSupport; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import javax.annotation.Nullable; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -49,7 +42,7 @@ public class HistoricalMetricsMonitorTest extends EasyMockSupport private DruidServerConfig druidServerConfig; private SegmentManager segmentManager; private SegmentLoadDropHandler segmentLoadDropMgr; - private ServiceEmitter serviceEmitter; + private StubServiceEmitter serviceEmitter; @Before public void setUp() @@ -57,7 +50,7 @@ public void setUp() druidServerConfig = EasyMock.createStrictMock(DruidServerConfig.class); segmentManager = EasyMock.createStrictMock(SegmentManager.class); segmentLoadDropMgr = EasyMock.createStrictMock(SegmentLoadDropHandler.class); - serviceEmitter = EasyMock.createStrictMock(ServiceEmitter.class); + serviceEmitter = new StubServiceEmitter("test", "localhost"); } @Test @@ -98,41 +91,16 @@ public void testSimple() segmentLoadDropMgr ); - final Capture> eventCapture = EasyMock.newCapture(CaptureType.ALL); - serviceEmitter.emit(EasyMock.capture(eventCapture)); - EasyMock.expectLastCall().times(5); - - EasyMock.replay(druidServerConfig, segmentManager, segmentLoadDropMgr, serviceEmitter); + EasyMock.replay(druidServerConfig, segmentManager, segmentLoadDropMgr); monitor.doMonitor(serviceEmitter); - EasyMock.verify(druidServerConfig, segmentManager, segmentLoadDropMgr, serviceEmitter); - - final String host = "host"; - final String service = "service"; - Assert.assertTrue(eventCapture.hasCaptured()); - final List> events = Lists.transform( - eventCapture.getValues(), - new Function, Map>() - { - @Nullable - @Override - public Map apply( - @Nullable ServiceEventBuilder input - ) - { - final HashMap map = new HashMap<>(input.build(service, host).toMap()); - Assert.assertNotNull(map.remove("feed")); - Assert.assertNotNull(map.remove("timestamp")); - Assert.assertNotNull(map.remove("service")); - Assert.assertNotNull(map.remove("host")); - return map; - } - } - ); + EasyMock.verify(druidServerConfig, segmentManager, segmentLoadDropMgr); + + final List events = serviceEmitter.getEvents(); Assert.assertEquals(ImmutableMap.of( "metric", "segment/max", "value", maxSize - ), events.get(0)); + ), asMap(events.get(0))); Assert.assertEquals(ImmutableMap.of( "dataSource", dataSource, @@ -140,7 +108,7 @@ public Map apply( "priority", String.valueOf(priority), "tier", tier, "value", dataSegment.getSize() - ), events.get(1)); + ), asMap(events.get(1))); Assert.assertEquals(ImmutableMap.of( "metric", "segment/used", @@ -148,7 +116,7 @@ public Map apply( "tier", tier, "priority", String.valueOf(priority), "dataSource", dataSource - ), events.get(2)); + ), asMap(events.get(2))); Assert.assertEquals(ImmutableMap.of( "metric", "segment/usedPercent", @@ -156,7 +124,7 @@ public Map apply( "tier", tier, "priority", String.valueOf(priority), "dataSource", dataSource - ), events.get(3)); + ), asMap(events.get(3))); Assert.assertEquals(ImmutableMap.of( "metric", "segment/count", @@ -164,6 +132,17 @@ public Map apply( "tier", tier, "priority", String.valueOf(priority), "dataSource", dataSource - ), events.get(4)); + ), asMap(events.get(4))); + } + + private Map asMap(Event event) + { + final Map map = event.toMap(); + Assert.assertNotNull(map.remove("feed")); + Assert.assertNotNull(map.remove("timestamp")); + Assert.assertNotNull(map.remove("service")); + Assert.assertNotNull(map.remove("host")); + + return map; } } diff --git a/server/src/test/java/org/apache/druid/server/metrics/SegmentStatsMonitorTest.java b/server/src/test/java/org/apache/druid/server/metrics/SegmentStatsMonitorTest.java index cb08a39fc36f..373b1c1dd2d6 100644 --- a/server/src/test/java/org/apache/druid/server/metrics/SegmentStatsMonitorTest.java +++ b/server/src/test/java/org/apache/druid/server/metrics/SegmentStatsMonitorTest.java @@ -200,7 +200,7 @@ private ServiceEventBuilder averageRowCountEvent(Number valu return new ServiceMetricEvent.Builder().setDimension(DruidMetrics.DATASOURCE, DATA_SOURCE) .setDimension("tier", TIER) .setDimension("priority", String.valueOf(PRIORITY)) - .build("segment/rowCount/avg", value); + .setMetric("segment/rowCount/avg", value); } private ServiceEventBuilder rowCountRangeEvent(String range, Number value) @@ -209,6 +209,6 @@ private ServiceEventBuilder rowCountRangeEvent(String range, .setDimension("tier", TIER) .setDimension("priority", String.valueOf(PRIORITY)) .setDimension("range", range) - .build("segment/rowCount/range/count", value); + .setMetric("segment/rowCount/range/count", value); } } diff --git a/sql/src/main/java/org/apache/druid/sql/SqlExecutionReporter.java b/sql/src/main/java/org/apache/druid/sql/SqlExecutionReporter.java index 922e4fe219c6..f55f2f3f1233 100644 --- a/sql/src/main/java/org/apache/druid/sql/SqlExecutionReporter.java +++ b/sql/src/main/java/org/apache/druid/sql/SqlExecutionReporter.java @@ -112,12 +112,12 @@ public void emit() } metricBuilder.setDimension("remoteAddress", StringUtils.nullToEmptyNonDruidDataString(remoteAddress)); metricBuilder.setDimension("success", String.valueOf(success)); - emitter.emit(metricBuilder.build("sqlQuery/time", TimeUnit.NANOSECONDS.toMillis(queryTimeNs))); + emitter.emit(metricBuilder.setMetric("sqlQuery/time", TimeUnit.NANOSECONDS.toMillis(queryTimeNs))); if (bytesWritten >= 0) { - emitter.emit(metricBuilder.build("sqlQuery/bytes", bytesWritten)); + emitter.emit(metricBuilder.setMetric("sqlQuery/bytes", bytesWritten)); } if (planningTimeNanos >= 0) { - emitter.emit(metricBuilder.build( + emitter.emit(metricBuilder.setMetric( "sqlQuery/planningTimeMs", TimeUnit.NANOSECONDS.toMillis(planningTimeNanos) )); diff --git a/sql/src/main/java/org/apache/druid/sql/avatica/AvaticaMonitor.java b/sql/src/main/java/org/apache/druid/sql/avatica/AvaticaMonitor.java index 45befef63253..af17abb1b333 100644 --- a/sql/src/main/java/org/apache/druid/sql/avatica/AvaticaMonitor.java +++ b/sql/src/main/java/org/apache/druid/sql/avatica/AvaticaMonitor.java @@ -50,14 +50,14 @@ public boolean doMonitor(final ServiceEmitter emitter) for (final Map.Entry entry : counters.entrySet()) { final String name = entry.getKey(); final long value = entry.getValue().getAndSet(0); - emitter.emit(ServiceMetricEvent.builder().build(fullMetricName(name), value)); + emitter.emit(ServiceMetricEvent.builder().setMetric(fullMetricName(name), value)); } for (Map.Entry> entry : gauges.entrySet()) { final String name = entry.getKey(); final Object value = entry.getValue().getValue(); if (value instanceof Number) { - emitter.emit(ServiceMetricEvent.builder().build(fullMetricName(name), (Number) value)); + emitter.emit(ServiceMetricEvent.builder().setMetric(fullMetricName(name), (Number) value)); } else { log.debug("Not emitting gauge[%s] since value[%s] type was[%s].", name, value, value.getClass().getName()); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCache.java index 37ff3bda313b..381f0ff4afd0 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCache.java @@ -396,7 +396,7 @@ public void start() throws InterruptedException awaitInitialization(); final long endMillis = System.currentTimeMillis(); log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); - emitter.emit(ServiceMetricEvent.builder().build( + emitter.emit(ServiceMetricEvent.builder().setMetric( "metadatacache/init/time", endMillis - startMillis )); @@ -722,7 +722,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder().setDimension(DruidMetrics.DATASOURCE, dataSource); - emitter.emit(builder.build("metadatacache/refresh/count", segments.size())); + emitter.emit(builder.setMetric("metadatacache/refresh/count", segments.size())); // Segment id string -> SegmentId object. final Map segmentIdMap = Maps.uniqueIndex(segments, SegmentId::toString); @@ -793,7 +793,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin long refreshDurationMillis = stopwatch.elapsed(TimeUnit.MILLISECONDS); - emitter.emit(builder.build("metadatacache/refresh/time", refreshDurationMillis)); + emitter.emit(builder.setMetric("metadatacache/refresh/time", refreshDurationMillis)); log.debug( "Refreshed metadata for dataSource [%s] in %,d ms (%d segments queried, %d segments left).", From ec630e3671045340e463111692ec054aa615ea42 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Mon, 4 Sep 2023 11:54:36 +0530 Subject: [PATCH 016/258] Remove deprecated coordinator dynamic configs (#14923) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes: [A] Remove config `decommissioningMaxPercentOfMaxSegmentsToMove` - It is a complicated config 😅 , - It is always desirable to prioritize move from decommissioning servers so that they can be terminated quickly, so this should always be 100% - It is already handled by `smartSegmentLoading` (enabled by default) [B] Remove config `maxNonPrimaryReplicantsToLoad` This was added in #11135 to address two requirements: - Prevent coordinator runs from getting stuck assigning too many segments to historicals - Prevent load of replicas from competing with load of unavailable segments Both of these requirements are now already met thanks to: - Round-robin segment assignment - Prioritization in the new coordinator - Modifications to `replicationThrottleLimit` - `smartSegmentLoading` (enabled by default) --- docs/configuration/index.md | 10 +- .../coordinator/CoordinatorDynamicConfig.java | 140 +----------------- .../balancer/TierSegmentBalancer.java | 6 +- .../loading/ReplicationThrottler.java | 26 +--- .../loading/SegmentLoadingConfig.java | 10 -- .../loading/StrategicSegmentAssigner.java | 3 +- .../coordinator/duty/BalanceSegmentsTest.java | 69 +-------- .../server/coordinator/duty/RunRulesTest.java | 9 +- .../loading/ReplicationThrottlerTest.java | 6 +- .../http/CoordinatorDynamicConfigTest.java | 117 +++------------ .../coordinator-dynamic-config.mock.ts | 2 - .../coordinator-dynamic-config.tsx | 48 +----- 12 files changed, 46 insertions(+), 400 deletions(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index da8ce4235b3c..b8d4d6a4a502 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -940,10 +940,8 @@ A sample Coordinator dynamic config JSON object is shown below: "killTaskSlotRatio": 0.10, "maxKillTaskSlots": 5, "decommissioningNodes": ["localhost:8182", "localhost:8282"], - "decommissioningMaxPercentOfMaxSegmentsToMove": 70, "pauseCoordination": false, - "replicateAfterLoadTimeout": false, - "maxNonPrimaryReplicantsToLoad": 2147483647 + "replicateAfterLoadTimeout": false } ``` @@ -965,11 +963,9 @@ Issuing a GET request at the same URL will return the spec that is currently in |`killPendingSegmentsSkipList`|List of data sources for which pendingSegments are _NOT_ cleaned up if property `druid.coordinator.kill.pendingSegments.on` is true. This can be a list of comma-separated data sources or a JSON array.|none| |`maxSegmentsInNodeLoadingQueue`|The maximum number of segments allowed in the load queue of any given server. Use this parameter to load segments faster if, for example, the cluster contains slow-loading nodes or if there are too many segments to be replicated to a particular node (when faster loading is preferred to better segments distribution). The optimal value depends on the loading speed of segments, acceptable replication time and number of nodes.|500| |`useRoundRobinSegmentAssignment`|Boolean flag for whether segments should be assigned to historicals in a round robin fashion. When disabled, segment assignment is done using the chosen balancer strategy. When enabled, this can speed up segment assignments leaving balancing to move the segments to their optimal locations (based on the balancer strategy) lazily.|true| -|`decommissioningNodes`|List of historical servers to 'decommission'. Coordinator will not assign new segments to 'decommissioning' servers, and segments will be moved away from them to be placed on non-decommissioning servers at the maximum rate specified by `decommissioningMaxPercentOfMaxSegmentsToMove`.|none| -|`decommissioningMaxPercentOfMaxSegmentsToMove`|Upper limit of segments the Coordinator can move from decommissioning servers to active non-decommissioning servers during a single run. This value is relative to the total maximum number of segments that can be moved at any given time based upon the value of `maxSegmentsToMove`.

If `decommissioningMaxPercentOfMaxSegmentsToMove` is 0, the Coordinator does not move segments to decommissioning servers, effectively putting them in a type of "maintenance" mode. In this case, decommissioning servers do not participate in balancing or assignment by load rules. The Coordinator still considers segments on decommissioning servers as candidates to replicate on active servers.

Decommissioning can stall if there are no available active servers to move the segments to. You can use the maximum percent of decommissioning segment movements to prioritize balancing or to decrease commissioning time to prevent active servers from being overloaded. The value must be between 0 and 100.|70| +|`decommissioningNodes`|List of historical servers to 'decommission'. Coordinator will not assign new segments to 'decommissioning' servers, and segments will be moved away from them to be placed on non-decommissioning servers at the maximum rate specified by `maxSegmentsToMove`.|none| |`pauseCoordination`|Boolean flag for whether or not the coordinator should execute its various duties of coordinating the cluster. Setting this to true essentially pauses all coordination work while allowing the API to remain up. Duties that are paused include all classes that implement the `CoordinatorDuty` Interface. Such duties include: Segment balancing, Segment compaction, Submitting kill tasks for unused segments (if enabled), Logging of used segments in the cluster, Marking of newly unused or overshadowed segments, Matching and execution of load/drop rules for used segments, Unloading segments that are no longer marked as used from Historical servers. An example of when an admin may want to pause coordination would be if they are doing deep storage maintenance on HDFS Name Nodes with downtime and don't want the coordinator to be directing Historical Nodes to hit the Name Node with API requests until maintenance is done and the deep store is declared healthy for use again.|false| |`replicateAfterLoadTimeout`|Boolean flag for whether or not additional replication is needed for segments that have failed to load due to the expiry of `druid.coordinator.load.timeout`. If this is set to true, the coordinator will attempt to replicate the failed segment on a different historical server. This helps improve the segment availability if there are a few slow historicals in the cluster. However, the slow historical may still load the segment later and the coordinator may issue drop requests if the segment is over-replicated.|false| -|`maxNonPrimaryReplicantsToLoad`|The maximum number of replicas that can be assigned across all tiers in a single Coordinator run. This parameter serves the same purpose as `replicationThrottleLimit` except this limit applies at the cluster-level instead of per tier. The default value does not apply a limit to the number of replicas assigned per coordination cycle. If you want to use a non-default value for this property, you may want to start with `~20%` of the number of segments found on the historical server with the most segments. Use the Druid metric, `coordinator/time` with the filter `duty=org.apache.druid.server.coordinator.duty.RunRules` to see how different values of this property impact your Coordinator execution time.|`Integer.MAX_VALUE` (no limit)| ##### Smart segment loading @@ -986,9 +982,7 @@ Druid computes the values to optimize Coordinator performance, based on the curr |`maxSegmentsInNodeLoadingQueue`|0|Removes the limit on load queue size.| |`replicationThrottleLimit`|5% of used segments, minimum value 100|Prevents aggressive replication when a historical disappears only intermittently.| |`replicantLifetime`|60|Allows segments to wait about an hour (assuming a Coordinator period of 1 minute) in the load queue before an alert is raised. In `smartSegmentLoading` mode, load queues are not limited by size. Segments might therefore assigned to a load queue even if the corresponding server is slow to load them.| -|`maxNonPrimaryReplicantsToLoad`|`Integer.MAX_VALUE` (no limit)|This throttling is already handled by `replicationThrottleLimit`.| |`maxSegmentsToMove`|2% of used segments, minimum value 100, maximum value 1000|Ensures that some segments are always moving in the cluster to keep it well balanced. The maximum value keeps the Coordinator run times bounded.| -|`decommissioningMaxPercentOfMaxSegmentsToMove`|100|Prioritizes the move of segments from decommissioning servers so that they can be terminated quickly.| |`balancerComputeThreads`|`num_cores` / 2|Ensures that there are enough threads to perform balancing computations without hogging all Coordinator resources.| When `smartSegmentLoading` is disabled, Druid uses the configured values of these properties. diff --git a/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java index c9811fa3186f..0b39688dbf55 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java @@ -26,7 +26,6 @@ import com.google.common.collect.ImmutableSet; import org.apache.druid.common.config.JacksonConfigManager; import org.apache.druid.error.InvalidInput; -import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.server.coordinator.duty.KillUnusedSegments; import org.apache.druid.server.coordinator.loading.LoadQueuePeon; import org.apache.druid.server.coordinator.stats.Dimension; @@ -34,8 +33,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -import javax.validation.constraints.Max; -import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; import java.util.Collection; import java.util.EnumMap; @@ -79,9 +76,6 @@ public class CoordinatorDynamicConfig private final Map debugDimensions; private final Map validDebugDimensions; - @Deprecated - private final int decommissioningMaxPercentOfMaxSegmentsToMove; - /** * Stale pending segments belonging to the data sources in this list are not killed by {@link * KillStalePendingSegments}. In other words, segments in these data sources are "protected". @@ -92,10 +86,10 @@ public class CoordinatorDynamicConfig private final Set dataSourcesToNotKillStalePendingSegmentsIn; /** - * The maximum number of segments that could be queued for loading to any given server. - * Default values is 0 with the meaning of "unbounded" (any number of - * segments could be added to the loading queue for any server). - * See {@link LoadQueuePeon}, {@link org.apache.druid.server.coordinator.rules.LoadRule#run} + * The maximum number of segments that can be queued for loading to any given server. + * + * @see LoadQueuePeon + * @see org.apache.druid.server.coordinator.rules.LoadRule#run */ private final int maxSegmentsInNodeLoadingQueue; private final boolean pauseCoordination; @@ -108,16 +102,6 @@ public class CoordinatorDynamicConfig */ private final boolean replicateAfterLoadTimeout; - /** - * This is the maximum number of non-primary segment replicants to load per Coordination run. This number can - * be set to put a hard upper limit on the number of replicants loaded. It is a tool that can help prevent - * long delays in new data loads after events such as a Historical server leaving the cluster. - */ - @Deprecated - private final int maxNonPrimaryReplicantsToLoad; - - private static final Logger log = new Logger(CoordinatorDynamicConfig.class); - @JsonCreator public CoordinatorDynamicConfig( // Keeping the legacy 'millisToWaitBeforeDeleting' property name for backward compatibility. When the project is @@ -144,10 +128,8 @@ public CoordinatorDynamicConfig( @JsonProperty("killPendingSegmentsSkipList") Object dataSourcesToNotKillStalePendingSegmentsIn, @JsonProperty("maxSegmentsInNodeLoadingQueue") @Nullable Integer maxSegmentsInNodeLoadingQueue, @JsonProperty("decommissioningNodes") Object decommissioningNodes, - @JsonProperty("decommissioningMaxPercentOfMaxSegmentsToMove") int decommissioningMaxPercentOfMaxSegmentsToMove, @JsonProperty("pauseCoordination") boolean pauseCoordination, @JsonProperty("replicateAfterLoadTimeout") boolean replicateAfterLoadTimeout, - @JsonProperty("maxNonPrimaryReplicantsToLoad") @Nullable Integer maxNonPrimaryReplicantsToLoad, @JsonProperty("useRoundRobinSegmentAssignment") @Nullable Boolean useRoundRobinSegmentAssignment, @JsonProperty("smartSegmentLoading") @Nullable Boolean smartSegmentLoading, @JsonProperty("debugDimensions") @Nullable Map debugDimensions @@ -186,31 +168,10 @@ public CoordinatorDynamicConfig( Defaults.MAX_SEGMENTS_IN_NODE_LOADING_QUEUE ); this.decommissioningNodes = parseJsonStringOrArray(decommissioningNodes); - Preconditions.checkArgument( - decommissioningMaxPercentOfMaxSegmentsToMove >= 0 && decommissioningMaxPercentOfMaxSegmentsToMove <= 100, - "'decommissioningMaxPercentOfMaxSegmentsToMove' should be in range [0, 100]" - ); - this.decommissioningMaxPercentOfMaxSegmentsToMove = decommissioningMaxPercentOfMaxSegmentsToMove; this.pauseCoordination = pauseCoordination; this.replicateAfterLoadTimeout = replicateAfterLoadTimeout; - if (maxNonPrimaryReplicantsToLoad == null) { - log.debug( - "maxNonPrimaryReplicantsToLoad was null! This is likely because your metastore does not " - + "reflect this configuration being added to Druid in a recent release. Druid is defaulting the value " - + "to the Druid default of %d. It is recommended that you re-submit your dynamic config with your " - + "desired value for maxNonPrimaryReplicantsToLoad", - Defaults.MAX_NON_PRIMARY_REPLICANTS_TO_LOAD - ); - maxNonPrimaryReplicantsToLoad = Defaults.MAX_NON_PRIMARY_REPLICANTS_TO_LOAD; - } - Preconditions.checkArgument( - maxNonPrimaryReplicantsToLoad >= 0, - "maxNonPrimaryReplicantsToLoad must be greater than or equal to 0." - ); - this.maxNonPrimaryReplicantsToLoad = maxNonPrimaryReplicantsToLoad; - this.useRoundRobinSegmentAssignment = Builder.valueOrDefault( useRoundRobinSegmentAssignment, Defaults.USE_ROUND_ROBIN_ASSIGNMENT @@ -362,8 +323,7 @@ public boolean isSmartSegmentLoading() /** * List of historical servers to 'decommission'. Coordinator will not assign new segments to 'decommissioning' - * servers, and segments will be moved away from them to be placed on non-decommissioning servers at the maximum rate - * specified by {@link CoordinatorDynamicConfig#getDecommissioningMaxPercentOfMaxSegmentsToMove}. + * servers, and segments will be moved away from them to be placed on non-decommissioning servers. * * @return list of host:port entries */ @@ -385,27 +345,6 @@ public Map getValidatedDebugDimensions() return validDebugDimensions; } - /** - * The percent of {@link CoordinatorDynamicConfig#getMaxSegmentsToMove()} that determines the maximum number of - * segments that may be moved away from 'decommissioning' servers (specified by - * {@link CoordinatorDynamicConfig#getDecommissioningNodes()}) to non-decommissioning servers during one Coordinator - * balancer run. If this value is 0, segments will neither be moved from or to 'decommissioning' servers, effectively - * putting them in a sort of "maintenance" mode that will not participate in balancing or assignment by load rules. - * Decommissioning can also become stalled if there are no available active servers to place the segments. By - * adjusting this value, an operator can prevent active servers from overload by prioritizing balancing, or - * decrease decommissioning time instead. - * - * @return number in range [0, 100] - */ - @Min(0) - @Max(100) - @Deprecated - @JsonProperty - public int getDecommissioningMaxPercentOfMaxSegmentsToMove() - { - return decommissioningMaxPercentOfMaxSegmentsToMove; - } - @JsonProperty public boolean getPauseCoordination() { @@ -418,14 +357,6 @@ public boolean getReplicateAfterLoadTimeout() return replicateAfterLoadTimeout; } - @Min(0) - @Deprecated - @JsonProperty - public int getMaxNonPrimaryReplicantsToLoad() - { - return maxNonPrimaryReplicantsToLoad; - } - @Override public String toString() { @@ -444,10 +375,8 @@ public String toString() ", dataSourcesToNotKillStalePendingSegmentsIn=" + dataSourcesToNotKillStalePendingSegmentsIn + ", maxSegmentsInNodeLoadingQueue=" + maxSegmentsInNodeLoadingQueue + ", decommissioningNodes=" + decommissioningNodes + - ", decommissioningMaxPercentOfMaxSegmentsToMove=" + decommissioningMaxPercentOfMaxSegmentsToMove + ", pauseCoordination=" + pauseCoordination + ", replicateAfterLoadTimeout=" + replicateAfterLoadTimeout + - ", maxNonPrimaryReplicantsToLoad=" + maxNonPrimaryReplicantsToLoad + '}'; } @@ -467,13 +396,11 @@ public boolean equals(Object o) && mergeBytesLimit == that.mergeBytesLimit && mergeSegmentsLimit == that.mergeSegmentsLimit && maxSegmentsToMove == that.maxSegmentsToMove - && decommissioningMaxPercentOfMaxSegmentsToMove == that.decommissioningMaxPercentOfMaxSegmentsToMove && balancerComputeThreads == that.balancerComputeThreads && replicantLifetime == that.replicantLifetime && replicationThrottleLimit == that.replicationThrottleLimit && replicateAfterLoadTimeout == that.replicateAfterLoadTimeout && maxSegmentsInNodeLoadingQueue == that.maxSegmentsInNodeLoadingQueue - && maxNonPrimaryReplicantsToLoad == that.maxNonPrimaryReplicantsToLoad && useRoundRobinSegmentAssignment == that.useRoundRobinSegmentAssignment && pauseCoordination == that.pauseCoordination && Objects.equals( @@ -505,9 +432,7 @@ public int hashCode() maxKillTaskSlots, dataSourcesToNotKillStalePendingSegmentsIn, decommissioningNodes, - decommissioningMaxPercentOfMaxSegmentsToMove, pauseCoordination, - maxNonPrimaryReplicantsToLoad, debugDimensions ); } @@ -535,10 +460,8 @@ private static class Defaults static final int REPLICANT_LIFETIME = 15; static final int REPLICATION_THROTTLE_LIMIT = 500; static final int MAX_SEGMENTS_IN_NODE_LOADING_QUEUE = 500; - static final int DECOMMISSIONING_MAX_SEGMENTS_TO_MOVE_PERCENT = 70; static final boolean PAUSE_COORDINATION = false; static final boolean REPLICATE_AFTER_LOAD_TIMEOUT = false; - static final int MAX_NON_PRIMARY_REPLICANTS_TO_LOAD = Integer.MAX_VALUE; static final boolean USE_ROUND_ROBIN_ASSIGNMENT = true; static final boolean SMART_SEGMENT_LOADING = true; @@ -566,10 +489,8 @@ public static class Builder private Integer maxSegmentsInNodeLoadingQueue; private Object decommissioningNodes; private Map debugDimensions; - private Integer decommissioningMaxPercentOfMaxSegmentsToMove; private Boolean pauseCoordination; private Boolean replicateAfterLoadTimeout; - private Integer maxNonPrimaryReplicantsToLoad; private Boolean useRoundRobinSegmentAssignment; private Boolean smartSegmentLoading; @@ -592,11 +513,8 @@ public Builder( @JsonProperty("killPendingSegmentsSkipList") @Nullable Object dataSourcesToNotKillStalePendingSegmentsIn, @JsonProperty("maxSegmentsInNodeLoadingQueue") @Nullable Integer maxSegmentsInNodeLoadingQueue, @JsonProperty("decommissioningNodes") @Nullable Object decommissioningNodes, - @JsonProperty("decommissioningMaxPercentOfMaxSegmentsToMove") - @Nullable Integer decommissioningMaxPercentOfMaxSegmentsToMove, @JsonProperty("pauseCoordination") @Nullable Boolean pauseCoordination, @JsonProperty("replicateAfterLoadTimeout") @Nullable Boolean replicateAfterLoadTimeout, - @JsonProperty("maxNonPrimaryReplicantsToLoad") @Nullable Integer maxNonPrimaryReplicantsToLoad, @JsonProperty("useRoundRobinSegmentAssignment") @Nullable Boolean useRoundRobinSegmentAssignment, @JsonProperty("smartSegmentLoading") @Nullable Boolean smartSegmentLoading, @JsonProperty("debugDimensions") @Nullable Map debugDimensions @@ -615,10 +533,8 @@ public Builder( this.dataSourcesToNotKillStalePendingSegmentsIn = dataSourcesToNotKillStalePendingSegmentsIn; this.maxSegmentsInNodeLoadingQueue = maxSegmentsInNodeLoadingQueue; this.decommissioningNodes = decommissioningNodes; - this.decommissioningMaxPercentOfMaxSegmentsToMove = decommissioningMaxPercentOfMaxSegmentsToMove; this.pauseCoordination = pauseCoordination; this.replicateAfterLoadTimeout = replicateAfterLoadTimeout; - this.maxNonPrimaryReplicantsToLoad = maxNonPrimaryReplicantsToLoad; this.useRoundRobinSegmentAssignment = useRoundRobinSegmentAssignment; this.smartSegmentLoading = smartSegmentLoading; this.debugDimensions = debugDimensions; @@ -630,18 +546,6 @@ public Builder withMarkSegmentAsUnusedDelayMillis(long leadingTimeMillis) return this; } - public Builder withMergeBytesLimit(long mergeBytesLimit) - { - this.mergeBytesLimit = mergeBytesLimit; - return this; - } - - public Builder withMergeSegmentsLimit(int mergeSegmentsLimit) - { - this.mergeSegmentsLimit = mergeSegmentsLimit; - return this; - } - public Builder withMaxSegmentsToMove(int maxSegmentsToMove) { this.maxSegmentsToMove = maxSegmentsToMove; @@ -666,18 +570,6 @@ public Builder withReplicationThrottleLimit(int replicationThrottleLimit) return this; } - public Builder withDebugDimensions(Map debugDimensions) - { - this.debugDimensions = debugDimensions; - return this; - } - - public Builder withBalancerComputeThreads(int balancerComputeThreads) - { - this.balancerComputeThreads = balancerComputeThreads; - return this; - } - public Builder withSpecificDataSourcesToKillUnusedSegmentsIn(Set dataSources) { this.specificDataSourcesToKillUnusedSegmentsIn = dataSources; @@ -708,12 +600,6 @@ public Builder withDecommissioningNodes(Set decommissioning) return this; } - public Builder withDecommissioningMaxPercentOfMaxSegmentsToMove(Integer percent) - { - this.decommissioningMaxPercentOfMaxSegmentsToMove = percent; - return this; - } - public Builder withPauseCoordination(boolean pauseCoordination) { this.pauseCoordination = pauseCoordination; @@ -726,12 +612,6 @@ public Builder withReplicateAfterLoadTimeout(boolean replicateAfterLoadTimeout) return this; } - public Builder withMaxNonPrimaryReplicantsToLoad(int maxNonPrimaryReplicantsToLoad) - { - this.maxNonPrimaryReplicantsToLoad = maxNonPrimaryReplicantsToLoad; - return this; - } - public Builder withUseRoundRobinSegmentAssignment(boolean useRoundRobinSegmentAssignment) { this.useRoundRobinSegmentAssignment = useRoundRobinSegmentAssignment; @@ -761,13 +641,8 @@ public CoordinatorDynamicConfig build() dataSourcesToNotKillStalePendingSegmentsIn, valueOrDefault(maxSegmentsInNodeLoadingQueue, Defaults.MAX_SEGMENTS_IN_NODE_LOADING_QUEUE), decommissioningNodes, - valueOrDefault( - decommissioningMaxPercentOfMaxSegmentsToMove, - Defaults.DECOMMISSIONING_MAX_SEGMENTS_TO_MOVE_PERCENT - ), valueOrDefault(pauseCoordination, Defaults.PAUSE_COORDINATION), valueOrDefault(replicateAfterLoadTimeout, Defaults.REPLICATE_AFTER_LOAD_TIMEOUT), - valueOrDefault(maxNonPrimaryReplicantsToLoad, Defaults.MAX_NON_PRIMARY_REPLICANTS_TO_LOAD), valueOrDefault(useRoundRobinSegmentAssignment, Defaults.USE_ROUND_ROBIN_ASSIGNMENT), valueOrDefault(smartSegmentLoading, Defaults.SMART_SEGMENT_LOADING), debugDimensions @@ -798,13 +673,8 @@ public CoordinatorDynamicConfig build(CoordinatorDynamicConfig defaults) valueOrDefault(dataSourcesToNotKillStalePendingSegmentsIn, defaults.getDataSourcesToNotKillStalePendingSegmentsIn()), valueOrDefault(maxSegmentsInNodeLoadingQueue, defaults.getMaxSegmentsInNodeLoadingQueue()), valueOrDefault(decommissioningNodes, defaults.getDecommissioningNodes()), - valueOrDefault( - decommissioningMaxPercentOfMaxSegmentsToMove, - defaults.getDecommissioningMaxPercentOfMaxSegmentsToMove() - ), valueOrDefault(pauseCoordination, defaults.getPauseCoordination()), valueOrDefault(replicateAfterLoadTimeout, defaults.getReplicateAfterLoadTimeout()), - valueOrDefault(maxNonPrimaryReplicantsToLoad, defaults.getMaxNonPrimaryReplicantsToLoad()), valueOrDefault(useRoundRobinSegmentAssignment, defaults.isUseRoundRobinSegmentAssignment()), valueOrDefault(smartSegmentLoading, defaults.isSmartSegmentLoading()), valueOrDefault(debugDimensions, defaults.getDebugDimensions()) diff --git a/server/src/main/java/org/apache/druid/server/coordinator/balancer/TierSegmentBalancer.java b/server/src/main/java/org/apache/druid/server/coordinator/balancer/TierSegmentBalancer.java index 1523576b11c3..95dc87d09a24 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/balancer/TierSegmentBalancer.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/balancer/TierSegmentBalancer.java @@ -201,17 +201,13 @@ private void markUnmoved(String reason, DataSegment segment) */ private int getNumDecommSegmentsToMove(int maxSegmentsToMove) { - final CoordinatorDynamicConfig dynamicConfig = params.getCoordinatorDynamicConfig(); if (decommissioningServers.isEmpty() || activeServers.isEmpty()) { return 0; - } else if (dynamicConfig.isSmartSegmentLoading()) { + } else { final int decommSegmentsToMove = decommissioningServers.stream().mapToInt( server -> server.getProjectedSegments().getTotalSegmentCount() ).sum(); return Math.min(decommSegmentsToMove, maxSegmentsToMove); - } else { - int maxPercentageToMove = dynamicConfig.getDecommissioningMaxPercentOfMaxSegmentsToMove(); - return (int) Math.ceil(maxSegmentsToMove * (maxPercentageToMove / 100.0)); } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/loading/ReplicationThrottler.java b/server/src/main/java/org/apache/druid/server/coordinator/loading/ReplicationThrottler.java index 55f5143f31b3..3cb524d42266 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/loading/ReplicationThrottler.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/loading/ReplicationThrottler.java @@ -31,21 +31,15 @@ *
    *
  • {@link CoordinatorDynamicConfig#getReplicationThrottleLimit()} - Maximum * number of replicas that can be assigned to a tier in a single run.
    • - *
  • {@link CoordinatorDynamicConfig#getMaxNonPrimaryReplicantsToLoad()} - - * Maximum number of total replicas that can be assigned across all tiers in a - * single run.
    • *
*/ public class ReplicationThrottler { private final int replicationThrottleLimit; - private final int maxReplicaAssignmentsInRun; private final Object2IntOpenHashMap tierToNumAssigned = new Object2IntOpenHashMap<>(); private final Object2IntOpenHashMap tierToMaxAssignments = new Object2IntOpenHashMap<>(); - private int totalReplicasAssignedInRun; - /** * Creates a new ReplicationThrottler for use during a single coordinator run. * The number of replicas loading on a tier must always be within the current @@ -53,22 +47,17 @@ public class ReplicationThrottler * replicas at the start of a coordinator run, it may be assigned only * {@code replicationThrottleLimit - k} more replicas during the run. * - * @param tierToLoadingReplicaCount Map from tier name to number of replicas - * already being loaded. - * @param replicationThrottleLimit Maximum number of replicas that can be - * assigned to a single tier in the current run. - * @param maxReplicaAssignmentsInRun Max number of total replicas that can be - * assigned across all tiers in the current run. + * @param tierToLoadingReplicaCount Map from tier name to number of replicas + * already being loaded. + * @param replicationThrottleLimit Maximum number of replicas that can be + * assigned to a single tier in the current run. */ public ReplicationThrottler( Map tierToLoadingReplicaCount, - int replicationThrottleLimit, - int maxReplicaAssignmentsInRun + int replicationThrottleLimit ) { this.replicationThrottleLimit = replicationThrottleLimit; - this.maxReplicaAssignmentsInRun = maxReplicaAssignmentsInRun; - this.totalReplicasAssignedInRun = 0; if (tierToLoadingReplicaCount != null) { tierToLoadingReplicaCount.forEach( @@ -82,13 +71,12 @@ public ReplicationThrottler( public boolean isReplicationThrottledForTier(String tier) { - return tierToNumAssigned.getInt(tier) >= tierToMaxAssignments.getOrDefault(tier, replicationThrottleLimit) - || totalReplicasAssignedInRun >= maxReplicaAssignmentsInRun; + return tierToNumAssigned.getOrDefault(tier, 0) + >= tierToMaxAssignments.getOrDefault(tier, replicationThrottleLimit); } public void incrementAssignedReplicas(String tier) { - ++totalReplicasAssignedInRun; tierToNumAssigned.addTo(tier, 1); } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/loading/SegmentLoadingConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/loading/SegmentLoadingConfig.java index d1f01043ba24..6b15c3c01a84 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/loading/SegmentLoadingConfig.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/loading/SegmentLoadingConfig.java @@ -32,7 +32,6 @@ public class SegmentLoadingConfig private final int maxSegmentsInLoadQueue; private final int replicationThrottleLimit; - private final int maxReplicaAssignmentsInRun; private final int maxLifetimeInLoadQueue; private final int balancerComputeThreads; @@ -58,7 +57,6 @@ public static SegmentLoadingConfig create(CoordinatorDynamicConfig dynamicConfig return new SegmentLoadingConfig( 0, replicationThrottleLimit, - Integer.MAX_VALUE, 60, true, CoordinatorDynamicConfig.getDefaultBalancerComputeThreads() @@ -68,7 +66,6 @@ public static SegmentLoadingConfig create(CoordinatorDynamicConfig dynamicConfig return new SegmentLoadingConfig( dynamicConfig.getMaxSegmentsInNodeLoadingQueue(), dynamicConfig.getReplicationThrottleLimit(), - dynamicConfig.getMaxNonPrimaryReplicantsToLoad(), dynamicConfig.getReplicantLifetime(), dynamicConfig.isUseRoundRobinSegmentAssignment(), dynamicConfig.getBalancerComputeThreads() @@ -79,7 +76,6 @@ public static SegmentLoadingConfig create(CoordinatorDynamicConfig dynamicConfig private SegmentLoadingConfig( int maxSegmentsInLoadQueue, int replicationThrottleLimit, - int maxReplicaAssignmentsInRun, int maxLifetimeInLoadQueue, boolean useRoundRobinSegmentAssignment, int balancerComputeThreads @@ -87,7 +83,6 @@ private SegmentLoadingConfig( { this.maxSegmentsInLoadQueue = maxSegmentsInLoadQueue; this.replicationThrottleLimit = replicationThrottleLimit; - this.maxReplicaAssignmentsInRun = maxReplicaAssignmentsInRun; this.maxLifetimeInLoadQueue = maxLifetimeInLoadQueue; this.useRoundRobinSegmentAssignment = useRoundRobinSegmentAssignment; this.balancerComputeThreads = balancerComputeThreads; @@ -113,11 +108,6 @@ public int getMaxLifetimeInLoadQueue() return maxLifetimeInLoadQueue; } - public int getMaxReplicaAssignmentsInRun() - { - return maxReplicaAssignmentsInRun; - } - public int getBalancerComputeThreads() { return balancerComputeThreads; diff --git a/server/src/main/java/org/apache/druid/server/coordinator/loading/StrategicSegmentAssigner.java b/server/src/main/java/org/apache/druid/server/coordinator/loading/StrategicSegmentAssigner.java index 0cc57db98809..d371cb3e40b9 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/loading/StrategicSegmentAssigner.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/loading/StrategicSegmentAssigner.java @@ -573,8 +573,7 @@ private static ReplicationThrottler createReplicationThrottler( ); return new ReplicationThrottler( tierToLoadingReplicaCount, - loadingConfig.getReplicationThrottleLimit(), - loadingConfig.getMaxReplicaAssignmentsInRun() + loadingConfig.getReplicationThrottleLimit() ); } diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/BalanceSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/BalanceSegmentsTest.java index 4122a4c752aa..bd2f00b74e24 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/BalanceSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/BalanceSegmentsTest.java @@ -121,68 +121,6 @@ public void testMoveToEmptyServerBalancer() Assert.assertEquals(2L, totalMoved); } - /** - * Server 1 has 2 segments. - * Server 2 (decommissioning) has 2 segments. - * Server 3 is empty. - * Decommissioning percent is 60. - * Max segments to move is 3. - * 2 (of 2) segments should be moved from Server 2 and 1 (of 2) from Server 1. - */ - @Test - public void testMoveDecommissioningMaxPercentOfMaxSegmentsToMove() - { - final ServerHolder serverHolder1 = createHolder(server1, true, segment1, segment2, segment3, segment4); - final ServerHolder serverHolder2 = createHolder(server2, false); - - // ceil(3 * 0.6) = 2 segments from decommissioning servers - CoordinatorDynamicConfig dynamicConfig = - CoordinatorDynamicConfig.builder() - .withSmartSegmentLoading(false) - .withMaxSegmentsToMove(3) - .withDecommissioningMaxPercentOfMaxSegmentsToMove(60) - .build(); - DruidCoordinatorRuntimeParams params = - defaultRuntimeParamsBuilder(serverHolder1, serverHolder2) - .withDynamicConfigs(dynamicConfig) - .withBalancerStrategy(balancerStrategy) - .withBroadcastDatasources(broadcastDatasources) - .withSegmentAssignerUsing(loadQueueManager) - .build(); - - CoordinatorRunStats stats = runBalancer(params); - - // 2 segments are moved from the decommissioning server - long totalMoved = stats.getSegmentStat(Stats.Segments.MOVED, "normal", "datasource1") - + stats.getSegmentStat(Stats.Segments.MOVED, "normal", "datasource2"); - Assert.assertEquals(2L, totalMoved); - Set segmentsMoved = serverHolder2.getPeon().getSegmentsToLoad(); - Assert.assertEquals(2, segmentsMoved.size()); - } - - @Test - public void testZeroDecommissioningMaxPercentOfMaxSegmentsToMove() - { - final ServerHolder holder1 = createHolder(server1, false, segment1, segment2); - final ServerHolder holder2 = createHolder(server2, true, segment3, segment4); - final ServerHolder holder3 = createHolder(server3, false); - - CoordinatorDynamicConfig dynamicConfig = - CoordinatorDynamicConfig.builder() - .withSmartSegmentLoading(false) - .withDecommissioningMaxPercentOfMaxSegmentsToMove(0) - .withMaxSegmentsToMove(1).build(); - DruidCoordinatorRuntimeParams params = - defaultRuntimeParamsBuilder(holder1, holder2, holder3).withDynamicConfigs(dynamicConfig).build(); - - CoordinatorRunStats stats = runBalancer(params); - - // Verify that either segment1 or segment2 is chosen for move - Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.MOVED, "normal", segment1.getDataSource())); - DataSegment movingSegment = holder3.getPeon().getSegmentsToLoad().iterator().next(); - Assert.assertEquals(segment1.getDataSource(), movingSegment.getDataSource()); - } - @Test public void testMaxDecommissioningMaxPercentOfMaxSegmentsToMove() { @@ -193,7 +131,6 @@ public void testMaxDecommissioningMaxPercentOfMaxSegmentsToMove() CoordinatorDynamicConfig dynamicConfig = CoordinatorDynamicConfig.builder() .withSmartSegmentLoading(false) - .withDecommissioningMaxPercentOfMaxSegmentsToMove(100) .withMaxSegmentsToMove(1).build(); DruidCoordinatorRuntimeParams params = defaultRuntimeParamsBuilder(holder1, holder2, holder3).withDynamicConfigs(dynamicConfig).build(); @@ -206,11 +143,8 @@ public void testMaxDecommissioningMaxPercentOfMaxSegmentsToMove() Assert.assertEquals(segment3.getDataSource(), movingSegment.getDataSource()); } - /** - * Should balance segments as usual (ignoring percent) with empty decommissioningNodes. - */ @Test - public void testMoveDecommissioningMaxPercentOfMaxSegmentsToMoveWithNoDecommissioning() + public void testMoveWithNoDecommissioning() { final ServerHolder serverHolder1 = createHolder(server1, segment1, segment2, segment3, segment4); final ServerHolder serverHolder2 = createHolder(server2); @@ -219,7 +153,6 @@ public void testMoveDecommissioningMaxPercentOfMaxSegmentsToMoveWithNoDecommissi CoordinatorDynamicConfig.builder() .withSmartSegmentLoading(false) .withMaxSegmentsToMove(4) - .withDecommissioningMaxPercentOfMaxSegmentsToMove(9) .build(); DruidCoordinatorRuntimeParams params = defaultRuntimeParamsBuilder(serverHolder1, serverHolder2) diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/RunRulesTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/RunRulesTest.java index a10a2731fc5b..a284a27c0071 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/RunRulesTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/RunRulesTest.java @@ -114,7 +114,7 @@ public void tearDown() /** * Nodes: * normal - 2 replicants - * maxNonPrimaryReplicantsToLoad - 10 + * replicationThrottleLimit - 10 * Expect only 34 segments to be loaded despite there being 48 primary + non-primary replicants to load! */ @Test @@ -152,7 +152,7 @@ public void testOneTierTwoReplicantsWithStrictReplicantLimit() .withDynamicConfigs( CoordinatorDynamicConfig .builder() - .withMaxNonPrimaryReplicantsToLoad(10) + .withReplicationThrottleLimit(10) .withSmartSegmentLoading(false) .build() ) @@ -171,7 +171,7 @@ public void testOneTierTwoReplicantsWithStrictReplicantLimit() * Nodes: * normal - 2 replicants * hot - 2 replicants - * maxNonPrimaryReplicantsToLoad - 48 + * replicationThrottleLimit - 48 * Expect only 72 segments to be loaded despite there being 96 primary + non-primary replicants to load! */ @Test @@ -216,7 +216,7 @@ public void testTwoTiersTwoReplicantsWithStrictReplicantLimit() .withBalancerStrategy(new CostBalancerStrategy(balancerExecutor)) .withDynamicConfigs( CoordinatorDynamicConfig.builder() - .withMaxNonPrimaryReplicantsToLoad(10) + .withReplicationThrottleLimit(10) .withSmartSegmentLoading(false) .build() ) @@ -225,7 +225,6 @@ public void testTwoTiersTwoReplicantsWithStrictReplicantLimit() CoordinatorRunStats stats = runDutyAndGetStats(params); - // maxNonPrimaryReplicantsToLoad takes effect on hot tier, but not normal tier Assert.assertEquals(10L, stats.getSegmentStat(Stats.Segments.ASSIGNED, "hot", DATASOURCE)); Assert.assertEquals(48L, stats.getSegmentStat(Stats.Segments.ASSIGNED, "normal", DATASOURCE)); diff --git a/server/src/test/java/org/apache/druid/server/coordinator/loading/ReplicationThrottlerTest.java b/server/src/test/java/org/apache/druid/server/coordinator/loading/ReplicationThrottlerTest.java index 4e1de51d36ab..b3911728bb94 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/loading/ReplicationThrottlerTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/loading/ReplicationThrottlerTest.java @@ -34,8 +34,7 @@ public void testTierDoesNotViolateThrottleLimit() final int replicationThrottleLimit = 10; ReplicationThrottler throttler = new ReplicationThrottler( ImmutableMap.of(), - replicationThrottleLimit, - 1000 + replicationThrottleLimit ); // Verify that both the tiers can be assigned replicas upto the limit @@ -54,8 +53,7 @@ public void testTierWithLoadingReplicasDoesNotViolateThrottleLimit() final int replicationThrottleLimit = 10; ReplicationThrottler throttler = new ReplicationThrottler( ImmutableMap.of(TIER_1, 10, TIER_2, 7), - replicationThrottleLimit, - 1000 + replicationThrottleLimit ); // T1 cannot be assigned any more replicas diff --git a/server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java b/server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java index 895d6b08b401..4c6b7cb5587f 100644 --- a/server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java +++ b/server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java @@ -56,10 +56,8 @@ public void testSerde() throws Exception + " \"maxKillTaskSlots\": 2,\n" + " \"maxSegmentsInNodeLoadingQueue\": 1,\n" + " \"decommissioningNodes\": [\"host1\", \"host2\"],\n" - + " \"decommissioningMaxPercentOfMaxSegmentsToMove\": 9,\n" + " \"pauseCoordination\": false,\n" - + " \"replicateAfterLoadTimeout\": false,\n" - + " \"maxNonPrimaryReplicantsToLoad\": 2147483647\n" + + " \"replicateAfterLoadTimeout\": false\n" + "}\n"; CoordinatorDynamicConfig actual = mapper.readValue( @@ -88,10 +86,8 @@ public void testSerde() throws Exception false, 1, decommissioning, - 9, false, - false, - Integer.MAX_VALUE + false ); actual = CoordinatorDynamicConfig.builder().withDecommissioningNodes(ImmutableSet.of("host1")).build(actual); @@ -110,13 +106,11 @@ public void testSerde() throws Exception false, 1, ImmutableSet.of("host1"), - 9, - false, false, - Integer.MAX_VALUE + false ); - actual = CoordinatorDynamicConfig.builder().withDecommissioningMaxPercentOfMaxSegmentsToMove(5).build(actual); + actual = CoordinatorDynamicConfig.builder().build(actual); assertConfig( actual, 1, @@ -132,10 +126,8 @@ public void testSerde() throws Exception false, 1, ImmutableSet.of("host1"), - 5, false, - false, - Integer.MAX_VALUE + false ); actual = CoordinatorDynamicConfig.builder().withPauseCoordination(true).build(actual); @@ -154,10 +146,8 @@ public void testSerde() throws Exception false, 1, ImmutableSet.of("host1"), - 5, true, - false, - Integer.MAX_VALUE + false ); actual = CoordinatorDynamicConfig.builder().withReplicateAfterLoadTimeout(true).build(actual); @@ -176,13 +166,11 @@ public void testSerde() throws Exception false, 1, ImmutableSet.of("host1"), - 5, - true, true, - Integer.MAX_VALUE + true ); - actual = CoordinatorDynamicConfig.builder().withMaxNonPrimaryReplicantsToLoad(10).build(actual); + actual = CoordinatorDynamicConfig.builder().build(actual); assertConfig( actual, 1, @@ -198,10 +186,8 @@ public void testSerde() throws Exception false, 1, ImmutableSet.of("host1"), - 5, true, - true, - 10 + true ); actual = CoordinatorDynamicConfig.builder().withKillTaskSlotRatio(1.0).build(actual); @@ -220,10 +206,8 @@ public void testSerde() throws Exception false, 1, ImmutableSet.of("host1"), - 5, true, - true, - 10 + true ); actual = CoordinatorDynamicConfig.builder().withMaxKillTaskSlots(5).build(actual); @@ -242,10 +226,8 @@ public void testSerde() throws Exception false, 1, ImmutableSet.of("host1"), - 5, - true, true, - 10 + true ); } @@ -279,10 +261,8 @@ public void testConstructorWithNullsShouldKillUnusedSegmentsInAllDataSources() null, null, ImmutableSet.of("host1"), - 5, true, true, - 10, false, false, null @@ -308,10 +288,8 @@ public void testConstructorWithSpecificDataSourcesToKillShouldNotKillUnusedSegme null, null, ImmutableSet.of("host1"), - 5, true, true, - 10, false, false, null @@ -361,10 +339,8 @@ public void testDecommissioningParametersBackwardCompatibility() throws Exceptio false, 1, decommissioning, - 0, - false, false, - Integer.MAX_VALUE + false ); actual = CoordinatorDynamicConfig.builder().withDecommissioningNodes(ImmutableSet.of("host1")).build(actual); @@ -383,13 +359,11 @@ public void testDecommissioningParametersBackwardCompatibility() throws Exceptio false, 1, ImmutableSet.of("host1"), - 0, - false, false, - Integer.MAX_VALUE + false ); - actual = CoordinatorDynamicConfig.builder().withDecommissioningMaxPercentOfMaxSegmentsToMove(5).build(actual); + actual = CoordinatorDynamicConfig.builder().build(actual); assertConfig( actual, 1, @@ -405,10 +379,8 @@ public void testDecommissioningParametersBackwardCompatibility() throws Exceptio false, 1, ImmutableSet.of("host1"), - 5, - false, false, - Integer.MAX_VALUE + false ); } @@ -451,24 +423,8 @@ public void testSerdeWithStringInKillDataSourceWhitelist() throws Exception false, 1, ImmutableSet.of(), - 0, false, - false, - Integer.MAX_VALUE - ); - } - - @Test - public void testSerdeHandlesInvalidDecommissioningPercentToMove() - { - final String errorMsg = "'decommissioningMaxPercentOfMaxSegmentsToMove' should be in range [0, 100]"; - assertThatDeserializationFailsWithMessage( - "{\"decommissioningMaxPercentOfMaxSegmentsToMove\": -1}", - errorMsg - ); - assertThatDeserializationFailsWithMessage( - "{\"decommissioningMaxPercentOfMaxSegmentsToMove\": 105}", - errorMsg + false ); } @@ -486,7 +442,6 @@ public void testHandleMissingPercentOfSegmentsToConsiderPerMove() throws Excepti + " \"killDataSourceWhitelist\": [\"test1\",\"test2\"],\n" + " \"maxSegmentsInNodeLoadingQueue\": 1,\n" + " \"decommissioningNodes\": [\"host1\", \"host2\"],\n" - + " \"decommissioningMaxPercentOfMaxSegmentsToMove\": 9,\n" + " \"pauseCoordination\": false\n" + "}\n"; CoordinatorDynamicConfig actual = mapper.readValue( @@ -512,10 +467,8 @@ public void testHandleMissingPercentOfSegmentsToConsiderPerMove() throws Excepti false, 1, decommissioning, - 9, - false, false, - Integer.MAX_VALUE + false ); } @@ -556,10 +509,8 @@ public void testSerdeWithKillAllDataSources() throws Exception true, 1, ImmutableSet.of(), - 0, - false, false, - Integer.MAX_VALUE + false ); // killAllDataSources is a config in versions 0.22.x and older and is no longer used. @@ -611,10 +562,8 @@ public void testDeserializeWithoutMaxSegmentsInNodeLoadingQueue() throws Excepti true, EXPECTED_DEFAULT_MAX_SEGMENTS_IN_NODE_LOADING_QUEUE, ImmutableSet.of(), - 0, false, - false, - Integer.MAX_VALUE + false ); } @@ -638,10 +587,8 @@ public void testBuilderDefaults() true, EXPECTED_DEFAULT_MAX_SEGMENTS_IN_NODE_LOADING_QUEUE, emptyList, - 70, - false, false, - Integer.MAX_VALUE + false ); } @@ -668,10 +615,8 @@ public void testBuilderWithDefaultSpecificDataSourcesToKillUnusedSegmentsInSpeci false, EXPECTED_DEFAULT_MAX_SEGMENTS_IN_NODE_LOADING_QUEUE, ImmutableSet.of(), - 70, false, - false, - Integer.MAX_VALUE + false ); } @@ -703,22 +648,11 @@ public void testUpdate() null, null, null, - null, - null, null ).build(current) ); } - @Test - public void testSerdeHandleInvalidMaxNonPrimaryReplicantsToLoad() - { - assertThatDeserializationFailsWithMessage( - "{\"maxNonPrimaryReplicantsToLoad\": -1}", - "maxNonPrimaryReplicantsToLoad must be greater than or equal to 0." - ); - } - private void assertThatDeserializationFailsWithMessage(String json, String message) { JsonMappingException e = Assert.assertThrows( @@ -759,10 +693,8 @@ private void assertConfig( boolean expectedKillUnusedSegmentsInAllDataSources, int expectedMaxSegmentsInNodeLoadingQueue, Set decommissioningNodes, - int decommissioningMaxPercentOfMaxSegmentsToMove, boolean pauseCoordination, - boolean replicateAfterLoadTimeout, - int maxNonPrimaryReplicantsToLoad + boolean replicateAfterLoadTimeout ) { Assert.assertEquals( @@ -784,13 +716,8 @@ private void assertConfig( Assert.assertEquals((int) expectedMaxKillTaskSlots, config.getMaxKillTaskSlots()); Assert.assertEquals(expectedMaxSegmentsInNodeLoadingQueue, config.getMaxSegmentsInNodeLoadingQueue()); Assert.assertEquals(decommissioningNodes, config.getDecommissioningNodes()); - Assert.assertEquals( - decommissioningMaxPercentOfMaxSegmentsToMove, - config.getDecommissioningMaxPercentOfMaxSegmentsToMove() - ); Assert.assertEquals(pauseCoordination, config.getPauseCoordination()); Assert.assertEquals(replicateAfterLoadTimeout, config.getReplicateAfterLoadTimeout()); - Assert.assertEquals(maxNonPrimaryReplicantsToLoad, config.getMaxNonPrimaryReplicantsToLoad()); } private static int getDefaultNumBalancerThreads() diff --git a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.mock.ts b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.mock.ts index 1951359a76fb..06224afb28be 100644 --- a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.mock.ts +++ b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.mock.ts @@ -32,10 +32,8 @@ export const DEFAULT_COORDINATOR_DYNAMIC_CONFIG: CoordinatorDynamicConfig = { killPendingSegmentsSkipList: [], maxSegmentsInNodeLoadingQueue: 500, decommissioningNodes: [], - decommissioningMaxPercentOfMaxSegmentsToMove: 70, pauseCoordination: false, replicateAfterLoadTimeout: false, - maxNonPrimaryReplicantsToLoad: 2147483647, useRoundRobinSegmentAssignment: true, smartSegmentLoading: true, debugDimensions: null, diff --git a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx index e5865949a8b7..eaf29a51e2e2 100644 --- a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx +++ b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx @@ -38,9 +38,7 @@ export interface CoordinatorDynamicConfig { replicantLifetime?: number; replicationThrottleLimit?: number; decommissioningNodes?: string[]; - decommissioningMaxPercentOfMaxSegmentsToMove?: number; pauseCoordination?: boolean; - maxNonPrimaryReplicantsToLoad?: number; replicateAfterLoadTimeout?: boolean; useRoundRobinSegmentAssignment?: boolean; smartSegmentLoading?: boolean; @@ -146,50 +144,6 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field[ ), }, - { - name: 'maxNonPrimaryReplicantsToLoad', - type: 'number', - defaultValue: 2147483647, - defined: cdc => (cdc.smartSegmentLoading === false ? true : undefined), - info: ( - <> - The maximum number of non-primary replicants to load in a single Coordinator cycle. Once - this limit is hit, only primary replicants will be loaded for the remainder of the cycle. - Tuning this value lower can help reduce the delay in loading primary segments when the - cluster has a very large number of non-primary replicants to load (such as when a single - historical drops out of the cluster leaving many under-replicated segments). - - ), - }, - { - name: 'decommissioningMaxPercentOfMaxSegmentsToMove', - type: 'number', - defaultValue: 70, - defined: cdc => (cdc.smartSegmentLoading === false ? true : undefined), - info: ( - <> -

- Upper limit of segments the Coordinator can move from decommissioning servers to active - non-decommissioning servers during a single run. This value is relative to the total - maximum number of segments that can be moved at any given time based upon the value of - maxSegmentsToMove. -

-

- If decommissioningMaxPercentOfMaxSegmentsToMove is 0, the Coordinator does - not move segments to decommissioning servers, effectively putting them in a type of - "maintenance" mode. In this case, decommissioning servers do not participate in - balancing or assignment by load rules. The Coordinator still considers segments on - decommissioning servers as candidates to replicate on active servers. -

-

- Decommissioning can stall if there are no available active servers to move the segments - to. You can use the maximum percent of decommissioning segment movements to prioritize - balancing or to decrease commissioning time to prevent active servers from being - overloaded. The value must be between 0 and 100. -

- - ), - }, // End "smart" segment loading section @@ -202,7 +156,7 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field[ List of historical services to 'decommission'. Coordinator will not assign new segments to 'decommissioning' services, and segments will be moved away from them to be placed on non-decommissioning services at the maximum rate specified by{' '} - decommissioningMaxPercentOfMaxSegmentsToMove. + maxSegmentsToMove. ), }, From d4e972e1e4b798d7e057e910deea4d192fe57952 Mon Sep 17 00:00:00 2001 From: panhongan Date: Mon, 4 Sep 2023 15:48:55 +0800 Subject: [PATCH 017/258] Add checking for new checkpoint (#14353) Check that a checkpoint is non-empty before adding it to the checkpoint sequence in a SeekableStreamSupervisor --- .../supervisor/SeekableStreamSupervisor.java | 9 +- .../SeekableStreamSupervisorStateTest.java | 263 ++++++++++++++++++ 2 files changed, 270 insertions(+), 2 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index f79785b82370..2f6cb008b842 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -40,6 +40,7 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectLinkedOpenHashMap; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.druid.common.guava.FutureUtils; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.error.DruidException; @@ -691,8 +692,12 @@ public void handle() throws ExecutionException, InterruptedException return; } final Map newCheckpoint = checkpointTaskGroup(taskGroup, false).get(); - taskGroup.addNewCheckpoint(newCheckpoint); - log.info("Handled checkpoint notice, new checkpoint is [%s] for taskGroup [%s]", newCheckpoint, taskGroupId); + if (MapUtils.isNotEmpty(newCheckpoint)) { + taskGroup.addNewCheckpoint(newCheckpoint); + log.info("Handled checkpoint notice, new checkpoint is [%s] for taskGroup [%s]", newCheckpoint, taskGroupId); + } else { + log.warn("New checkpoint is null for taskGroup [%s]", taskGroupId); + } } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java index 7e8afdf6bc5d..819a6baacd8e 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java @@ -19,12 +19,16 @@ package org.apache.druid.indexing.seekablestream.supervisor; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import com.google.errorprone.annotations.concurrent.GuardedBy; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionSchema; @@ -34,6 +38,8 @@ import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.error.DruidException; import org.apache.druid.error.DruidExceptionMatcher; +import org.apache.druid.indexer.TaskLocation; +import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.TaskToolbox; import org.apache.druid.indexing.common.TestUtils; import org.apache.druid.indexing.common.task.Task; @@ -44,6 +50,7 @@ import org.apache.druid.indexing.overlord.TaskQueue; import org.apache.druid.indexing.overlord.TaskRunner; import org.apache.druid.indexing.overlord.TaskRunnerListener; +import org.apache.druid.indexing.overlord.TaskRunnerWorkItem; import org.apache.druid.indexing.overlord.TaskStorage; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManager; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManager.BasicState; @@ -57,6 +64,7 @@ import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskIOConfig; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskTuningConfig; +import org.apache.druid.indexing.seekablestream.SeekableStreamSequenceNumbers; import org.apache.druid.indexing.seekablestream.SeekableStreamStartSequenceNumbers; import org.apache.druid.indexing.seekablestream.TestSeekableStreamDataSourceMetadata; import org.apache.druid.indexing.seekablestream.common.OrderedSequenceNumber; @@ -98,6 +106,7 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -771,6 +780,169 @@ public void testStoppingGracefully() verifyAll(); } + @Test(timeout = 60_000L) + public void testCheckpointForActiveTaskGroup() throws InterruptedException, JsonProcessingException + { + DateTime startTime = DateTimes.nowUtc(); + SeekableStreamSupervisorIOConfig ioConfig = new SeekableStreamSupervisorIOConfig( + STREAM, + new JsonInputFormat(new JSONPathSpec(true, ImmutableList.of()), ImmutableMap.of(), false, false, false), + 1, + 1, + new Period("PT1H"), + new Period("PT1S"), + new Period("PT30S"), + false, + new Period("PT30M"), + null, + null, + null, + null, + new IdleConfig(true, 200L), + null + ) {}; + + EasyMock.reset(spec); + EasyMock.expect(spec.isSuspended()).andReturn(false).anyTimes(); + EasyMock.expect(spec.getDataSchema()).andReturn(getDataSchema()).anyTimes(); + EasyMock.expect(spec.getIoConfig()).andReturn(ioConfig).anyTimes(); + EasyMock.expect(spec.getTuningConfig()).andReturn(getTuningConfig()).anyTimes(); + EasyMock.expect(spec.getEmitter()).andReturn(emitter).anyTimes(); + EasyMock.expect(spec.getMonitorSchedulerConfig()).andReturn(new DruidMonitorSchedulerConfig() { + @Override + public Duration getEmissionDuration() + { + return new Period("PT2S").toStandardDuration(); + } + }).anyTimes(); + EasyMock.expect(spec.getType()).andReturn("stream").anyTimes(); + EasyMock.expect(spec.getSupervisorStateManagerConfig()).andReturn(supervisorConfig).anyTimes(); + EasyMock.expect(spec.getContextValue("tags")).andReturn("").anyTimes(); + EasyMock.expect(recordSupplier.getPartitionIds(STREAM)).andReturn(ImmutableSet.of(SHARD_ID)).anyTimes(); + EasyMock.expect(taskQueue.add(EasyMock.anyObject())).andReturn(true).anyTimes(); + + SeekableStreamIndexTaskIOConfig taskIoConfig = createTaskIoConfigExt( + 0, + Collections.singletonMap("0", "10"), + Collections.singletonMap("0", "20"), + "test", + startTime, + null, + Collections.emptySet(), + ioConfig + ); + + SeekableStreamIndexTaskTuningConfig taskTuningConfig = getTuningConfig().convertToTaskTuningConfig(); + + TreeMap> sequenceOffsets = new TreeMap<>(); + sequenceOffsets.put(0, ImmutableMap.of("0", 10L, "1", 20L)); + + Map context = new HashMap<>(); + context.put("checkpoints", new ObjectMapper().writeValueAsString(sequenceOffsets)); + + SeekableStreamIndexTask id1 = new TestSeekableStreamIndexTask( + "id1", + null, + getDataSchema(), + taskTuningConfig, + taskIoConfig, + context, + "0" + ); + + SeekableStreamIndexTask id2 = new TestSeekableStreamIndexTask( + "id2", + null, + getDataSchema(), + taskTuningConfig, + taskIoConfig, + context, + "0" + ); + + final TaskLocation location1 = TaskLocation.create("testHost", 1234, -1); + final TaskLocation location2 = TaskLocation.create("testHost2", 145, -1); + + Collection workItems = new ArrayList<>(); + workItems.add(new TestTaskRunnerWorkItem(id1, null, location1)); + workItems.add(new TestTaskRunnerWorkItem(id2, null, location2)); + + EasyMock.expect(taskRunner.getRunningTasks()).andReturn(workItems).anyTimes(); + EasyMock.expect(taskStorage.getActiveTasksByDatasource(DATASOURCE)) + .andReturn(ImmutableList.of(id1, id2)) + .anyTimes(); + EasyMock.expect(taskStorage.getStatus("id1")).andReturn(Optional.of(TaskStatus.running("id1"))).anyTimes(); + EasyMock.expect(taskStorage.getStatus("id2")).andReturn(Optional.of(TaskStatus.running("id2"))).anyTimes(); + EasyMock.expect(taskStorage.getTask("id1")).andReturn(Optional.of(id1)).anyTimes(); + EasyMock.expect(taskStorage.getTask("id2")).andReturn(Optional.of(id2)).anyTimes(); + + EasyMock.reset(indexerMetadataStorageCoordinator); + EasyMock.expect( + indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(new TestSeekableStreamDataSourceMetadata(null) + ).anyTimes(); + EasyMock.expect(indexTaskClient.getStatusAsync("id1")).andReturn(Futures.immediateFuture(SeekableStreamIndexTaskRunner.Status.READING)).anyTimes(); + EasyMock.expect(indexTaskClient.getStatusAsync("id2")).andReturn(Futures.immediateFuture(SeekableStreamIndexTaskRunner.Status.READING)).anyTimes(); + + EasyMock.expect(indexTaskClient.getStartTimeAsync("id1")).andReturn(Futures.immediateFuture(startTime)).anyTimes(); + EasyMock.expect(indexTaskClient.getStartTimeAsync("id2")).andReturn(Futures.immediateFuture(startTime)).anyTimes(); + + ImmutableMap partitionOffset = ImmutableMap.of("0", "10"); + final TreeMap> checkpoints = new TreeMap<>(); + checkpoints.put(0, partitionOffset); + + EasyMock.expect(indexTaskClient.getCheckpointsAsync(EasyMock.contains("id1"), EasyMock.anyBoolean())) + .andReturn(Futures.immediateFuture(checkpoints)) + .anyTimes(); + EasyMock.expect(indexTaskClient.getCheckpointsAsync(EasyMock.contains("id2"), EasyMock.anyBoolean())) + .andReturn(Futures.immediateFuture(checkpoints)) + .anyTimes(); + EasyMock.expect(indexTaskClient.pauseAsync("id1")) + .andReturn(Futures.immediateFuture(partitionOffset)) + .anyTimes(); + EasyMock.expect(indexTaskClient.pauseAsync("id2")) + .andReturn(Futures.immediateFuture(partitionOffset)) + .anyTimes(); + EasyMock.expect(indexTaskClient.setEndOffsetsAsync("id1", partitionOffset, false)) + .andReturn(Futures.immediateFuture(true)) + .anyTimes(); + EasyMock.expect(indexTaskClient.setEndOffsetsAsync("id2", partitionOffset, false)) + .andReturn(Futures.immediateFuture(true)) + .anyTimes(); + EasyMock.expect(indexTaskClient.resumeAsync("id1")) + .andReturn(Futures.immediateFuture(true)) + .anyTimes(); + EasyMock.expect(indexTaskClient.resumeAsync("id2")) + .andReturn(Futures.immediateFuture(true)) + .anyTimes(); + EasyMock.expect(indexTaskClient.stopAsync("id1", false)) + .andReturn(Futures.immediateFuture(true)) + .anyTimes(); + EasyMock.expect(indexTaskClient.stopAsync("id2", false)) + .andReturn(Futures.immediateFuture(true)) + .anyTimes(); + + replayAll(); + + SeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(); + + supervisor.start(); + supervisor.runInternal(); + + supervisor.checkpoint( + 0, + new TestSeekableStreamDataSourceMetadata( + new SeekableStreamStartSequenceNumbers<>(STREAM, checkpoints.get(0), ImmutableSet.of()) + ) + ); + + while (supervisor.getNoticesQueueSize() > 0) { + Thread.sleep(100); + } + + verifyAll(); + + Assert.assertTrue(supervisor.getNoticesQueueSize() == 0); + } @Test public void testEmitBothLag() throws Exception @@ -2208,4 +2380,95 @@ public List getEvents() } } } + + private static class TestTaskRunnerWorkItem extends TaskRunnerWorkItem + { + private final String taskType; + private final TaskLocation location; + private final String dataSource; + + TestTaskRunnerWorkItem(Task task, ListenableFuture result, TaskLocation location) + { + super(task.getId(), result); + this.taskType = task.getType(); + this.location = location; + this.dataSource = task.getDataSource(); + } + + @Override + public TaskLocation getLocation() + { + return location; + } + + @Override + public String getTaskType() + { + return taskType; + } + + @Override + public String getDataSource() + { + return dataSource; + } + } + + private static class TestSeekableStreamDataSourceMetadata extends SeekableStreamDataSourceMetadata + { + + @JsonCreator + public TestSeekableStreamDataSourceMetadata( + @JsonProperty("partitions") SeekableStreamSequenceNumbers partitions + ) + { + super(partitions); + } + + @Override + public DataSourceMetadata asStartMetadata() + { + final SeekableStreamSequenceNumbers sequenceNumbers = getSeekableStreamSequenceNumbers(); + if (sequenceNumbers instanceof SeekableStreamEndSequenceNumbers) { + return createConcreteDataSourceMetaData( + ((SeekableStreamEndSequenceNumbers) sequenceNumbers).asStartPartitions(true) + ); + } else { + return this; + } + } + + @Override + protected SeekableStreamDataSourceMetadata createConcreteDataSourceMetaData( + SeekableStreamSequenceNumbers seekableStreamSequenceNumbers + ) + { + return new TestSeekableStreamDataSourceMetadata(seekableStreamSequenceNumbers); + } + } + + private static SeekableStreamIndexTaskIOConfig createTaskIoConfigExt( + int groupId, + Map startPartitions, + Map endPartitions, + String baseSequenceName, + DateTime minimumMessageTime, + DateTime maximumMessageTime, + Set exclusiveStartSequenceNumberPartitions, + SeekableStreamSupervisorIOConfig ioConfig + ) + { + return new SeekableStreamIndexTaskIOConfig( + groupId, + baseSequenceName, + new SeekableStreamStartSequenceNumbers<>(STREAM, startPartitions, exclusiveStartSequenceNumberPartitions), + new SeekableStreamEndSequenceNumbers<>(STREAM, endPartitions), + true, + minimumMessageTime, + maximumMessageTime, + ioConfig.getInputFormat() + ) + { + }; + } } From 289ee1e0114efc08b86c1e63b6d195897aa5baa7 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Tue, 5 Sep 2023 09:15:41 +0530 Subject: [PATCH 018/258] Refactor: Cleanup NoopTask (#14938) Changes: - Simplify static `create` methods for `NoopTask` - Remove `FirehoseFactory`, `IsReadyResult`, `readyTime` from `NoopTask` as these fields were not being used anywhere - Update tests --- .../overlord/KubernetesPeonLifecycleTest.java | 4 +- .../overlord/KubernetesTaskRunnerTest.java | 12 +- .../k8s/overlord/KubernetesWorkItemTest.java | 2 +- .../k8s/overlord/common/K8sTestUtils.java | 8 ++ .../taskadapter/K8sTaskAdapterTest.java | 3 +- .../MultiContainerTaskAdapterTest.java | 6 +- .../PodTemplateTaskAdapterTest.java | 41 +----- .../SingleContainerTaskAdapterTest.java | 2 +- .../src/test/resources/expectedNoopJob.yaml | 2 +- .../resources/expectedNoopJobLongIds.yaml | 2 +- .../resources/expectedNoopJobTlsEnabled.yaml | 2 +- .../druid/indexing/common/task/NoopTask.java | 102 ++------------ .../actions/RemoteTaskActionClientTest.java | 4 +- .../actions/SegmentAllocationQueueTest.java | 2 +- .../common/task/AbstractTaskTest.java | 6 +- .../indexing/common/task/NoopTaskTest.java | 2 +- .../ParallelIndexPhaseRunnerTest.java | 2 - .../task/batch/parallel/TaskMonitorTest.java | 2 +- .../overlord/HeapMemoryTaskStorageTest.java | 8 +- .../IndexerMetadataStorageAdapterTest.java | 8 +- .../SingleTaskBackgroundRunnerTest.java | 22 +-- .../indexing/overlord/TaskLifecycleTest.java | 133 ++++++------------ .../overlord/TaskLockBoxConcurrencyTest.java | 4 +- .../indexing/overlord/TaskLockboxTest.java | 62 ++++---- .../indexing/overlord/TaskQueueScaleTest.java | 29 +--- .../druid/indexing/overlord/ZkWorkerTest.java | 6 +- .../hrtr/HttpRemoteTaskRunnerTest.java | 54 +++---- .../overlord/hrtr/WorkerHolderTest.java | 8 +- .../overlord/http/OverlordResourceTest.java | 23 +-- .../indexing/overlord/http/OverlordTest.java | 57 ++++---- ...nWithAffinityWorkerSelectStrategyTest.java | 13 +- ...hCategorySpecWorkerSelectStrategyTest.java | 2 +- ...lDistributionWorkerSelectStrategyTest.java | 45 +----- ...yWithAffinityWorkerSelectStrategyTest.java | 13 +- ...hCategorySpecWorkerSelectStrategyTest.java | 2 +- .../worker/WorkerTaskManagerTest.java | 6 +- 36 files changed, 234 insertions(+), 465 deletions(-) diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleTest.java index 084d1db62d2b..4c46c278e260 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleTest.java @@ -30,10 +30,10 @@ import org.apache.druid.indexer.TaskLocation; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.TestUtils; -import org.apache.druid.indexing.common.task.NoopTask; import org.apache.druid.indexing.common.task.Task; import org.apache.druid.k8s.overlord.common.JobResponse; import org.apache.druid.k8s.overlord.common.K8sTaskId; +import org.apache.druid.k8s.overlord.common.K8sTestUtils; import org.apache.druid.k8s.overlord.common.KubernetesPeonClient; import org.apache.druid.k8s.overlord.common.PeonPhase; import org.apache.druid.tasklogs.TaskLogs; @@ -73,7 +73,7 @@ public class KubernetesPeonLifecycleTest extends EasyMockSupport public void setup() { mapper = new TestUtils().getTestObjectMapper(); - task = NoopTask.create(ID, 0); + task = K8sTestUtils.createTask(ID, 0); k8sTaskId = new K8sTaskId(task); EasyMock.expect(logWatch.getOutput()).andReturn(IOUtils.toInputStream("", StandardCharsets.UTF_8)).anyTimes(); } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerTest.java index bee3a533c7b8..11e7a75af695 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerTest.java @@ -29,7 +29,6 @@ import org.apache.druid.indexer.RunnerTaskState; import org.apache.druid.indexer.TaskLocation; import org.apache.druid.indexer.TaskStatus; -import org.apache.druid.indexing.common.task.NoopTask; import org.apache.druid.indexing.common.task.Task; import org.apache.druid.indexing.overlord.TaskRunnerWorkItem; import org.apache.druid.java.util.common.Pair; @@ -38,6 +37,7 @@ import org.apache.druid.java.util.http.client.HttpClient; import org.apache.druid.java.util.http.client.Request; import org.apache.druid.java.util.http.client.response.InputStreamResponseHandler; +import org.apache.druid.k8s.overlord.common.K8sTestUtils; import org.apache.druid.k8s.overlord.common.KubernetesPeonClient; import org.apache.druid.k8s.overlord.taskadapter.TaskAdapter; import org.easymock.EasyMock; @@ -89,7 +89,7 @@ public void setup() .withCapacity(1) .build(); - task = NoopTask.create(ID, 0); + task = K8sTestUtils.createTask(ID, 0); runner = new KubernetesTaskRunner( taskAdapter, @@ -367,7 +367,7 @@ public void test_getKnownTasks() @Test public void test_getRunningTasks() { - Task pendingTask = NoopTask.create("pending-id", 0); + Task pendingTask = K8sTestUtils.createTask("pending-id", 0); KubernetesWorkItem pendingWorkItem = new KubernetesWorkItem(pendingTask, null) { @Override protected RunnerTaskState getRunnerTaskState() @@ -377,7 +377,7 @@ protected RunnerTaskState getRunnerTaskState() }; runner.tasks.put(pendingTask.getId(), pendingWorkItem); - Task runningTask = NoopTask.create("running-id", 0); + Task runningTask = K8sTestUtils.createTask("running-id", 0); KubernetesWorkItem runningWorkItem = new KubernetesWorkItem(runningTask, null) { @Override protected RunnerTaskState getRunnerTaskState() @@ -396,7 +396,7 @@ protected RunnerTaskState getRunnerTaskState() @Test public void test_getPendingTasks() { - Task pendingTask = NoopTask.create("pending-id", 0); + Task pendingTask = K8sTestUtils.createTask("pending-id", 0); KubernetesWorkItem pendingWorkItem = new KubernetesWorkItem(pendingTask, null) { @Override protected RunnerTaskState getRunnerTaskState() @@ -406,7 +406,7 @@ protected RunnerTaskState getRunnerTaskState() }; runner.tasks.put(pendingTask.getId(), pendingWorkItem); - Task runningTask = NoopTask.create("running-id", 0); + Task runningTask = K8sTestUtils.createTask("running-id", 0); KubernetesWorkItem runningWorkItem = new KubernetesWorkItem(runningTask, null) { @Override protected RunnerTaskState getRunnerTaskState() diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesWorkItemTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesWorkItemTest.java index d5cf2ea7252d..7d17193b1714 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesWorkItemTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesWorkItemTest.java @@ -44,7 +44,7 @@ public class KubernetesWorkItemTest extends EasyMockSupport @Before public void setup() { - task = NoopTask.create("id", 0); + task = NoopTask.create(); workItem = new KubernetesWorkItem(task, null); } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java index c0affe9573c3..ab5379ffa6fa 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java @@ -29,7 +29,9 @@ import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; import org.apache.druid.indexing.common.task.IndexTask; +import org.apache.druid.indexing.common.task.NoopTask; import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.common.task.Tasks; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -39,6 +41,7 @@ import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; import java.io.File; +import java.util.Collections; public class K8sTestUtils @@ -120,4 +123,9 @@ public static T fileToResource(String contents, Class type) type ); } + + public static NoopTask createTask(String id, int priority) + { + return new NoopTask(id, null, null, 0, 0, Collections.singletonMap(Tasks.PRIORITY_KEY, priority)); + } } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapterTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapterTest.java index 519e7177cbe5..19701e4f26e1 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapterTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapterTest.java @@ -354,7 +354,7 @@ void testEphemeralStorageIsRespected() throws IOException node, jsonMapper ); - NoopTask task = NoopTask.create("id", 1); + NoopTask task = K8sTestUtils.createTask("id", 1); Job actual = adapter.createJobFromPodSpec( pod.getSpec(), task, @@ -426,4 +426,5 @@ void testEphemeralStorage() ); assertEquals(1, additionalProperties.getAdditionalProperties().size()); } + } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/MultiContainerTaskAdapterTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/MultiContainerTaskAdapterTest.java index aa93856ded86..ac6e32d140ee 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/MultiContainerTaskAdapterTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/MultiContainerTaskAdapterTest.java @@ -100,7 +100,7 @@ public void testMultiContainerSupport() throws IOException druidNode, jsonMapper ); - NoopTask task = NoopTask.create("id", 1); + NoopTask task = K8sTestUtils.createTask("id", 1); Job actual = adapter.createJobFromPodSpec( pod.getSpec(), task, @@ -148,7 +148,7 @@ public void testMultiContainerSupportWithNamedContainer() throws IOException druidNode, jsonMapper ); - NoopTask task = NoopTask.create("id", 1); + NoopTask task = K8sTestUtils.createTask("id", 1); PodSpec spec = pod.getSpec(); K8sTaskAdapter.massageSpec(spec, "primary"); Job actual = adapter.createJobFromPodSpec( @@ -197,7 +197,7 @@ public void testOverridingPeonMonitors() throws IOException startupLoggingConfig, druidNode, jsonMapper); - NoopTask task = NoopTask.create("id", 1); + NoopTask task = K8sTestUtils.createTask("id", 1); PodSpec spec = pod.getSpec(); K8sTaskAdapter.massageSpec(spec, config.getPrimaryContainerName()); Job actual = adapter.createJobFromPodSpec( diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/PodTemplateTaskAdapterTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/PodTemplateTaskAdapterTest.java index 992a071e22a8..dd4eadfea299 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/PodTemplateTaskAdapterTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/PodTemplateTaskAdapterTest.java @@ -130,16 +130,7 @@ public void test_fromTask_withBasePodTemplateInRuntimeProperites() throws IOExce props ); - Task task = new NoopTask( - "id", - "id", - "datasource", - 0, - 0, - null, - null, - null - ); + Task task = new NoopTask("id", "id", "datasource", 0, 0, null); Job actual = adapter.fromTask(task); Job expected = K8sTestUtils.fileToResource("expectedNoopJob.yaml", Job.class); @@ -171,17 +162,7 @@ public void test_fromTask_withBasePodTemplateInRuntimeProperites_andTlsEnabled() props ); - Task task = new NoopTask( - "id", - "id", - "datasource", - 0, - 0, - null, - null, - null - ); - + Task task = new NoopTask("id", "id", "datasource", 0, 0, null); Job actual = adapter.fromTask(task); Job expected = K8sTestUtils.fileToResource("expectedNoopJobTlsEnabled.yaml", Job.class); @@ -226,17 +207,7 @@ public void test_fromTask_withNoopPodTemplateInRuntimeProperites() throws IOExce props ); - Task task = new NoopTask( - "id", - "id", - "datasource", - 0, - 0, - null, - null, - null - ); - + Task task = new NoopTask("id", "id", "datasource", 0, 0, null); Job actual = adapter.fromTask(task); Job expected = K8sTestUtils.fileToResource("expectedNoopJob.yaml", Job.class); @@ -316,7 +287,7 @@ public void test_fromTask() throws IOException Job job = K8sTestUtils.fileToResource("baseJob.yaml", Job.class); Task actual = adapter.toTask(job); - Task expected = NoopTask.create("id", 1); + Task expected = K8sTestUtils.createTask("id", 1); Assertions.assertEquals(expected, actual); } @@ -345,8 +316,6 @@ public void test_fromTask_withRealIds() throws IOException "data_source", 0, 0, - null, - null, null ); @@ -406,7 +375,7 @@ private void assertJobSpecsEqual(Job actual, Job expected) throws IOException expectedAnnotations.remove(DruidK8sConstants.TASK); expected.getSpec().getTemplate().getMetadata().setAnnotations(expectedAnnotations); - Assertions.assertEquals(actual, expected); + Assertions.assertEquals(expected, actual); Assertions.assertEquals( Base64Compression.decompressBase64(actualTaskAnnotation), Base64Compression.decompressBase64(expectedTaskAnnotation) diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/SingleContainerTaskAdapterTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/SingleContainerTaskAdapterTest.java index 3e51c5a7acbc..10e129a9c2b5 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/SingleContainerTaskAdapterTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/SingleContainerTaskAdapterTest.java @@ -98,7 +98,7 @@ public void testSingleContainerSupport() throws IOException druidNode, jsonMapper ); - NoopTask task = NoopTask.create("id", 1); + NoopTask task = K8sTestUtils.createTask("id", 1); Job actual = adapter.createJobFromPodSpec( pod.getSpec(), task, diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJob.yaml b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJob.yaml index 7c048ea6df48..0998d592fee4 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJob.yaml +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJob.yaml @@ -26,7 +26,7 @@ spec: druid.task.group.id: "id" druid.task.datasource: "datasource" annotations: - task: "H4sIAAAAAAAAAEVOuQ4CIRD9l6kpVhObbY0xtrs2liOMSoKAHEZC+HeHrEczmXfmVUjFE4xgnfMgQCv++Qi4Bpf94QcVJpxdDrKbO4gLEBCyPeo70+vNMHBDnAhVWag/nihmkzh72s0cuuhANxfZYrMxAqSziV6s18aN9CkfK+ATtcGzNjqVfZ/0HRTokblEbdGjZBHGVWtvT9WXlc8AAAA=" + task: "H4sIAAAAAAAAAD2MvQ4CIRCE32VqijsTG1qLi7W+wArEbHICrmC8EN7dJf40k/lmJtNQthxgEVPKMGCvXsXgKqnm4x89FTqlKm6MBzw+YCA1nvmm8W4/TQYuxRJeBbZ17cJ3ZhvoSbzShVcu2zLOf9cS7pUl+ANlclrCzr2/AQUK0FqZAAAA" tls.enabled: "false" task.id: "id" task.type: "noop" diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobLongIds.yaml b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobLongIds.yaml index b781070892c9..bb8f64c5e5ed 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobLongIds.yaml +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobLongIds.yaml @@ -26,7 +26,7 @@ spec: druid.task.group.id: "apiissuedkillwikipedia3omjobnbc10000101t000000000z20230514t0000" druid.task.datasource: "data_source" annotations: - task: "H4sIAAAAAAAAAMVQu07EMBD8F9fJae0QIblFCNHepeEay4kNLOezjR9AFOXf2XBIVNQnbbEzs6/ZhZU5WiaZDyGyhqGhXEdsMedqjTqhc+oTTxitQd2pcH4Lox8nxQGgBU4xAMif2BF1VAJE10Lf8pv/hH7gtxI6CXwnBBxp60sKNT5eZbXRRR9CTdP2hA2ofEENS9UPeCZe9AD0mry32swX6g/vba6uUPPT/YGanjHZ15CpxFfnGjYFX+wX6ctKE+3vcLkw/aHR6REdlvlh838N98m+VzrY3OmoJzqESb6u3yiWc3MUAgAA" + task: "H4sIAAAAAAAAAMVQPa/CMAz8L55b5KRUSFkZnt5Mpy6R20RPhtKENOFDVf87KbC+GekG352l83mG+PAWFIzOeSiATZ7Jc8nTlKzRJx4GfeMTe2uYKu3OR9eNXa8FIpYoMhpE9cImS62WKKsS61Js/zPqRuwUVgrFRkpsc+pfcMn/fiXaUKSDS6Ffn7ASPb1ZASGNDZ+zLmvEAno3RnuPoOYle/azpmagK/FAHQ8cHz9rk2/0CPaSOFizJ099PgSUWJYnqMIU2d4BAAA=" tls.enabled: "false" task.id: "api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z" task.type: "noop" diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobTlsEnabled.yaml b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobTlsEnabled.yaml index 0eb8f8b02f08..e6762af63cf0 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobTlsEnabled.yaml +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobTlsEnabled.yaml @@ -26,7 +26,7 @@ spec: druid.task.group.id: "id" druid.task.datasource: "datasource" annotations: - task: "H4sIAAAAAAAAAEVOuQ4CIRD9l6kpVhObbY0xtrs2liOMSoKAHEZC+HeHrEczmXfmVUjFE4xgnfMgQCv++Qi4Bpf94QcVJpxdDrKbO4gLEBCyPeo70+vNMHBDnAhVWag/nihmkzh72s0cuuhANxfZYrMxAqSziV6s18aN9CkfK+ATtcGzNjqVfZ/0HRTokblEbdGjZBHGVWtvT9WXlc8AAAA=" + task: "H4sIAAAAAAAAAD2MvQ4CIRCE32VqijsTG1qLi7W+wArEbHICrmC8EN7dJf40k/lmJtNQthxgEVPKMGCvXsXgKqnm4x89FTqlKm6MBzw+YCA1nvmm8W4/TQYuxRJeBbZ17cJ3ZhvoSbzShVcu2zLOf9cS7pUl+ANlclrCzr2/AQUK0FqZAAAA" tls.enabled: "true" task.id: "id" task.type: "noop" diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/NoopTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/NoopTask.java index e4c112100e45..678d8cafd01a 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/NoopTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/NoopTask.java @@ -22,18 +22,13 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; -import org.apache.druid.data.input.Firehose; -import org.apache.druid.data.input.FirehoseFactory; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.TaskToolbox; import org.apache.druid.indexing.common.actions.TaskActionClient; import org.apache.druid.indexing.common.config.TaskConfig; import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.server.security.ResourceAction; import javax.annotation.Nonnull; @@ -46,39 +41,18 @@ */ public class NoopTask extends AbstractTask { - private static final Logger log = new Logger(NoopTask.class); private static final int DEFAULT_RUN_TIME = 2500; - private static final int DEFAULT_IS_READY_TIME = 0; - private static final IsReadyResult DEFAULT_IS_READY_RESULT = IsReadyResult.YES; - - enum IsReadyResult - { - YES, - NO, - EXCEPTION - } @JsonIgnore private final long runTime; - @JsonIgnore - private final long isReadyTime; - - @JsonIgnore - private final IsReadyResult isReadyResult; - - @JsonIgnore - private final FirehoseFactory firehoseFactory; - @JsonCreator public NoopTask( @JsonProperty("id") String id, @JsonProperty("groupId") String groupId, @JsonProperty("dataSource") String dataSource, - @JsonProperty("runTime") long runTime, + @JsonProperty("runTime") long runTimeMillis, @JsonProperty("isReadyTime") long isReadyTime, - @JsonProperty("isReadyResult") String isReadyResult, - @JsonProperty("firehose") FirehoseFactory firehoseFactory, @JsonProperty("context") Map context ) { @@ -90,12 +64,7 @@ public NoopTask( context ); - this.runTime = (runTime == 0) ? DEFAULT_RUN_TIME : runTime; - this.isReadyTime = (isReadyTime == 0) ? DEFAULT_IS_READY_TIME : isReadyTime; - this.isReadyResult = (isReadyResult == null) - ? DEFAULT_IS_READY_RESULT - : IsReadyResult.valueOf(StringUtils.toUpperCase(isReadyResult)); - this.firehoseFactory = firehoseFactory; + this.runTime = (runTimeMillis == 0) ? DEFAULT_RUN_TIME : runTimeMillis; } @Override @@ -118,37 +87,10 @@ public long getRunTime() return runTime; } - @JsonProperty - public long getIsReadyTime() - { - return isReadyTime; - } - - @JsonProperty - public IsReadyResult getIsReadyResult() - { - return isReadyResult; - } - - @JsonProperty("firehose") - public FirehoseFactory getFirehoseFactory() - { - return firehoseFactory; - } - @Override public boolean isReady(TaskActionClient taskActionClient) { - switch (isReadyResult) { - case YES: - return true; - case NO: - return false; - case EXCEPTION: - throw new ISE("Not ready. Never will be ready. Go away!"); - default: - throw new AssertionError("#notreached"); - } + return true; } @Override @@ -159,17 +101,8 @@ public void stopGracefully(TaskConfig taskConfig) @Override public TaskStatus runTask(TaskToolbox toolbox) throws Exception { - if (firehoseFactory != null) { - log.info("Connecting firehose"); - } - try (Firehose firehose = firehoseFactory != null ? firehoseFactory.connect(null, null) : null) { - - log.info("Running noop task[%s]", getId()); - log.info("Sleeping for %,d millis.", runTime); - Thread.sleep(runTime); - log.info("Woke up!"); - return TaskStatus.success(getId()); - } + Thread.sleep(runTime); + return TaskStatus.success(getId()); } @Override @@ -180,33 +113,18 @@ public int getPriority() public static NoopTask create() { - return new NoopTask(null, null, null, 0, 0, null, null, null); - } - - public static NoopTask withGroupId(String groupId) - { - return new NoopTask(null, groupId, null, 0, 0, null, null, null); + return forDatasource(null); } - @VisibleForTesting - public static NoopTask create(String dataSource) + public static NoopTask forDatasource(String datasource) { - return new NoopTask(null, null, dataSource, 0, 0, null, null, null); - } - - @VisibleForTesting - public static NoopTask create(int priority) - { - final Map context = new HashMap<>(); - context.put(Tasks.PRIORITY_KEY, priority); - return new NoopTask(null, null, null, 0, 0, null, null, context); + return new NoopTask(null, null, datasource, 0, 0, null); } - @VisibleForTesting - public static NoopTask create(String id, int priority) + public static NoopTask ofPriority(int priority) { final Map context = new HashMap<>(); context.put(Tasks.PRIORITY_KEY, priority); - return new NoopTask(id, null, null, 0, 0, null, null, context); + return new NoopTask(null, null, null, 0, 0, context); } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/RemoteTaskActionClientTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/RemoteTaskActionClientTest.java index f93ae4852570..af3b9625e470 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/RemoteTaskActionClientTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/RemoteTaskActionClientTest.java @@ -87,7 +87,7 @@ public void testSubmitSimple() throws Exception final BytesFullResponseHolder responseHolder = new BytesFullResponseHolder(httpResponse); responseHolder.addChunk(objectMapper.writeValueAsBytes(expectedResponse)); - final Task task = NoopTask.create("id", 0); + final Task task = NoopTask.create(); final LockListAction action = new LockListAction(); EasyMock.expect( @@ -123,7 +123,7 @@ public void testSubmitWithIllegalStatusCode() throws Exception StandardCharsets.UTF_8 ).addChunk("testSubmitWithIllegalStatusCode"); - final Task task = NoopTask.create("id", 0); + final Task task = NoopTask.create(); final LockListAction action = new LockListAction(); EasyMock.expect( directOverlordClient.request( diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentAllocationQueueTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentAllocationQueueTest.java index 8adb18ddb763..a71cacca13a4 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentAllocationQueueTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentAllocationQueueTest.java @@ -358,7 +358,7 @@ private SegmentAllocateActionBuilder allocateRequest() private Task createTask(String datasource, String groupId) { - Task task = new NoopTask(null, groupId, datasource, 0, 0, null, null, null); + Task task = new NoopTask(null, groupId, datasource, 0, 0, null); taskActionTestKit.getTaskLockbox().add(task); return task; } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/AbstractTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/AbstractTaskTest.java index cf03134c4ab9..5bcadcfb7125 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/AbstractTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/AbstractTaskTest.java @@ -90,7 +90,7 @@ public void testSetupAndCleanupIsCalledWtihParameter() throws Exception when(toolbox.getTaskActionClient()).thenReturn(taskActionClient); - AbstractTask task = new NoopTask("myID", null, null, 1, 0, null, null, null) + AbstractTask task = new NoopTask("myID", null, null, 1, 0, null) { @Nullable @Override @@ -138,7 +138,7 @@ public void testWithNoEncapsulatedTask() throws Exception when(toolbox.getTaskActionClient()).thenReturn(taskActionClient); - AbstractTask task = new NoopTask("myID", null, null, 1, 0, null, null, null) + AbstractTask task = new NoopTask("myID", null, null, 1, 0, null) { @Nullable @Override @@ -182,7 +182,7 @@ public void testTaskFailureWithoutExceptionGetsReportedCorrectly() throws Except when(taskActionClient.submit(any())).thenReturn(TaskConfig.class); when(toolbox.getTaskActionClient()).thenReturn(taskActionClient); - AbstractTask task = new NoopTask("myID", null, null, 1, 0, null, null, null) + AbstractTask task = new NoopTask("myID", null, null, 1, 0, null) { @Override public TaskStatus runTask(TaskToolbox toolbox) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/NoopTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/NoopTaskTest.java index ad7a61fa204b..bb48e916de37 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/NoopTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/NoopTaskTest.java @@ -27,7 +27,7 @@ public class NoopTaskTest @Test public void testNullInputSources() { - NoopTask task = new NoopTask("myID", null, null, 1, 0, null, null, null); + NoopTask task = NoopTask.create(); Assert.assertTrue(task.getInputSourceResources().isEmpty()); } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexPhaseRunnerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexPhaseRunnerTest.java index b09841e916b8..ade1692ca1c4 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexPhaseRunnerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexPhaseRunnerTest.java @@ -232,8 +232,6 @@ private ReportingNoopTask(String groupId, TestPhaseRunner phaseRunner) null, 10, 0, - null, - null, Collections.singletonMap(AbstractParallelIndexSupervisorTaskTest.DISABLE_TASK_INJECT_CONTEXT_KEY, true) ); this.phaseRunner = phaseRunner; diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/TaskMonitorTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/TaskMonitorTest.java index a09644ea6f5f..4d04dbd86cd8 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/TaskMonitorTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/TaskMonitorTest.java @@ -231,7 +231,7 @@ private class TestTask extends NoopTask TestTask(String id, long runTime, boolean shouldFail, boolean throwUnknownTypeIdError) { - super(id, null, "testDataSource", runTime, 0, null, null, null); + super(id, null, "testDataSource", runTime, 0, null); this.shouldFail = shouldFail; this.throwUnknownTypeIdError = throwUnknownTypeIdError; } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorageTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorageTest.java index 12267a24f7c0..d300ceefb137 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorageTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorageTest.java @@ -47,8 +47,8 @@ public void setUp() @Test public void testRemoveTasksOlderThan() { - final NoopTask task1 = NoopTask.create("foo"); - final NoopTask task2 = NoopTask.create("bar"); + final NoopTask task1 = NoopTask.create(); + final NoopTask task2 = NoopTask.create(); storage.insert(task1, TaskStatus.success(task1.getId())); storage.insert(task2, TaskStatus.running(task2.getId())); @@ -64,8 +64,8 @@ public void testRemoveTasksOlderThan() @Test public void testGetTaskInfos() { - final NoopTask task1 = NoopTask.create("foo"); - final NoopTask task2 = NoopTask.create("bar"); + final NoopTask task1 = NoopTask.create(); + final NoopTask task2 = NoopTask.create(); storage.insert(task1, TaskStatus.success(task1.getId())); storage.insert(task2, TaskStatus.running(task2.getId())); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageAdapterTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageAdapterTest.java index 4a9c066416c8..196056341176 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageAdapterTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageAdapterTest.java @@ -66,14 +66,14 @@ public void testDeletePendingSegments() DateTimes.of("2017-12-01"), TaskStatus.running("id1"), "dataSource", - NoopTask.create("id1", 0) + NoopTask.create() ), new TaskInfo<>( "id1", DateTimes.of("2017-12-02"), TaskStatus.running("id2"), "dataSource", - NoopTask.create("id2", 0) + NoopTask.create() ) ); EasyMock.expect(taskStorageQueryAdapter.getActiveTaskInfo("dataSource")).andReturn(taskInfos); @@ -101,14 +101,14 @@ public void testDeletePendingSegmentsOfRunningTasks() DateTimes.of("2017-11-01"), TaskStatus.running("id1"), "dataSource", - NoopTask.create("id1", 0) + NoopTask.create() ), new TaskInfo<>( "id1", DateTimes.of("2017-12-02"), TaskStatus.running("id2"), "dataSource", - NoopTask.create("id2", 0) + NoopTask.create() ) ); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunnerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunnerTest.java index e683c828ddc6..fc7bd9236018 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunnerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunnerTest.java @@ -154,21 +154,7 @@ public void teardown() @Test public void testRun() throws ExecutionException, InterruptedException { - NoopTask task = new NoopTask(null, null, null, 500L, 0, null, null, null) - { - @Nullable - @Override - public String setup(TaskToolbox toolbox) - { - return null; - } - - @Override - public void cleanUp(TaskToolbox toolbox, TaskStatus taskStatus) - { - // do nothing - } - }; + NoopTask task = new NoopTask(null, null, null, 500L, 0, null); Assert.assertEquals( TaskState.SUCCESS, runner.run(task).get().getStatusCode() @@ -178,7 +164,7 @@ public void cleanUp(TaskToolbox toolbox, TaskStatus taskStatus) @Test public void testGetQueryRunner() throws ExecutionException, InterruptedException { - runner.run(new NoopTask(null, null, "foo", 500L, 0, null, null, null)).get().getStatusCode(); + runner.run(new NoopTask(null, null, "foo", 500L, 0, null)).get().getStatusCode(); final QueryRunner queryRunner = Druids.newScanQueryBuilder() @@ -194,7 +180,7 @@ public void testGetQueryRunner() throws ExecutionException, InterruptedException public void testStop() throws ExecutionException, InterruptedException, TimeoutException { final ListenableFuture future = runner.run( - new NoopTask(null, null, null, Long.MAX_VALUE, 0, null, null, null) // infinite task + new NoopTask(null, null, null, Long.MAX_VALUE, 0, null) // infinite task ); runner.stop(); Assert.assertEquals( @@ -320,8 +306,6 @@ public void statusChanged(String taskId, TaskStatus status) "datasource", 10000, // 10 sec 0, - null, - null, null ) ); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java index 31ec645b3fdd..185e1468d399 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java @@ -66,6 +66,7 @@ import org.apache.druid.indexing.common.actions.LocalTaskActionClientFactory; import org.apache.druid.indexing.common.actions.LockListAction; import org.apache.druid.indexing.common.actions.SegmentInsertAction; +import org.apache.druid.indexing.common.actions.TaskActionClient; import org.apache.druid.indexing.common.actions.TaskActionClientFactory; import org.apache.druid.indexing.common.actions.TaskActionToolbox; import org.apache.druid.indexing.common.actions.TaskAuditLogConfig; @@ -79,6 +80,7 @@ import org.apache.druid.indexing.common.task.IndexTask.IndexIngestionSpec; import org.apache.druid.indexing.common.task.IndexTask.IndexTuningConfig; import org.apache.druid.indexing.common.task.KillUnusedSegmentsTask; +import org.apache.druid.indexing.common.task.NoopTask; import org.apache.druid.indexing.common.task.NoopTestTaskReportFileWriter; import org.apache.druid.indexing.common.task.RealtimeIndexTask; import org.apache.druid.indexing.common.task.Task; @@ -88,6 +90,7 @@ import org.apache.druid.indexing.overlord.config.TaskLockConfig; import org.apache.druid.indexing.overlord.config.TaskQueueConfig; import org.apache.druid.indexing.overlord.supervisor.SupervisorManager; +import org.apache.druid.indexing.test.TestDataSegmentAnnouncer; import org.apache.druid.indexing.test.TestIndexerMetadataStorageCoordinator; import org.apache.druid.indexing.worker.config.WorkerConfig; import org.apache.druid.jackson.DefaultObjectMapper; @@ -98,6 +101,7 @@ import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.RE; +import org.apache.druid.java.util.common.Stopwatch; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.granularity.Granularities; @@ -129,18 +133,16 @@ import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; import org.apache.druid.segment.join.JoinableFactoryWrapperTest; import org.apache.druid.segment.join.NoopJoinableFactory; -import org.apache.druid.segment.loading.DataSegmentArchiver; -import org.apache.druid.segment.loading.DataSegmentMover; import org.apache.druid.segment.loading.DataSegmentPusher; import org.apache.druid.segment.loading.LocalDataSegmentKiller; import org.apache.druid.segment.loading.LocalDataSegmentPusherConfig; +import org.apache.druid.segment.loading.NoopDataSegmentArchiver; import org.apache.druid.segment.realtime.FireDepartment; import org.apache.druid.segment.realtime.FireDepartmentTest; import org.apache.druid.segment.realtime.appenderator.AppenderatorsManager; import org.apache.druid.segment.realtime.appenderator.UnifiedIndexerAppenderatorsManager; import org.apache.druid.segment.realtime.firehose.NoopChatHandlerProvider; import org.apache.druid.server.DruidNode; -import org.apache.druid.server.coordination.DataSegmentAnnouncer; import org.apache.druid.server.coordination.DataSegmentServerAnnouncer; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.initialization.ServerConfig; @@ -452,8 +454,6 @@ private TaskStorage setUpTaskStorage() case HEAP_TASK_STORAGE: { taskStorage = new HeapMemoryTaskStorage( new TaskStorageConfig(null) - { - } ); break; } @@ -479,9 +479,8 @@ private TaskStorage setUpTaskStorage() break; } - default: { + default: throw new RE("Unknown task storage type [%s]", taskStorageType); - } } tsqa = new TaskStorageQueryAdapter(taskStorage, taskLockbox); return taskStorage; @@ -622,29 +621,9 @@ private TaskToolboxFactory setUpTaskToolboxFactory( emitter, dataSegmentPusher, new LocalDataSegmentKiller(new LocalDataSegmentPusherConfig()), - new DataSegmentMover() - { - @Override - public DataSegment move(DataSegment dataSegment, Map targetLoadSpec) - { - return dataSegment; - } - }, - new DataSegmentArchiver() - { - @Override - public DataSegment archive(DataSegment segment) - { - return segment; - } - - @Override - public DataSegment restore(DataSegment segment) - { - return segment; - } - }, - new DataSegmentAnnouncer() + (dataSegment, targetLoadSpec) -> dataSegment, + new NoopDataSegmentArchiver(), + new TestDataSegmentAnnouncer() { @Override public void announceSegment(DataSegment segment) @@ -652,24 +631,7 @@ public void announceSegment(DataSegment segment) announcedSinks++; } - @Override - public void unannounceSegment(DataSegment segment) - { - - } - - @Override - public void announceSegments(Iterable segments) - { - - } - - @Override - public void unannounceSegments(Iterable segments) - { - - } - }, // segment announcer + }, EasyMock.createNiceMock(DataSegmentServerAnnouncer.class), handoffNotifierFactory, () -> queryRunnerFactoryConglomerate, // query runner factory conglomerate corporation unionized collective @@ -740,7 +702,7 @@ public void tearDown() } @Test - public void testIndexTask() throws Exception + public void testIndexTask() { final Task indexTask = new IndexTask( null, @@ -824,7 +786,7 @@ public void testIndexTask() throws Exception } @Test - public void testIndexTaskFailure() throws Exception + public void testIndexTaskFailure() { final Task indexTask = new IndexTask( null, @@ -1070,7 +1032,7 @@ public DataSegment apply(String input) } @Test - public void testRealtimeishTask() throws Exception + public void testRealtimeishTask() { final Task rtishTask = new RealtimeishTask(); final TaskStatus status = runTask(rtishTask); @@ -1082,13 +1044,9 @@ public void testRealtimeishTask() throws Exception } @Test - public void testNoopTask() throws Exception + public void testNoopTask() { - final Task noopTask = new DefaultObjectMapper().readValue( - "{\"type\":\"noop\", \"runTime\":\"100\"}\"", - Task.class - ); - final TaskStatus status = runTask(noopTask); + final TaskStatus status = runTask(NoopTask.create()); Assert.assertEquals("statusCode", TaskState.SUCCESS, status.getStatusCode()); Assert.assertEquals(taskLocation, status.getLocation()); @@ -1097,12 +1055,16 @@ public void testNoopTask() throws Exception } @Test - public void testNeverReadyTask() throws Exception + public void testNeverReadyTask() { - final Task neverReadyTask = new DefaultObjectMapper().readValue( - "{\"type\":\"noop\", \"isReadyResult\":\"exception\"}\"", - Task.class - ); + final Task neverReadyTask = new NoopTask(null, null, null, 0, 0, null) + { + @Override + public boolean isReady(TaskActionClient taskActionClient) + { + throw new ISE("Task will never be ready"); + } + }; final TaskStatus status = runTask(neverReadyTask); Assert.assertEquals("statusCode", TaskState.FAILED, status.getStatusCode()); @@ -1112,7 +1074,7 @@ public void testNeverReadyTask() throws Exception } @Test - public void testSimple() throws Exception + public void testSimple() { final Task task = new AbstractFixedIntervalTask( "id1", @@ -1169,7 +1131,7 @@ public TaskStatus runTask(TaskToolbox toolbox) throws Exception } @Test - public void testBadInterval() throws Exception + public void testBadInterval() { final Task task = new AbstractFixedIntervalTask("id1", "id1", "ds", Intervals.of("2012-01-01/P1D"), null) { @@ -1211,7 +1173,7 @@ public TaskStatus runTask(TaskToolbox toolbox) throws Exception } @Test - public void testBadVersion() throws Exception + public void testBadVersion() { final Task task = new AbstractFixedIntervalTask("id1", "id1", "ds", Intervals.of("2012-01-01/P1D"), null) { @@ -1541,7 +1503,7 @@ public void testUnifiedAppenderatorsManagerCleanup() throws Exception } @Test - public void testLockRevoked() throws Exception + public void testLockRevoked() { final Task task = new AbstractFixedIntervalTask( "id1", @@ -1595,15 +1557,9 @@ public TaskStatus runTask(TaskToolbox toolbox) throws Exception Assert.assertEquals("segments nuked", 0, mdc.getNuked().size()); } - private TaskStatus runTask(final Task task) throws Exception + private TaskStatus runTask(final Task task) { - final Task dummyTask = new DefaultObjectMapper().readValue( - "{\"type\":\"noop\", \"isReadyResult\":\"exception\"}\"", - Task.class - ); - final long startTime = System.currentTimeMillis(); - - Preconditions.checkArgument(!task.getId().equals(dummyTask.getId())); + final Stopwatch taskRunDuration = Stopwatch.createStarted(); // Since multiple tasks can be run in a single unit test using runTask(), hence this check and synchronization synchronized (this) { @@ -1611,29 +1567,26 @@ private TaskStatus runTask(final Task task) throws Exception taskQueue.start(); } } - taskQueue.add(dummyTask); taskQueue.add(task); + final String taskId = task.getId(); TaskStatus retVal = null; - - for (final String taskId : ImmutableList.of(dummyTask.getId(), task.getId())) { - try { - TaskStatus status; - while ((status = tsqa.getStatus(taskId).get()).isRunnable()) { - if (System.currentTimeMillis() > startTime + 10 * 1000) { - throw new ISE("Where did the task go?!: %s", task.getId()); - } - - Thread.sleep(100); - } - if (taskId.equals(task.getId())) { - retVal = status; + try { + TaskStatus status; + while ((status = tsqa.getStatus(taskId).get()).isRunnable()) { + if (taskRunDuration.millisElapsed() > 10_000) { + throw new ISE("Where did the task go?!: %s", task.getId()); } + + Thread.sleep(100); } - catch (Exception e) { - throw new RuntimeException(e); + if (taskId.equals(task.getId())) { + retVal = status; } } + catch (Exception e) { + throw new RuntimeException(e); + } return retVal; } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockBoxConcurrencyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockBoxConcurrencyTest.java index 9c198f429179..5a75e205f9d1 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockBoxConcurrencyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockBoxConcurrencyTest.java @@ -102,8 +102,8 @@ public void testDoInCriticalSectionWithDifferentTasks() throws ExecutionException, InterruptedException, EntryExistsException { final Interval interval = Intervals.of("2017-01-01/2017-01-02"); - final Task lowPriorityTask = NoopTask.create(10); - final Task highPriorityTask = NoopTask.create(100); + final Task lowPriorityTask = NoopTask.ofPriority(10); + final Task highPriorityTask = NoopTask.ofPriority(100); lockbox.add(lowPriorityTask); lockbox.add(highPriorityTask); taskStorage.insert(lowPriorityTask, TaskStatus.running(lowPriorityTask.getId())); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockboxTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockboxTest.java index c9f9d72beaae..6b9b4cd213d5 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockboxTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockboxTest.java @@ -221,9 +221,9 @@ public void testTrySharedLock() @Test public void testTryMixedLocks() throws EntryExistsException { - final Task lowPriorityTask = NoopTask.create(0); - final Task lowPriorityTask2 = NoopTask.create(0); - final Task highPiorityTask = NoopTask.create(10); + final Task lowPriorityTask = NoopTask.ofPriority(0); + final Task lowPriorityTask2 = NoopTask.ofPriority(0); + final Task highPiorityTask = NoopTask.ofPriority(10); final Interval interval1 = Intervals.of("2017-01-01/2017-01-02"); final Interval interval2 = Intervals.of("2017-01-02/2017-01-03"); final Interval interval3 = Intervals.of("2017-01-03/2017-01-04"); @@ -476,13 +476,13 @@ public void testRevokedLockSyncFromStorage() throws EntryExistsException { final TaskLockbox originalBox = new TaskLockbox(taskStorage, metadataStorageCoordinator); - final Task task1 = NoopTask.create("task1", 10); + final Task task1 = NoopTask.ofPriority(10); taskStorage.insert(task1, TaskStatus.running(task1.getId())); originalBox.add(task1); Assert.assertTrue(originalBox.tryLock(task1, new TimeChunkLockRequest(TaskLockType.EXCLUSIVE, task1, Intervals.of("2017/2018"), null)).isOk()); // task2 revokes task1 - final Task task2 = NoopTask.create("task2", 100); + final Task task2 = NoopTask.ofPriority(100); taskStorage.insert(task2, TaskStatus.running(task2.getId())); originalBox.add(task2); Assert.assertTrue(originalBox.tryLock(task2, new TimeChunkLockRequest(TaskLockType.EXCLUSIVE, task2, Intervals.of("2017/2018"), null)).isOk()); @@ -492,11 +492,11 @@ public void testRevokedLockSyncFromStorage() throws EntryExistsException .stream() .collect(Collectors.toMap(Task::getId, task -> taskStorage.getLocks(task.getId()))); - final List task1Locks = beforeLocksInStorage.get("task1"); + final List task1Locks = beforeLocksInStorage.get(task1.getId()); Assert.assertEquals(1, task1Locks.size()); Assert.assertTrue(task1Locks.get(0).isRevoked()); - final List task2Locks = beforeLocksInStorage.get("task1"); + final List task2Locks = beforeLocksInStorage.get(task1.getId()); Assert.assertEquals(1, task2Locks.size()); Assert.assertTrue(task2Locks.get(0).isRevoked()); @@ -578,7 +578,7 @@ public void testPreemptionAndDoInCriticalSection() throws Exception Assert.assertTrue(tryTimeChunkLock(TaskLockType.SHARED, task, interval).isOk()); } - final Task highPriorityTask = NoopTask.create(100); + final Task highPriorityTask = NoopTask.ofPriority(100); lockbox.add(highPriorityTask); taskStorage.insert(highPriorityTask, TaskStatus.running(highPriorityTask.getId())); final TaskLock lock = tryTimeChunkLock(TaskLockType.EXCLUSIVE, highPriorityTask, interval).getTaskLock(); @@ -597,8 +597,8 @@ public void testPreemptionAndDoInCriticalSection() throws Exception public void testDoInCriticalSectionWithRevokedLock() throws Exception { final Interval interval = Intervals.of("2017-01-01/2017-01-02"); - final Task lowPriorityTask = NoopTask.create("task1", 0); - final Task highPriorityTask = NoopTask.create("task2", 10); + final Task lowPriorityTask = NoopTask.ofPriority(0); + final Task highPriorityTask = NoopTask.ofPriority(10); lockbox.add(lowPriorityTask); lockbox.add(highPriorityTask); taskStorage.insert(lowPriorityTask, TaskStatus.running(lowPriorityTask.getId())); @@ -622,8 +622,8 @@ public void testDoInCriticalSectionWithRevokedLock() throws Exception public void testAcquireLockAfterRevoked() throws EntryExistsException, InterruptedException { final Interval interval = Intervals.of("2017-01-01/2017-01-02"); - final Task lowPriorityTask = NoopTask.create("task1", 0); - final Task highPriorityTask = NoopTask.create("task2", 10); + final Task lowPriorityTask = NoopTask.ofPriority(0); + final Task highPriorityTask = NoopTask.ofPriority(10); lockbox.add(lowPriorityTask); lockbox.add(highPriorityTask); taskStorage.insert(lowPriorityTask, TaskStatus.running(lowPriorityTask.getId())); @@ -650,7 +650,7 @@ public void testUnlock() throws EntryExistsException final List highPriorityTasks = new ArrayList<>(); for (int i = 0; i < 8; i++) { - final Task task = NoopTask.create(10); + final Task task = NoopTask.ofPriority(10); lowPriorityTasks.add(task); taskStorage.insert(task, TaskStatus.running(task.getId())); lockbox.add(task); @@ -665,7 +665,7 @@ public void testUnlock() throws EntryExistsException // Revoke some locks for (int i = 0; i < 4; i++) { - final Task task = NoopTask.create(100); + final Task task = NoopTask.ofPriority(100); highPriorityTasks.add(task); taskStorage.insert(task, TaskStatus.running(task.getId())); lockbox.add(task); @@ -711,8 +711,8 @@ public void testUnlock() throws EntryExistsException @Test public void testFindLockPosseAfterRevokeWithDifferentLockIntervals() throws EntryExistsException { - final Task lowPriorityTask = NoopTask.create(0); - final Task highPriorityTask = NoopTask.create(10); + final Task lowPriorityTask = NoopTask.ofPriority(0); + final Task highPriorityTask = NoopTask.ofPriority(10); taskStorage.insert(lowPriorityTask, TaskStatus.running(lowPriorityTask.getId())); taskStorage.insert(highPriorityTask, TaskStatus.running(highPriorityTask.getId())); @@ -821,11 +821,11 @@ public void testSegmentAndTimeChunkLockForSameInterval() @Test public void testSegmentAndTimeChunkLockForSameIntervalWithDifferentPriority() throws EntryExistsException { - final Task task1 = NoopTask.create(10); + final Task task1 = NoopTask.ofPriority(10); lockbox.add(task1); taskStorage.insert(task1, TaskStatus.running(task1.getId())); - final Task task2 = NoopTask.create(100); + final Task task2 = NoopTask.ofPriority(100); lockbox.add(task2); taskStorage.insert(task2, TaskStatus.running(task2.getId())); @@ -1086,8 +1086,8 @@ public void testLockPosseEquals() @Test public void testGetTimeChunkAndSegmentLockForSameGroup() { - final Task task1 = NoopTask.withGroupId("groupId"); - final Task task2 = NoopTask.withGroupId("groupId"); + final Task task1 = new NoopTask(null, "groupId", null, 0, 0, null); + final Task task2 = new NoopTask(null, "groupId", null, 0, 0, null); lockbox.add(task1); lockbox.add(task2); @@ -1130,8 +1130,8 @@ public void testGetTimeChunkAndSegmentLockForSameGroup() @Test public void testGetTimeChunkAndSegmentLockForDifferentGroup() { - final Task task1 = NoopTask.withGroupId("groupId"); - final Task task2 = NoopTask.withGroupId("groupId2"); + final Task task1 = new NoopTask(null, "group1", "wiki", 0, 0, null); + final Task task2 = new NoopTask(null, "group2", "wiki", 0, 0, null); lockbox.add(task1); lockbox.add(task2); @@ -1155,7 +1155,7 @@ public void testGetTimeChunkAndSegmentLockForDifferentGroup() public void testGetLockedIntervals() { // Acquire locks for task1 - final Task task1 = NoopTask.create("ds1"); + final Task task1 = NoopTask.forDatasource("ds1"); lockbox.add(task1); tryTimeChunkLock( @@ -1170,7 +1170,7 @@ public void testGetLockedIntervals() ); // Acquire locks for task2 - final Task task2 = NoopTask.create("ds2"); + final Task task2 = NoopTask.forDatasource("ds2"); lockbox.add(task2); tryTimeChunkLock( TaskLockType.EXCLUSIVE, @@ -1204,7 +1204,7 @@ public void testGetLockedIntervals() public void testGetLockedIntervalsForLowPriorityTask() { // Acquire lock for a low priority task - final Task lowPriorityTask = NoopTask.create(5); + final Task lowPriorityTask = NoopTask.ofPriority(5); lockbox.add(lowPriorityTask); taskStorage.insert(lowPriorityTask, TaskStatus.running(lowPriorityTask.getId())); tryTimeChunkLock( @@ -1224,7 +1224,7 @@ public void testGetLockedIntervalsForLowPriorityTask() public void testGetLockedIntervalsForEqualPriorityTask() { // Acquire lock for a low priority task - final Task task = NoopTask.create(5); + final Task task = NoopTask.ofPriority(5); lockbox.add(task); taskStorage.insert(task, TaskStatus.running(task.getId())); tryTimeChunkLock( @@ -1622,7 +1622,7 @@ public void testReplaceLockCanRevokeAllIncompatible() public void testGetLockedIntervalsForRevokedLocks() { // Acquire lock for a low priority task - final Task lowPriorityTask = NoopTask.create(5); + final Task lowPriorityTask = NoopTask.ofPriority(5); lockbox.add(lowPriorityTask); taskStorage.insert(lowPriorityTask, TaskStatus.running(lowPriorityTask.getId())); tryTimeChunkLock( @@ -1643,7 +1643,7 @@ public void testGetLockedIntervalsForRevokedLocks() ); // Revoke the lowPriorityTask - final Task highPriorityTask = NoopTask.create(10); + final Task highPriorityTask = NoopTask.ofPriority(10); lockbox.add(highPriorityTask); tryTimeChunkLock( TaskLockType.EXCLUSIVE, @@ -1667,8 +1667,8 @@ public void testFailedToReacquireTaskLock() { // Tasks to be failed have a group id with the substring "FailingLockAcquisition" // Please refer to NullLockPosseTaskLockbox - final Task taskWithFailingLockAcquisition0 = NoopTask.withGroupId("FailingLockAcquisition"); - final Task taskWithFailingLockAcquisition1 = NoopTask.withGroupId("FailingLockAcquisition"); + final Task taskWithFailingLockAcquisition0 = new NoopTask(null, "FailingLockAcquisition", null, 0, 0, null); + final Task taskWithFailingLockAcquisition1 = new NoopTask(null, "FailingLockAcquisition", null, 0, 0, null); final Task taskWithSuccessfulLockAcquisition = NoopTask.create(); taskStorage.insert(taskWithFailingLockAcquisition0, TaskStatus.running(taskWithFailingLockAcquisition0.getId())); taskStorage.insert(taskWithFailingLockAcquisition1, TaskStatus.running(taskWithFailingLockAcquisition1.getId())); @@ -1787,7 +1787,7 @@ public void expectActiveLocks(TaskLock... locks) private TaskLock tryTaskLock(TaskLockType type, Interval interval, int priority) { - final Task task = NoopTask.create(priority); + final Task task = NoopTask.ofPriority(priority); tasks.add(task); lockbox.add(task); taskStorage.insert(task, TaskStatus.running(task.getId())); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueScaleTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueScaleTest.java index 2d2937c814e5..ade1314884e7 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueScaleTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueScaleTest.java @@ -144,7 +144,7 @@ public void doMassLaunchAndExit() throws Exception // Add all tasks. for (int i = 0; i < numTasks; i++) { - final TestTask testTask = new TestTask(i, 2000L /* runtime millis */); + final NoopTask testTask = createTestTask(2000L); taskQueue.add(testTask); } @@ -182,8 +182,7 @@ public void doMassLaunchAndShutdown() throws Exception // Add all tasks. final List taskIds = new ArrayList<>(); for (int i = 0; i < numTasks; i++) { - final TestTask testTask = new TestTask( - i, + final NoopTask testTask = createTestTask( Duration.standardHours(1).getMillis() /* very long runtime millis, so we can do a shutdown */ ); taskQueue.add(testTask); @@ -215,27 +214,9 @@ public void doMassLaunchAndShutdown() throws Exception Assert.assertEquals("all tasks should have completed", numTasks, completed); } - private static class TestTask extends NoopTask + private NoopTask createTestTask(long runtimeMillis) { - private final int number; - private final long runtime; - - public TestTask(int number, long runtime) - { - super(null, null, DATASOURCE, 0, 0, null, null, Collections.emptyMap()); - this.number = number; - this.runtime = runtime; - } - - public int getNumber() - { - return number; - } - - public long getRuntimeMillis() - { - return runtime; - } + return new NoopTask(null, null, DATASOURCE, runtimeMillis, 0, Collections.emptyMap()); } private static class TestTaskRunner implements TaskRunner @@ -289,7 +270,7 @@ public ListenableFuture run(Task task) log.error(e, "Error in scheduled executor"); } }, - ((TestTask) task).getRuntimeMillis(), + ((NoopTask) task).getRunTime(), TimeUnit.MILLISECONDS ); } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/ZkWorkerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/ZkWorkerTest.java index a051737a0ed4..fac929912088 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/ZkWorkerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/ZkWorkerTest.java @@ -129,9 +129,7 @@ public void testIdWithWrongTypeReturnsNull() @Test public void testCanReadIdFromAJacksonSerializedTaskAnnouncement() throws JsonProcessingException { - final String expectedTaskId = "task01234"; - - Task task0 = NoopTask.create(expectedTaskId, 0); + Task task0 = NoopTask.create(); TaskAnnouncement taskAnnouncement = TaskAnnouncement.create( task0, TaskStatus.running(task0.getId()), @@ -145,6 +143,6 @@ public void testCanReadIdFromAJacksonSerializedTaskAnnouncement() throws JsonPro ChildData zkNode = new ChildData("/a/b/c", new Stat(), serialized); String actualExtractedTaskId = extract.apply(zkNode); - Assert.assertEquals(expectedTaskId, actualExtractedTaskId); + Assert.assertEquals(task0.getId(), actualExtractedTaskId); } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunnerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunnerTest.java index 7b2a6f618b94..54092ddcf18a 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunnerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunnerTest.java @@ -137,7 +137,7 @@ WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 2, "0", Wo int numTasks = 8; List> futures = new ArrayList<>(); for (int i = 0; i < numTasks; i++) { - futures.add(taskRunner.run(NoopTask.create("task-id-" + i, 0))); + futures.add(taskRunner.run(NoopTask.create())); } for (Future future : futures) { @@ -284,9 +284,9 @@ public void testOneStuckTaskAssignmentDoesntBlockOthers() throws Exception .andReturn(druidNodeDiscovery); EasyMock.replay(druidNodeDiscoveryProvider); - Task task1 = NoopTask.create("task-id-1", 0); - Task task2 = NoopTask.create("task-id-2", 0); - Task task3 = NoopTask.create("task-id-3", 0); + Task task1 = NoopTask.create(); + Task task2 = NoopTask.create(); + Task task3 = NoopTask.create(); HttpRemoteTaskRunner taskRunner = new HttpRemoteTaskRunner( TestHelper.makeJsonMapper(), @@ -379,11 +379,11 @@ public void testTaskRunnerRestart() throws Exception ConcurrentMap workerHolders = new ConcurrentHashMap<>(); - Task task1 = NoopTask.create("task-id-1", 0); - Task task2 = NoopTask.create("task-id-2", 0); - Task task3 = NoopTask.create("task-id-3", 0); - Task task4 = NoopTask.create("task-id-4", 0); - Task task5 = NoopTask.create("task-id-5", 0); + Task task1 = NoopTask.create(); + Task task2 = NoopTask.create(); + Task task3 = NoopTask.create(); + Task task4 = NoopTask.create(); + Task task5 = NoopTask.create(); TaskStorage taskStorageMock = EasyMock.createStrictMock(TaskStorage.class); EasyMock.expect(taskStorageMock.getStatus(task1.getId())).andReturn(Optional.absent()); @@ -585,8 +585,8 @@ protected WorkerHolder createWorkerHolder( taskRunner.start(); - Task task1 = NoopTask.create("task-id-1", 0); - Task task2 = NoopTask.create("task-id-2", 0); + Task task1 = NoopTask.create(); + Task task2 = NoopTask.create(); DiscoveryDruidNode druidNode = new DiscoveryDruidNode( new DruidNode("service", "host", false, 1234, null, true, false), @@ -761,8 +761,8 @@ protected WorkerHolder createWorkerHolder( taskRunner.start(); - Task task1 = NoopTask.create("task-id-1", 0); - Task task2 = NoopTask.create("task-id-2", 0); + Task task1 = NoopTask.create(); + Task task2 = NoopTask.create(); DiscoveryDruidNode druidNode = new DiscoveryDruidNode( new DruidNode("service", "host", false, 1234, null, true, false), @@ -904,8 +904,8 @@ public void testMarkWorkersLazy() throws Exception .andReturn(druidNodeDiscovery); EasyMock.replay(druidNodeDiscoveryProvider); - Task task1 = NoopTask.create("task-id-1", 0); - Task task2 = NoopTask.create("task-id-2", 0); + Task task1 = NoopTask.create(); + Task task2 = NoopTask.create(); String additionalWorkerCategory = "category2"; ConcurrentMap workerHolders = new ConcurrentHashMap<>(); @@ -1140,7 +1140,7 @@ WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 1, "0", Wo @Test public void testTaskAddedOrUpdated1() throws Exception { - Task task = NoopTask.create("task"); + Task task = NoopTask.create(); List listenerNotificationsAccumulator = new ArrayList<>(); HttpRemoteTaskRunner taskRunner = createTaskRunnerForTestTaskAddedOrUpdated( EasyMock.createStrictMock(TaskStorage.class), @@ -1269,7 +1269,7 @@ public void testTaskAddedOrUpdated1() throws Exception @Test public void testTaskAddedOrUpdated2() throws Exception { - Task task = NoopTask.create("task"); + Task task = NoopTask.create(); List listenerNotificationsAccumulator = new ArrayList<>(); HttpRemoteTaskRunner taskRunner = createTaskRunnerForTestTaskAddedOrUpdated( EasyMock.createStrictMock(TaskStorage.class), @@ -1315,12 +1315,12 @@ public void testTaskAddedOrUpdated2() throws Exception @Test public void testTaskAddedOrUpdated3() { - Task task1 = NoopTask.create("task1"); - Task task2 = NoopTask.create("task2"); - Task task3 = NoopTask.create("task3"); - Task task4 = NoopTask.create("task4"); - Task task5 = NoopTask.create("task5"); - Task task6 = NoopTask.create("task6"); + Task task1 = NoopTask.create(); + Task task2 = NoopTask.create(); + Task task3 = NoopTask.create(); + Task task4 = NoopTask.create(); + Task task5 = NoopTask.create(); + Task task6 = NoopTask.create(); TaskStorage taskStorage = EasyMock.createMock(TaskStorage.class); EasyMock.expect(taskStorage.getStatus(task1.getId())).andReturn(Optional.of(TaskStatus.running(task1.getId()))); @@ -1501,7 +1501,7 @@ WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", Wo druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode1)); - Future future = taskRunner.run(NoopTask.create("task-id", 0)); + Future future = taskRunner.run(NoopTask.create()); Assert.assertTrue(future.get().isFailure()); Assert.assertNotNull(future.get().getErrorMsg()); Assert.assertTrue( @@ -1613,7 +1613,7 @@ WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", Wo druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode1)); - Future future = taskRunner.run(NoopTask.create("task-id", 0)); + Future future = taskRunner.run(NoopTask.create()); Assert.assertTrue(future.get().isFailure()); Assert.assertNotNull(future.get().getErrorMsg()); Assert.assertTrue( @@ -1646,7 +1646,7 @@ public void testShutdown() taskRunner.start(); - Task pendingTask = NoopTask.create("pendingTask"); + Task pendingTask = NoopTask.create(); taskRunner.run(pendingTask); // Pending task is not cleaned up immediately taskRunner.shutdown(pendingTask.getId(), "Forced shutdown"); @@ -1657,7 +1657,7 @@ public void testShutdown() .contains(pendingTask.getId()) ); - Task completedTask = NoopTask.create("completedTask"); + Task completedTask = NoopTask.create(); taskRunner.run(completedTask); taskRunner.taskAddedOrUpdated(TaskAnnouncement.create( completedTask, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolderTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolderTest.java index f13a0b0cce80..3d0336029c11 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolderTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolderTest.java @@ -49,10 +49,10 @@ public void testSyncListener() { List updates = new ArrayList<>(); - Task task0 = NoopTask.create("task0", 0); - Task task1 = NoopTask.create("task1", 0); - Task task2 = NoopTask.create("task2", 0); - Task task3 = NoopTask.create("task3", 0); + Task task0 = NoopTask.create(); + Task task1 = NoopTask.create(); + Task task2 = NoopTask.create(); + Task task3 = NoopTask.create(); WorkerHolder workerHolder = new WorkerHolder( TestHelper.makeJsonMapper(), diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java index d244fb0ba304..6eed9e32df38 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java @@ -897,7 +897,7 @@ public void testTaskPostDeniesDatasourceReadUser() ); // Verify that taskPost fails for user who has only datasource read access - Task task = NoopTask.create(Datasources.WIKIPEDIA); + Task task = NoopTask.forDatasource(Datasources.WIKIPEDIA); expectedException.expect(ForbiddenException.class); expectedException.expect(ForbiddenException.class); overlordResource.taskPost(task, req); @@ -941,7 +941,7 @@ public void testGetTaskPayload() throws Exception // set authorization token properly, but isn't called in this test. // This should be fixed in https://github.com/apache/druid/issues/6685. // expectAuthorizationTokenCheck(); - final NoopTask task = NoopTask.create("mydatasource"); + final NoopTask task = NoopTask.create(); EasyMock.expect(taskStorageQueryAdapter.getTask("mytask")) .andReturn(Optional.of(task)); @@ -980,10 +980,11 @@ public void testGetTaskStatus() throws Exception // set authorization token properly, but isn't called in this test. // This should be fixed in https://github.com/apache/druid/issues/6685. // expectAuthorizationTokenCheck(); - final Task task = NoopTask.create("mytask", 0); - final TaskStatus status = TaskStatus.running("mytask"); + final Task task = NoopTask.create(); + final String taskId = task.getId(); + final TaskStatus status = TaskStatus.running(taskId); - EasyMock.expect(taskStorageQueryAdapter.getTaskInfo("mytask")) + EasyMock.expect(taskStorageQueryAdapter.getTaskInfo(taskId)) .andReturn(new TaskInfo( task.getId(), DateTimes.of("2018-01-01"), @@ -1008,7 +1009,7 @@ public void testGetTaskStatus() throws Exception authConfig ); - final Response response1 = overlordResource.getTaskStatus("mytask"); + final Response response1 = overlordResource.getTaskStatus(taskId); final TaskStatusResponse taskStatusResponse1 = TestHelper.makeJsonMapper().readValue( TestHelper.makeJsonMapper().writeValueAsString(response1.getEntity()), TaskStatusResponse.class @@ -1017,10 +1018,10 @@ public void testGetTaskStatus() throws Exception Assert.assertEquals(tsp.getStatusCode(), tsp.getStatus()); Assert.assertEquals( new TaskStatusResponse( - "mytask", + taskId, new TaskStatusPlus( - "mytask", - "mytask", + task.getId(), + task.getGroupId(), "noop", DateTimes.of("2018-01-01"), DateTimes.EPOCH, @@ -1155,14 +1156,14 @@ public void testShutdownAllTasks() DateTime.now(ISOChronology.getInstanceUTC()), TaskStatus.success("id_1"), "datasource", - NoopTask.create("id_1", 1) + NoopTask.create() ), new TaskInfo<>( "id_2", DateTime.now(ISOChronology.getInstanceUTC()), TaskStatus.success("id_2"), "datasource", - NoopTask.create("id_2", 1) + NoopTask.create() ) )); mockQueue.shutdown("id_1", "Shutdown request from user"); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java index 3f8c1a987056..12be2ceafa61 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java @@ -119,6 +119,11 @@ public class OverlordTest private final String goodTaskId = "aaa"; private final String badTaskId = "zzz"; + private Task task0; + private Task task1; + private String taskId0; + private String taskId1; + private void setupServerAndCurator() throws Exception { server = new TestingServer(); @@ -163,12 +168,18 @@ public void setUp() throws Exception taskLockbox = new TaskLockbox(taskStorage, mdc); + task0 = NoopTask.create(); + taskId0 = task0.getId(); + + task1 = NoopTask.create(); + taskId1 = task1.getId(); + runTaskCountDownLatches = new HashMap<>(); - runTaskCountDownLatches.put("0", new CountDownLatch(1)); - runTaskCountDownLatches.put("1", new CountDownLatch(1)); + runTaskCountDownLatches.put(taskId0, new CountDownLatch(1)); + runTaskCountDownLatches.put(taskId1, new CountDownLatch(1)); taskCompletionCountDownLatches = new HashMap<>(); - taskCompletionCountDownLatches.put("0", new CountDownLatch(1)); - taskCompletionCountDownLatches.put("1", new CountDownLatch(1)); + taskCompletionCountDownLatches.put(taskId0, new CountDownLatch(1)); + taskCompletionCountDownLatches.put(taskId1, new CountDownLatch(1)); announcementLatch = new CountDownLatch(1); setupServerAndCurator(); curator.start(); @@ -178,9 +189,9 @@ public void setUp() throws Exception // Add two tasks with conflicting locks // The bad task (The one with a lexicographically larger name) must be failed - Task badTask = new NoopTask(badTaskId, badTaskId, "datasource", 10_000, 0, null, null, null); + Task badTask = new NoopTask(badTaskId, badTaskId, "datasource", 10_000, 0, null); TaskLock badLock = new TimeChunkLock(null, badTaskId, "datasource", Intervals.ETERNITY, "version1", 50); - Task goodTask = new NoopTask(goodTaskId, goodTaskId, "datasource", 0, 0, null, null, null); + Task goodTask = new NoopTask(goodTaskId, goodTaskId, "datasource", 0, 0, null); TaskLock goodLock = new TimeChunkLock(null, goodTaskId, "datasource", Intervals.ETERNITY, "version0", 50); taskStorage.insert(goodTask, TaskStatus.running(goodTaskId)); taskStorage.insert(badTask, TaskStatus.running(badTaskId)); @@ -271,57 +282,53 @@ public void testOverlordRun() throws Exception taskCompletionCountDownLatches.get(goodTaskId).countDown(); waitForTaskStatus(goodTaskId, TaskState.SUCCESS); - final String taskId_0 = "0"; - NoopTask task_0 = NoopTask.create(taskId_0, 0); - response = overlordResource.taskPost(task_0, req); + response = overlordResource.taskPost(task0, req); Assert.assertEquals(200, response.getStatus()); - Assert.assertEquals(ImmutableMap.of("task", taskId_0), response.getEntity()); + Assert.assertEquals(ImmutableMap.of("task", taskId0), response.getEntity()); // Duplicate task - should fail - response = overlordResource.taskPost(task_0, req); + response = overlordResource.taskPost(task0, req); Assert.assertEquals(400, response.getStatus()); // Task payload for task_0 should be present in taskStorage - response = overlordResource.getTaskPayload(taskId_0); - Assert.assertEquals(task_0, ((TaskPayloadResponse) response.getEntity()).getPayload()); + response = overlordResource.getTaskPayload(taskId0); + Assert.assertEquals(task0, ((TaskPayloadResponse) response.getEntity()).getPayload()); // Task not present in taskStorage - should fail response = overlordResource.getTaskPayload("whatever"); Assert.assertEquals(404, response.getStatus()); // Task status of the submitted task should be running - response = overlordResource.getTaskStatus(taskId_0); - Assert.assertEquals(taskId_0, ((TaskStatusResponse) response.getEntity()).getTask()); + response = overlordResource.getTaskStatus(taskId0); + Assert.assertEquals(taskId0, ((TaskStatusResponse) response.getEntity()).getTask()); Assert.assertEquals( - TaskStatus.running(taskId_0).getStatusCode(), + TaskStatus.running(taskId0).getStatusCode(), ((TaskStatusResponse) response.getEntity()).getStatus().getStatusCode() ); // Simulate completion of task_0 - taskCompletionCountDownLatches.get(taskId_0).countDown(); + taskCompletionCountDownLatches.get(taskId0).countDown(); // Wait for taskQueue to handle success status of task_0 - waitForTaskStatus(taskId_0, TaskState.SUCCESS); + waitForTaskStatus(taskId0, TaskState.SUCCESS); // Manually insert task in taskStorage // Verifies sync from storage - final String taskId_1 = "1"; - NoopTask task_1 = NoopTask.create(taskId_1, 0); - taskStorage.insert(task_1, TaskStatus.running(taskId_1)); + taskStorage.insert(task1, TaskStatus.running(taskId1)); // Wait for task runner to run task_1 - runTaskCountDownLatches.get(taskId_1).await(); + runTaskCountDownLatches.get(taskId1).await(); response = overlordResource.getRunningTasks(null, req); // 1 task that was manually inserted should be in running state Assert.assertEquals(1, (((List) response.getEntity()).size())); final TaskStatusPlus taskResponseObject = ((List) response .getEntity()).get(0); - Assert.assertEquals(taskId_1, taskResponseObject.getId()); + Assert.assertEquals(taskId1, taskResponseObject.getId()); Assert.assertEquals(TASK_LOCATION, taskResponseObject.getLocation()); // Simulate completion of task_1 - taskCompletionCountDownLatches.get(taskId_1).countDown(); + taskCompletionCountDownLatches.get(taskId1).countDown(); // Wait for taskQueue to handle success status of task_1 - waitForTaskStatus(taskId_1, TaskState.SUCCESS); + waitForTaskStatus(taskId1, TaskState.SUCCESS); // should return number of tasks which are not in running state response = overlordResource.getCompleteTasks(null, req); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithAffinityWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithAffinityWorkerSelectStrategyTest.java index 05d1bd0ed76f..0641d3a8d919 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithAffinityWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithAffinityWorkerSelectStrategyTest.java @@ -43,14 +43,7 @@ public void testFindWorkerForTask() new AffinityConfig(ImmutableMap.of("foo", ImmutableSet.of("localhost1", "localhost2", "localhost3")), false) ); - NoopTask noopTask = new NoopTask(null, null, null, 1, 0, null, null, null) - { - @Override - public String getDataSource() - { - return "foo"; - } - }; + NoopTask noopTask = NoopTask.forDatasource("foo"); ImmutableWorkerInfo worker = strategy.findWorkerForTask( new RemoteTaskRunnerConfig(), ImmutableMap.of( @@ -113,7 +106,7 @@ public void testFindWorkerForTaskWithNulls() DateTimes.nowUtc() ) ), - new NoopTask(null, null, null, 1, 0, null, null, null) + NoopTask.create() ); Assert.assertEquals("lhost", worker.getWorker().getHost()); } @@ -136,7 +129,7 @@ public void testIsolation() DateTimes.nowUtc() ) ), - new NoopTask(null, null, null, 1, 0, null, null, null) + NoopTask.create() ); Assert.assertNull(worker); } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithCategorySpecWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithCategorySpecWorkerSelectStrategyTest.java index 80366ea03ad6..99a690a738df 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithCategorySpecWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithCategorySpecWorkerSelectStrategyTest.java @@ -182,7 +182,7 @@ private ImmutableWorkerInfo selectWorker(WorkerCategorySpec workerCategorySpec) ImmutableWorkerInfo worker = strategy.findWorkerForTask( new RemoteTaskRunnerConfig(), WORKERS_FOR_TIER_TESTS, - new NoopTask(null, null, "ds1", 1, 0, null, null, null) + NoopTask.forDatasource("ds1") ); return worker; diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java index 20259c3a52ed..dac25a3c9e34 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java @@ -89,14 +89,7 @@ public void testFindWorkerForTask() DateTimes.nowUtc() ) ), - new NoopTask(null, null, null, 1, 0, null, null, null) - { - @Override - public String getDataSource() - { - return "foo"; - } - } + NoopTask.forDatasource("foo") ); Assert.assertEquals("lhost", worker.getWorker().getHost()); } @@ -124,14 +117,7 @@ public void testFindWorkerForTaskWhenSameCurrCapacityUsed() DateTimes.nowUtc() ) ), - new NoopTask(null, null, null, 1, 0, null, null, null) - { - @Override - public String getDataSource() - { - return "foo"; - } - } + NoopTask.forDatasource("foo") ); Assert.assertEquals("localhost", worker.getWorker().getHost()); } @@ -160,14 +146,7 @@ public void testOneDisableWorkerDifferentUsedCapacity() DateTimes.nowUtc() ) ), - new NoopTask(null, null, null, 1, 0, null, null, null) - { - @Override - public String getDataSource() - { - return "foo"; - } - } + NoopTask.forDatasource("foo") ); Assert.assertEquals("enableHost", worker.getWorker().getHost()); } @@ -196,14 +175,7 @@ public void testOneDisableWorkerSameUsedCapacity() DateTimes.nowUtc() ) ), - new NoopTask(null, null, null, 1, 0, null, null, null) - { - @Override - public String getDataSource() - { - return "foo"; - } - } + NoopTask.forDatasource("foo") ); Assert.assertEquals("enableHost", worker.getWorker().getHost()); } @@ -282,13 +254,6 @@ public void testStrongAffinity() private static NoopTask createDummyTask(final String dataSource) { - return new NoopTask(null, null, null, 1, 0, null, null, null) - { - @Override - public String getDataSource() - { - return dataSource; - } - }; + return NoopTask.forDatasource(dataSource); } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithAffinityWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithAffinityWorkerSelectStrategyTest.java index efdcb8235565..93f6c39b0f0c 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithAffinityWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithAffinityWorkerSelectStrategyTest.java @@ -59,14 +59,7 @@ public void testFindWorkerForTask() DateTimes.nowUtc() ) ), - new NoopTask(null, null, null, 1, 0, null, null, null) - { - @Override - public String getDataSource() - { - return "foo"; - } - } + NoopTask.forDatasource("foo") ); Assert.assertEquals("localhost", worker.getWorker().getHost()); } @@ -96,7 +89,7 @@ public void testFindWorkerForTaskWithNulls() DateTimes.nowUtc() ) ), - new NoopTask(null, null, null, 1, 0, null, null, null) + NoopTask.create() ); Assert.assertEquals("lhost", worker.getWorker().getHost()); } @@ -119,7 +112,7 @@ public void testIsolation() DateTimes.nowUtc() ) ), - new NoopTask(null, null, null, 1, 0, null, null, null) + NoopTask.create() ); Assert.assertNull(worker); } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithCategorySpecWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithCategorySpecWorkerSelectStrategyTest.java index 8968a7538555..bb4795d029e1 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithCategorySpecWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithCategorySpecWorkerSelectStrategyTest.java @@ -182,7 +182,7 @@ private ImmutableWorkerInfo selectWorker(WorkerCategorySpec workerCategorySpec) ImmutableWorkerInfo worker = strategy.findWorkerForTask( new RemoteTaskRunnerConfig(), WORKERS_FOR_TIER_TESTS, - new NoopTask(null, null, "ds1", 1, 0, null, null, null) + NoopTask.forDatasource("ds1") ); return worker; diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/worker/WorkerTaskManagerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/worker/WorkerTaskManagerTest.java index d36bf1c04bab..93c5635492da 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/worker/WorkerTaskManagerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/worker/WorkerTaskManagerTest.java @@ -295,7 +295,7 @@ public void testTaskRun() throws Exception @Test(timeout = 30_000L) public void testTaskStatusWhenTaskRunnerFutureThrowsException() throws Exception { - Task task = new NoopTask("id", null, null, 100, 0, null, null, ImmutableMap.of(Tasks.PRIORITY_KEY, 0)) + Task task = new NoopTask("id", null, null, 100, 0, ImmutableMap.of(Tasks.PRIORITY_KEY, 0)) { @Override public TaskStatus runTask(TaskToolbox toolbox) @@ -444,7 +444,7 @@ public void test_completedTasksCleanup_ioException() throws Exception private NoopTask createNoopTask(String id) { - return new NoopTask(id, null, null, 100, 0, null, null, ImmutableMap.of(Tasks.PRIORITY_KEY, 0)); + return new NoopTask(id, null, null, 100, 0, ImmutableMap.of(Tasks.PRIORITY_KEY, 0)); } /** @@ -456,7 +456,7 @@ private Task setUpCompletedTasksCleanupTest() throws Exception EasyMock.expect(overlordClient.withRetryPolicy(EasyMock.anyObject())).andReturn(overlordClient).anyTimes(); EasyMock.replay(overlordClient); - final Task task = new NoopTask("id", null, null, 100, 0, null, null, ImmutableMap.of(Tasks.PRIORITY_KEY, 0)); + final Task task = new NoopTask("id", null, null, 100, 0, ImmutableMap.of(Tasks.PRIORITY_KEY, 0)); // Scheduled scheduleCompletedTasksCleanup will not run, because initialDelay is 1 minute, which is longer than // the 30-second timeout of this test case. From 9d6ca61ac1f00fcd23ba165ce2af4640b15e9df3 Mon Sep 17 00:00:00 2001 From: Abhishek Radhakrishnan Date: Tue, 5 Sep 2023 07:34:22 -0700 Subject: [PATCH 019/258] Verify statsd mock client interaction in unit test (#14939) --- .../emitter/statsd/StatsDEmitterTest.java | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/StatsDEmitterTest.java b/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/StatsDEmitterTest.java index 79d95e83f0e2..05a185c8b7e7 100644 --- a/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/StatsDEmitterTest.java +++ b/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/StatsDEmitterTest.java @@ -47,12 +47,12 @@ public void testConvertRange() new ObjectMapper(), client ); - client.gauge("broker.query.cache.total.hitRate", 54); emitter.emit(new ServiceMetricEvent.Builder() .setDimension("dataSource", "data-source") .setMetric("query/cache/total/hitRate", 0.54) .build("broker", "brokerHost1") ); + verify(client).gauge("broker.query.cache.total.hitRate", 54); } @Test @@ -64,12 +64,12 @@ public void testConvertRangeWithDogstatsd() new ObjectMapper(), client ); - client.gauge("broker.query.cache.total.hitRate", 0.54); emitter.emit(new ServiceMetricEvent.Builder() .setDimension("dataSource", "data-source") .setMetric("query/cache/total/hitRate", 0.54) .build("broker", "brokerHost1") ); + verify(client).gauge("broker.query.cache.total.hitRate", 0.54); } @Test @@ -81,7 +81,6 @@ public void testNoConvertRange() new ObjectMapper(), client ); - client.time("broker.query.time.data-source.groupBy", 10); emitter.emit(new ServiceMetricEvent.Builder() .setDimension("dataSource", "data-source") .setDimension("type", "groupBy") @@ -96,6 +95,7 @@ public void testNoConvertRange() .setMetric("query/time", 10) .build("broker", "brokerHost1") ); + verify(client).time("broker.query.time.data-source.groupBy", 10); } @Test @@ -107,7 +107,6 @@ public void testConfigOptions() new ObjectMapper(), client ); - client.time("brokerHost1#broker#query#time#data-source#groupBy", 10); emitter.emit(new ServiceMetricEvent.Builder() .setDimension("dataSource", "data-source") .setDimension("type", "groupBy") @@ -122,6 +121,7 @@ public void testConfigOptions() .setMetric("query/time", 10) .build("broker", "brokerHost1") ); + verify(client).time("brokerHost1#broker#query#time#data-source#groupBy", 10); } @Test @@ -133,9 +133,6 @@ public void testDogstatsdEnabled() new ObjectMapper(), client ); - client.time("broker#query#time", 10, - "dataSource:data-source", "type:groupBy", "hostname:brokerHost1" - ); emitter.emit(new ServiceMetricEvent.Builder() .setDimension("dataSource", "data-source") .setDimension("type", "groupBy") @@ -150,6 +147,8 @@ public void testDogstatsdEnabled() .setMetric("query/time", 10) .build("broker", "brokerHost1") ); + verify(client).time("broker#query#time", 10, + "dataSource:data-source", "type:groupBy", "hostname:brokerHost1"); } @Test @@ -161,12 +160,12 @@ public void testBlankHolderOptions() new ObjectMapper(), client ); - client.count("brokerHost1.broker.jvm.gc.count.G1-GC", 1); emitter.emit(new ServiceMetricEvent.Builder() .setDimension("gcName", "G1 GC") .setMetric("jvm/gc/count", 1) .build("broker", "brokerHost1") ); + verify(client).count("brokerHost1.broker.jvm.gc.count.G1-GC", 1); } @Test @@ -178,15 +177,14 @@ public void testServiceAsTagOption() new ObjectMapper(), client ); - client.time("druid.query.time", 10, - "druid_service:druid/broker", "dataSource:data-source", "type:groupBy", "hostname:brokerHost1" - ); emitter.emit(new ServiceMetricEvent.Builder() .setDimension("dataSource", "data-source") .setDimension("type", "groupBy") .setMetric("query/time", 10) .build("druid/broker", "brokerHost1") ); + verify(client).time("druid.query.time", 10, + "druid_service:druid/broker", "dataSource:data-source", "type:groupBy", "hostname:brokerHost1"); } @Test From 8088a763a69e54cca9b4c1d350e0b68b1eccd243 Mon Sep 17 00:00:00 2001 From: Soumyava <93540295+somu-imply@users.noreply.github.com> Date: Tue, 5 Sep 2023 08:41:42 -0700 Subject: [PATCH 020/258] Vectorize earliest aggregator for both numeric and string types (#14408) * Vectorizing earliest for numeric * Vectorizing earliest string aggregator * checkstyle fix * Removing unnecessary exceptions * Ignoring tests in MSQ as earliest is not supported for numeric there * Fixing benchmarks * Updating tests as MSQ does not support earliest for some cases * Addressing review comments by adding the following: 1. Checking capabilities first before creating selectors 2. Removing mockito in tests for numeric first aggs 3. Removing unnecessary tests * Addressing issues for dictionary encoded single string columns where we can use the dictionary ids instead of the entire string * Adding a flag for multi value dimension selector * Addressing comments * 1 more change * Handling review comments part 1 * Handling review comments and correctness fix for latest_by when the time expression need not be in sorted order * Updating numeric first vector agg * Revert "Updating numeric first vector agg" This reverts commit 429170990192883e51812311c49d2e461e6db732. * Updating code for correctness issues * fixing an issue with latest agg * Adding more comments and removing an unnecessary check * Addressing null checks for tie selector and only vectorize false for quantile sketches --- .../query/SqlExpressionBenchmark.java | 16 +- .../queries/wikipedia_editstream_queries.json | 18 +- .../first/DoubleFirstAggregatorFactory.java | 28 ++ .../first/DoubleFirstVectorAggregator.java | 61 +++++ .../first/FloatFirstAggregatorFactory.java | 25 ++ .../first/FloatFirstVectorAggregator.java | 61 +++++ .../first/LongFirstAggregatorFactory.java | 26 ++ .../first/LongFirstVectorAggregator.java | 60 +++++ .../first/NumericFirstVectorAggregator.java | 174 ++++++++++++ ...eStringFirstDimensionVectorAggregator.java | 125 +++++++++ .../first/StringFirstAggregatorFactory.java | 64 +++++ .../first/StringFirstVectorAggregator.java | 185 +++++++++++++ .../last/StringLastVectorAggregator.java | 4 +- .../DoubleFirstVectorAggregationTest.java | 246 +++++++++++++++++ .../FloatFirstVectorAggregationTest.java | 250 ++++++++++++++++++ .../first/LongFirstVectorAggregationTest.java | 241 +++++++++++++++++ .../StringFirstVectorAggregatorTest.java | 167 ++++++++++++ .../query/groupby/GroupByQueryRunnerTest.java | 9 - .../timeseries/TimeseriesQueryRunnerTest.java | 6 - .../druid/sql/calcite/CalciteQueryTest.java | 91 +++++-- 20 files changed, 1813 insertions(+), 44 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregator.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregator.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregator.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/first/SingleStringFirstDimensionVectorAggregator.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregator.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregationTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregationTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregationTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregatorTest.java diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java index 6f96a2cde568..498a9c2bdacd 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java @@ -197,7 +197,7 @@ public String getFormatString() "SELECT TIME_SHIFT(MILLIS_TO_TIMESTAMP(long4), 'PT1H', 1), string2, SUM(long1 * double4) FROM foo GROUP BY 1,2 ORDER BY 3", // 37: time shift + expr agg (group by), uniform distribution high cardinality "SELECT TIME_SHIFT(MILLIS_TO_TIMESTAMP(long5), 'PT1H', 1), string2, SUM(long1 * double4) FROM foo GROUP BY 1,2 ORDER BY 3", - // 38: LATEST aggregator + // 38: LATEST aggregator long "SELECT LATEST(long1) FROM foo", // 39: LATEST aggregator double "SELECT LATEST(double4) FROM foo", @@ -207,7 +207,13 @@ public String getFormatString() "SELECT LATEST(float3), LATEST(long1), LATEST(double4) FROM foo", // 42,43: filter numeric nulls "SELECT SUM(long5) FROM foo WHERE long5 IS NOT NULL", - "SELECT string2, SUM(long5) FROM foo WHERE long5 IS NOT NULL GROUP BY 1" + "SELECT string2, SUM(long5) FROM foo WHERE long5 IS NOT NULL GROUP BY 1", + // 44: EARLIEST aggregator long + "SELECT EARLIEST(long1) FROM foo", + // 45: EARLIEST aggregator double + "SELECT EARLIEST(double4) FROM foo", + // 46: EARLIEST aggregator float + "SELECT EARLIEST(float3) FROM foo" ); @Param({"5000000"}) @@ -265,7 +271,11 @@ public String getFormatString() "40", "41", "42", - "43" + "43", + "44", + "45", + "46", + "47" }) private String query; diff --git a/integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json b/integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json index 0d0290d23243..4cb4c0ec4857 100644 --- a/integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json +++ b/integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json @@ -119,7 +119,8 @@ } ], "context": { - "useCache": "true", + "useCache": "true", + "vectorize": "false", "populateCache": "true", "timeout": 360000 } @@ -270,7 +271,8 @@ } ], "context": { - "useCache": "true", + "useCache": "true", + "vectorize": "false", "populateCache": "true", "timeout": 360000 } @@ -514,7 +516,8 @@ "metric": "unique_users", "threshold": 3, "context": { - "useCache": "true", + "useCache": "true", + "vectorize": "false", "populateCache": "true", "timeout": 360000 } @@ -693,7 +696,8 @@ "metric": "count", "threshold": 3, "context": { - "useCache": "true", + "useCache": "true", + "vectorize": "false", "populateCache": "true", "timeout": 360000 } @@ -878,7 +882,8 @@ "metric": "count", "threshold": 3, "context": { - "useCache": "true", + "useCache": "true", + "vectorize": "false", "populateCache": "true", "timeout": 360000 } @@ -1243,7 +1248,8 @@ "orderBy": ["robot", "namespace"] }, "context": { - "useCache": "true", + "useCache": "true", + "vectorize": "false", "populateCache": "true", "timeout": 360000 } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index d3b0e4449530..7cee6f5ca64d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -29,14 +29,21 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.aggregation.any.NumericNilVectorAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseDoubleColumnValueSelector; +import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; +import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.Types; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -96,6 +103,12 @@ public DoubleFirstAggregatorFactory( this.storeDoubleAsFloat = ColumnHolder.storeDoubleAsFloat(); } + @Override + public boolean canVectorize(ColumnInspector columnInspector) + { + return true; + } + @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { @@ -124,6 +137,21 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) } } + @Override + public VectorAggregator factorizeVector( + VectorColumnSelectorFactory columnSelectorFactory + ) + { + ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); + if (Types.isNumeric(capabilities)) { + VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); + VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector( + timeColumn); + return new DoubleFirstVectorAggregator(timeSelector, valueSelector); + } + return NumericNilVectorAggregator.doubleNilVectorAggregator(); + } + @Override public Comparator getComparator() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregator.java new file mode 100644 index 000000000000..562f14547a6a --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregator.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.first; + +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.segment.vector.VectorValueSelector; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +public class DoubleFirstVectorAggregator extends NumericFirstVectorAggregator +{ + + public DoubleFirstVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + { + super(timeSelector, valueSelector); + } + + @Override + public void initValue(ByteBuffer buf, int position) + { + buf.putDouble(position, 0); + } + + + @Override + void putValue(ByteBuffer buf, int position, int index) + { + double firstValue = valueSelector.getDoubleVector()[index]; + buf.putDouble(position, firstValue); + } + + + /** + * @return The object as a pair with the position and the value stored at the position in the buffer. + */ + @Nullable + @Override + public Object get(ByteBuffer buf, int position) + { + final boolean rhsNull = isValueNull(buf, position); + return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getDouble(position + VALUE_OFFSET)); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 809cc8183ed5..68826bc2c0a1 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -29,14 +29,21 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.aggregation.any.NumericNilVectorAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseFloatColumnValueSelector; +import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; +import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.Types; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -122,6 +129,24 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) } } + @Override + public VectorAggregator factorizeVector(VectorColumnSelectorFactory columnSelectorFactory) + { + ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); + if (Types.isNumeric(capabilities)) { + VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); + VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); + return new FloatFirstVectorAggregator(timeSelector, valueSelector); + } + return NumericNilVectorAggregator.floatNilVectorAggregator(); + } + + @Override + public boolean canVectorize(ColumnInspector columnInspector) + { + return true; + } + @Override public Comparator getComparator() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregator.java new file mode 100644 index 000000000000..7c8c4e76f6b4 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregator.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.first; + +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.segment.vector.VectorValueSelector; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +public class FloatFirstVectorAggregator extends NumericFirstVectorAggregator +{ + + public FloatFirstVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + { + super(timeSelector, valueSelector); + } + + @Override + public void initValue(ByteBuffer buf, int position) + { + buf.putFloat(position, 0); + } + + + @Override + void putValue(ByteBuffer buf, int position, int index) + { + float firstValue = valueSelector.getFloatVector()[index]; + buf.putFloat(position, firstValue); + } + + + /** + * @return The object as a pair with the position and the value stored at the position in the buffer. + */ + @Nullable + @Override + public Object get(ByteBuffer buf, int position) + { + final boolean rhsNull = isValueNull(buf, position); + return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getFloat(position + VALUE_OFFSET)); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java index c767cf0e09c7..729a1bef26e2 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -29,14 +29,21 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.aggregation.any.NumericNilVectorAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; +import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.Types; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -121,6 +128,25 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) } } + @Override + public VectorAggregator factorizeVector(VectorColumnSelectorFactory columnSelectorFactory) + { + ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); + if (Types.isNumeric(capabilities)) { + VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); + VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector( + timeColumn); + return new LongFirstVectorAggregator(timeSelector, valueSelector); + } + return NumericNilVectorAggregator.longNilVectorAggregator(); + } + + @Override + public boolean canVectorize(ColumnInspector columnInspector) + { + return true; + } + @Override public Comparator getComparator() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregator.java new file mode 100644 index 000000000000..769b148b5e49 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregator.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.first; + +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.segment.vector.VectorValueSelector; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +public class LongFirstVectorAggregator extends NumericFirstVectorAggregator +{ + public LongFirstVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + { + super(timeSelector, valueSelector); + } + + @Override + public void initValue(ByteBuffer buf, int position) + { + buf.putLong(position, 0); + } + + + @Override + void putValue(ByteBuffer buf, int position, int index) + { + long firstValue = valueSelector.getLongVector()[index]; + buf.putLong(position, firstValue); + } + + + /** + * @return The object as a pair with the position and the value stored at the position in the buffer. + */ + @Nullable + @Override + public Object get(ByteBuffer buf, int position) + { + final boolean rhsNull = isValueNull(buf, position); + return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getLong(position + VALUE_OFFSET)); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java new file mode 100644 index 000000000000..7fcd10352da9 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.first; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.segment.vector.VectorValueSelector; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +/** + * Class for vectorized version of first/earliest aggregator over numeric types + */ +public abstract class NumericFirstVectorAggregator implements VectorAggregator +{ + static final int NULL_OFFSET = Long.BYTES; + static final int VALUE_OFFSET = NULL_OFFSET + Byte.BYTES; + final VectorValueSelector valueSelector; + private final boolean useDefault = NullHandling.replaceWithDefault(); + private final VectorValueSelector timeSelector; + private long firstTime; + + public NumericFirstVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + { + this.timeSelector = timeSelector; + this.valueSelector = valueSelector; + firstTime = Long.MAX_VALUE; + } + + @Override + public void init(ByteBuffer buf, int position) + { + buf.putLong(position, Long.MAX_VALUE); + buf.put(position + NULL_OFFSET, useDefault ? NullHandling.IS_NOT_NULL_BYTE : NullHandling.IS_NULL_BYTE); + initValue(buf, position + VALUE_OFFSET); + } + + @Override + public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) + { + final long[] timeVector = timeSelector.getLongVector(); + final boolean[] nullTimeVector = timeSelector.getNullVector(); + final boolean[] nullValueVector = valueSelector.getNullVector(); + firstTime = buf.getLong(position); + + // the time vector is already sorted + // if earliest is on the default time dimension + // but if earliest uses earliest_by it might use a secondary timestamp + // which is not sorted. For correctness, we need to go over all elements. + // A possible optimization here is to have 2 paths one for earliest where + // we can take advantage of the sorted nature of time + // and the earliest_by where we have to go over all elements. + int index; + + for (int i = startRow; i < endRow; i++) { + index = i; + if (nullTimeVector != null && nullTimeVector[index]) { + continue; + } + final long earliestTime = timeVector[index]; + if (earliestTime >= firstTime) { + continue; + } + firstTime = earliestTime; + if (useDefault || nullValueVector == null || !nullValueVector[index]) { + updateTimeWithValue(buf, position, firstTime, index); + } else { + updateTimeWithNull(buf, position, firstTime); + } + } + } + + /** + * + * Checks if the aggregated value at a position in the buffer is null or not + * + * @param buf byte buffer storing the byte array representation of the aggregate + * @param position offset within the byte buffer at which the current aggregate value is stored + * @return + */ + boolean isValueNull(ByteBuffer buf, int position) + { + return buf.get(position + NULL_OFFSET) == NullHandling.IS_NULL_BYTE; + } + + @Override + public void aggregate( + ByteBuffer buf, + int numRows, + int[] positions, + @Nullable int[] rows, + int positionOffset + ) + { + boolean[] nulls = useDefault ? null : valueSelector.getNullVector(); + long[] timeVector = timeSelector.getLongVector(); + + for (int i = 0; i < numRows; i++) { + int position = positions[i] + positionOffset; + int row = rows == null ? i : rows[i]; + long firstTime = buf.getLong(position); + if (timeVector[row] < firstTime) { + if (useDefault || nulls == null || !nulls[row]) { + updateTimeWithValue(buf, position, timeVector[row], row); + } else { + updateTimeWithNull(buf, position, timeVector[row]); + } + } + } + } + + /** + * Updates the time and the non null values to the appropriate position in buffer + * + * @param buf byte buffer storing the byte array representation of the aggregate + * @param position offset within the byte buffer at which the current aggregate value is stored + * @param time the time to be updated in the buffer as the last time + * @param index the index of the vectorized vector which is the last value + */ + void updateTimeWithValue(ByteBuffer buf, int position, long time, int index) + { + buf.putLong(position, time); + buf.put(position + NULL_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + putValue(buf, position + VALUE_OFFSET, index); + } + + /** + * Updates the time only to the appropriate position in buffer as the value is null + * + * @param buf byte buffer storing the byte array representation of the aggregate + * @param position offset within the byte buffer at which the current aggregate value is stored + * @param time the time to be updated in the buffer as the last time + */ + void updateTimeWithNull(ByteBuffer buf, int position, long time) + { + buf.putLong(position, time); + buf.put(position + NULL_OFFSET, NullHandling.IS_NULL_BYTE); + } + + /** + *Abstract function which needs to be overridden by subclasses to set the initial value + */ + abstract void initValue(ByteBuffer buf, int position); + + /** + * Abstract function which needs to be overridden by subclasses to set the + * latest value in the buffer depending on the datatype + */ + abstract void putValue(ByteBuffer buf, int position, int index); + + @Override + public void close() + { + // no resources to cleanup + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/SingleStringFirstDimensionVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/SingleStringFirstDimensionVectorAggregator.java new file mode 100644 index 000000000000..d5aa31444e0e --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/SingleStringFirstDimensionVectorAggregator.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.first; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.query.aggregation.SerializablePairLongString; +import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.segment.vector.BaseLongVectorValueSelector; +import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +public class SingleStringFirstDimensionVectorAggregator implements VectorAggregator +{ + private final BaseLongVectorValueSelector timeSelector; + private final SingleValueDimensionVectorSelector valueDimensionVectorSelector; + private long firstTime; + private final int maxStringBytes; + private final boolean useDefault = NullHandling.replaceWithDefault(); + + public SingleStringFirstDimensionVectorAggregator( + BaseLongVectorValueSelector timeSelector, + SingleValueDimensionVectorSelector valueDimensionVectorSelector, + int maxStringBytes + ) + { + this.timeSelector = timeSelector; + this.valueDimensionVectorSelector = valueDimensionVectorSelector; + this.maxStringBytes = maxStringBytes; + this.firstTime = Long.MAX_VALUE; + } + + @Override + public void init(ByteBuffer buf, int position) + { + buf.putLong(position, Long.MAX_VALUE); + buf.put( + position + NumericFirstVectorAggregator.NULL_OFFSET, + useDefault ? NullHandling.IS_NOT_NULL_BYTE : NullHandling.IS_NULL_BYTE + ); + buf.putLong(position + NumericFirstVectorAggregator.VALUE_OFFSET, 0); + } + + @Override + public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) + { + final long[] timeVector = timeSelector.getLongVector(); + final boolean[] nullTimeVector = timeSelector.getNullVector(); + final int[] valueVector = valueDimensionVectorSelector.getRowVector(); + firstTime = buf.getLong(position); + int index; + + long earliestTime; + for (index = startRow; index < endRow; index++) { + if (nullTimeVector != null && nullTimeVector[index]) { + continue; + } + earliestTime = timeVector[index]; + if (earliestTime < firstTime) { + firstTime = earliestTime; + buf.putLong(position, firstTime); + buf.put(position + NumericFirstVectorAggregator.NULL_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + buf.putInt(position + NumericFirstVectorAggregator.VALUE_OFFSET, valueVector[index]); + } + } + } + + @Override + public void aggregate(ByteBuffer buf, int numRows, int[] positions, @Nullable int[] rows, int positionOffset) + { + final long[] timeVector = timeSelector.getLongVector(); + final boolean[] nullTimeVector = timeSelector.getNullVector(); + final int[] values = valueDimensionVectorSelector.getRowVector(); + for (int i = 0; i < numRows; i++) { + if (nullTimeVector != null && nullTimeVector[i]) { + continue; + } + int position = positions[i] + positionOffset; + int row = rows == null ? i : rows[i]; + long firstTime = buf.getLong(position); + if (timeVector[row] < firstTime) { + firstTime = timeVector[row]; + buf.putLong(position, firstTime); + buf.put(position + NumericFirstVectorAggregator.NULL_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + buf.putInt(position + NumericFirstVectorAggregator.VALUE_OFFSET, values[row]); + } + } + + } + + @Nullable + @Override + public Object get(ByteBuffer buf, int position) + { + int index = buf.getInt(position + NumericFirstVectorAggregator.VALUE_OFFSET); + long earliest = buf.getLong(position); + String strValue = valueDimensionVectorSelector.lookupName(index); + return new SerializablePairLongString(earliest, StringUtils.chop(strValue, maxStringBytes)); + } + + @Override + public void close() + { + // nothing to close + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java index 63e48e878a84..33dafa861093 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java @@ -32,12 +32,21 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.aggregation.SerializablePairLongString; +import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.segment.BaseObjectColumnValueSelector; +import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.NilColumnValueSelector; +import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.vector.BaseLongVectorValueSelector; +import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -66,6 +75,7 @@ public void aggregate() } }; + private static final BufferAggregator NIL_BUFFER_AGGREGATOR = new StringFirstBufferAggregator( NilColumnValueSelector.instance(), NilColumnValueSelector.instance(), @@ -80,6 +90,25 @@ public void aggregate(ByteBuffer buf, int position) } }; + public static final VectorAggregator NIL_VECTOR_AGGREGATOR = new StringFirstVectorAggregator( + null, + null, + 0 + ) + { + @Override + public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) + { + // no-op + } + + @Override + public void aggregate(ByteBuffer buf, int numRows, int[] positions, @Nullable int[] rows, int positionOffset) + { + // no-op + } + }; + public static final int DEFAULT_MAX_STRING_SIZE = 1024; public static final Comparator TIME_COMPARATOR = (o1, o2) -> Longs.compare( @@ -121,6 +150,7 @@ public StringFirstAggregatorFactory( : maxStringBytes; } + @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { @@ -153,6 +183,40 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) } } + @Override + public VectorAggregator factorizeVector(VectorColumnSelectorFactory selectorFactory) + { + BaseLongVectorValueSelector timeSelector = (BaseLongVectorValueSelector) selectorFactory.makeValueSelector( + timeColumn); + ColumnCapabilities capabilities = selectorFactory.getColumnCapabilities(fieldName); + if (capabilities != null) { + if (capabilities.is(ValueType.STRING) && capabilities.isDictionaryEncoded().isTrue()) { + // Case 1: Single value string with dimension selector + // For multivalue string we need to iterate a list of indexedInts which is also similar to iterating + // over elements for an ARRAY typed column. These two which requires an iteration will be done together. + if (!capabilities.hasMultipleValues().isTrue()) { + SingleValueDimensionVectorSelector sSelector = selectorFactory.makeSingleValueDimensionSelector( + DefaultDimensionSpec.of( + fieldName)); + return new SingleStringFirstDimensionVectorAggregator(timeSelector, sSelector, maxStringBytes); + } + } + } + // Case 2: return vector object selector + VectorObjectSelector vSelector = selectorFactory.makeObjectSelector(fieldName); + if (capabilities != null) { + return new StringFirstVectorAggregator(timeSelector, vSelector, maxStringBytes); + } else { + return NIL_VECTOR_AGGREGATOR; + } + } + + @Override + public boolean canVectorize(ColumnInspector columnInspector) + { + return true; + } + @Override public Comparator getComparator() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregator.java new file mode 100644 index 000000000000..7082b4c3dd00 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregator.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.first; + +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.query.aggregation.SerializablePairLongString; +import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.segment.DimensionHandlerUtils; +import org.apache.druid.segment.vector.BaseLongVectorValueSelector; +import org.apache.druid.segment.vector.VectorObjectSelector; +import org.apache.druid.segment.vector.VectorValueSelector; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +public class StringFirstVectorAggregator implements VectorAggregator +{ + private static final SerializablePairLongString INIT = new SerializablePairLongString( + DateTimes.MAX.getMillis(), + null + ); + private final VectorValueSelector timeSelector; + private final VectorObjectSelector valueSelector; + private final int maxStringBytes; + + + public StringFirstVectorAggregator( + BaseLongVectorValueSelector timeSelector, + VectorObjectSelector valueSelector, + int maxStringBytes + ) + { + this.timeSelector = timeSelector; + this.valueSelector = valueSelector; + this.maxStringBytes = maxStringBytes; + } + + @Override + public void init(ByteBuffer buf, int position) + { + StringFirstLastUtils.writePair(buf, position, INIT, maxStringBytes); + } + + @Override + public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) + { + if (timeSelector == null) { + return; + } + final long[] times = timeSelector.getLongVector(); + final boolean[] nullTimeVector = timeSelector.getNullVector(); + final Object[] objectsWhichMightBeStrings = valueSelector.getObjectVector(); + long firstTime = buf.getLong(position); + int index; + for (int i = startRow; i < endRow; i++) { + if (times[i] > firstTime) { + continue; + } + if (nullTimeVector != null && nullTimeVector[i]) { + continue; + } + index = i; + final boolean foldNeeded = StringFirstLastUtils.objectNeedsFoldCheck(objectsWhichMightBeStrings[index]); + if (foldNeeded) { + final SerializablePairLongString inPair = StringFirstLastUtils.readPairFromVectorSelectorsAtIndex( + timeSelector, + valueSelector, + index + ); + if (inPair != null) { + firstTime = buf.getLong(position); + if (inPair.lhs < firstTime) { + StringFirstLastUtils.writePair( + buf, + position, + new SerializablePairLongString(inPair.lhs, inPair.rhs), + maxStringBytes + ); + } + } + } else { + final long time = times[index]; + if (time < firstTime) { + final String value = DimensionHandlerUtils.convertObjectToString(objectsWhichMightBeStrings[index]); + firstTime = time; + StringFirstLastUtils.writePair( + buf, + position, + new SerializablePairLongString(time, value), + maxStringBytes + ); + } + } + } + + } + + @Override + public void aggregate(ByteBuffer buf, int numRows, int[] positions, @Nullable int[] rows, int positionOffset) + { + final long[] timeVector = timeSelector.getLongVector(); + final boolean[] nullTimeVector = timeSelector.getNullVector(); + final Object[] objectsWhichMightBeStrings = valueSelector.getObjectVector(); + + // iterate once over the object vector to find first non null element and + // determine if the type is Pair or not + boolean foldNeeded = false; + for (Object obj : objectsWhichMightBeStrings) { + if (obj == null) { + continue; + } else { + foldNeeded = StringFirstLastUtils.objectNeedsFoldCheck(obj); + break; + } + } + + for (int i = 0; i < numRows; i++) { + if (nullTimeVector != null && nullTimeVector[i]) { + continue; + } + int position = positions[i] + positionOffset; + int row = rows == null ? i : rows[i]; + long firstTime = buf.getLong(position); + if (timeVector[row] < firstTime) { + if (foldNeeded) { + final SerializablePairLongString inPair = StringFirstLastUtils.readPairFromVectorSelectorsAtIndex( + timeSelector, + valueSelector, + row + ); + if (inPair != null) { + if (inPair.lhs < firstTime) { + StringFirstLastUtils.writePair( + buf, + position, + new SerializablePairLongString(inPair.lhs, inPair.rhs), + maxStringBytes + ); + } + } + } else { + final String value = DimensionHandlerUtils.convertObjectToString(objectsWhichMightBeStrings[row]); + firstTime = timeVector[row]; + StringFirstLastUtils.writePair( + buf, + position, + new SerializablePairLongString(firstTime, value), + maxStringBytes + ); + } + } + } + + } + + @Nullable + @Override + public Object get(ByteBuffer buf, int position) + { + return StringFirstLastUtils.readPair(buf, position); + } + + @Override + public void close() + { + // nothing to close + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregator.java index a9c0b1e9ade1..a18a1d4c9631 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregator.java @@ -73,8 +73,8 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) if (objectsWhichMightBeStrings[i] == null) { continue; } - if (times[i] < lastTime) { - break; + if (times[i] <= lastTime) { + continue; } index = i; final boolean foldNeeded = StringFirstLastUtils.objectNeedsFoldCheck(objectsWhichMightBeStrings[index]); diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregationTest.java new file mode 100644 index 000000000000..49c15fa22a51 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregationTest.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.first; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.dimension.DimensionSpec; +import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.vector.BaseDoubleVectorValueSelector; +import org.apache.druid.segment.vector.BaseLongVectorValueSelector; +import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector; +import org.apache.druid.segment.vector.NoFilterVectorOffset; +import org.apache.druid.segment.vector.ReadableVectorInspector; +import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; +import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.testing.InitializedNullHandlingTest; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.util.concurrent.ThreadLocalRandom; + +public class DoubleFirstVectorAggregationTest extends InitializedNullHandlingTest +{ + private static final double EPSILON = 1e-5; + private static final double[] VALUES = new double[]{7.8d, 11, 23.67, 60}; + private static final boolean[] NULLS = new boolean[]{false, false, true, false}; + private long[] times = {2436, 6879, 7888, 8224}; + + private static final String NAME = "NAME"; + private static final String FIELD_NAME = "FIELD_NAME"; + private static final String TIME_COL = "__time"; + + private VectorValueSelector selector; + + private BaseLongVectorValueSelector timeSelector; + private ByteBuffer buf; + + private DoubleFirstVectorAggregator target; + + private DoubleFirstAggregatorFactory doubleFirstAggregatorFactory; + + private VectorColumnSelectorFactory selectorFactory; + + @Before + public void setup() + { + byte[] randomBytes = new byte[1024]; + ThreadLocalRandom.current().nextBytes(randomBytes); + buf = ByteBuffer.wrap(randomBytes); + timeSelector = new BaseLongVectorValueSelector(new NoFilterVectorOffset(times.length, 0, times.length) + { + }) + { + @Override + public long[] getLongVector() + { + return times; + } + + @Nullable + @Override + public boolean[] getNullVector() + { + return NULLS; + } + }; + selector = new BaseDoubleVectorValueSelector(new NoFilterVectorOffset(VALUES.length, 0, VALUES.length) + { + + }) + { + @Override + public double[] getDoubleVector() + { + return VALUES; + } + + @Nullable + @Override + public boolean[] getNullVector() + { + if (!NullHandling.replaceWithDefault()) { + return NULLS; + } + return null; + } + }; + + target = new DoubleFirstVectorAggregator(timeSelector, selector); + clearBufferForPositions(0, 0); + selectorFactory = new VectorColumnSelectorFactory() + { + @Override + public ReadableVectorInspector getReadableVectorInspector() + { + return null; + } + + @Override + public SingleValueDimensionVectorSelector makeSingleValueDimensionSelector(DimensionSpec dimensionSpec) + { + return null; + } + + @Override + public MultiValueDimensionVectorSelector makeMultiValueDimensionSelector(DimensionSpec dimensionSpec) + { + return null; + } + + @Override + public VectorValueSelector makeValueSelector(String column) + { + if (TIME_COL.equals(column)) { + return timeSelector; + } else if (FIELD_NAME.equals(column)) { + return selector; + } else { + return null; + } + } + + @Override + public VectorObjectSelector makeObjectSelector(String column) + { + return null; + } + + @Nullable + @Override + public ColumnCapabilities getColumnCapabilities(String column) + { + if (FIELD_NAME.equals(column)) { + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.DOUBLE); + } + return null; + } + }; + + doubleFirstAggregatorFactory = new DoubleFirstAggregatorFactory(NAME, FIELD_NAME, TIME_COL); + } + + @Test + public void testFactory() + { + Assert.assertTrue(doubleFirstAggregatorFactory.canVectorize(selectorFactory)); + VectorAggregator vectorAggregator = doubleFirstAggregatorFactory.factorizeVector(selectorFactory); + Assert.assertNotNull(vectorAggregator); + Assert.assertEquals(DoubleFirstVectorAggregator.class, vectorAggregator.getClass()); + } + + @Test + public void initValueShouldInitZero() + { + target.initValue(buf, 0); + double initVal = buf.getDouble(0); + Assert.assertEquals(0, initVal, EPSILON); + } + + @Test + public void aggregate() + { + target.aggregate(buf, 0, 0, VALUES.length); + Pair result = (Pair) target.get(buf, 0); + Assert.assertEquals(times[0], result.lhs.longValue()); + Assert.assertEquals(VALUES[0], result.rhs, EPSILON); + } + + @Test + public void aggregateWithNulls() + { + target.aggregate(buf, 0, 0, VALUES.length); + Pair result = (Pair) target.get(buf, 0); + Assert.assertEquals(times[0], result.lhs.longValue()); + Assert.assertEquals(VALUES[0], result.rhs, EPSILON); + } + + @Test + public void aggregateBatchWithoutRows() + { + int[] positions = new int[]{0, 43, 70}; + int positionOffset = 2; + clearBufferForPositions(positionOffset, positions); + target.aggregate(buf, 3, positions, null, positionOffset); + for (int i = 0; i < positions.length; i++) { + Pair result = (Pair) target.get(buf, positions[i] + positionOffset); + Assert.assertEquals(times[i], result.lhs.longValue()); + if (!NullHandling.replaceWithDefault() && NULLS[i]) { + Assert.assertNull(result.rhs); + } else { + Assert.assertEquals(VALUES[i], result.rhs, EPSILON); + } + } + } + + @Test + public void aggregateBatchWithRows() + { + int[] positions = new int[]{0, 43, 70}; + int[] rows = new int[]{3, 2, 0}; + int positionOffset = 2; + clearBufferForPositions(positionOffset, positions); + target.aggregate(buf, 3, positions, rows, positionOffset); + for (int i = 0; i < positions.length; i++) { + Pair result = (Pair) target.get(buf, positions[i] + positionOffset); + Assert.assertEquals(times[rows[i]], result.lhs.longValue()); + if (!NullHandling.replaceWithDefault() && NULLS[rows[i]]) { + Assert.assertNull(result.rhs); + } else { + Assert.assertEquals(VALUES[rows[i]], result.rhs, EPSILON); + } + } + } + + private void clearBufferForPositions(int offset, int... positions) + { + for (int position : positions) { + target.init(buf, offset + position); + } + } +} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregationTest.java new file mode 100644 index 000000000000..6b02037824a9 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregationTest.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.first; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.dimension.DimensionSpec; +import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.vector.BaseFloatVectorValueSelector; +import org.apache.druid.segment.vector.BaseLongVectorValueSelector; +import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector; +import org.apache.druid.segment.vector.NoFilterVectorOffset; +import org.apache.druid.segment.vector.ReadableVectorInspector; +import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; +import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.testing.InitializedNullHandlingTest; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.util.concurrent.ThreadLocalRandom; + +public class FloatFirstVectorAggregationTest extends InitializedNullHandlingTest +{ + private static final double EPSILON = 1e-5; + private static final float[] VALUES = new float[]{7.2f, 15.6f, 2.1f, 150.0f}; + private static final boolean[] NULLS = new boolean[]{false, false, true, false}; + private long[] times = {2436, 6879, 7888, 8224}; + + private static final String NAME = "NAME"; + private static final String FIELD_NAME = "FIELD_NAME"; + private static final String TIME_COL = "__time"; + + + private VectorValueSelector selector; + + private BaseLongVectorValueSelector timeSelector; + private ByteBuffer buf; + + private FloatFirstVectorAggregator target; + + private FloatFirstAggregatorFactory floatFirstAggregatorFactory; + + private VectorColumnSelectorFactory selectorFactory; + + @Before + public void setup() + { + byte[] randomBytes = new byte[1024]; + ThreadLocalRandom.current().nextBytes(randomBytes); + buf = ByteBuffer.wrap(randomBytes); + timeSelector = new BaseLongVectorValueSelector(new NoFilterVectorOffset(times.length, 0, times.length) + { + }) + { + @Override + public long[] getLongVector() + { + return times; + } + + @Nullable + @Override + public boolean[] getNullVector() + { + return NULLS; + } + }; + selector = new BaseFloatVectorValueSelector(new NoFilterVectorOffset(VALUES.length, 0, VALUES.length) + { + + }) + { + + @Override + public float[] getFloatVector() + { + return VALUES; + } + + @Nullable + @Override + public boolean[] getNullVector() + { + if (!NullHandling.replaceWithDefault()) { + return NULLS; + } + return null; + } + }; + + target = new FloatFirstVectorAggregator(timeSelector, selector); + clearBufferForPositions(0, 0); + + selectorFactory = new VectorColumnSelectorFactory() + { + @Override + public ReadableVectorInspector getReadableVectorInspector() + { + return null; + } + + @Override + public SingleValueDimensionVectorSelector makeSingleValueDimensionSelector(DimensionSpec dimensionSpec) + { + return null; + } + + @Override + public MultiValueDimensionVectorSelector makeMultiValueDimensionSelector(DimensionSpec dimensionSpec) + { + return null; + } + + @Override + public VectorValueSelector makeValueSelector(String column) + { + if (TIME_COL.equals(column)) { + return timeSelector; + } else if (FIELD_NAME.equals(column)) { + return selector; + } else { + return null; + } + } + + @Override + public VectorObjectSelector makeObjectSelector(String column) + { + return null; + } + + @Nullable + @Override + public ColumnCapabilities getColumnCapabilities(String column) + { + if (FIELD_NAME.equals(column)) { + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.FLOAT); + } + return null; + } + }; + floatFirstAggregatorFactory = new FloatFirstAggregatorFactory(NAME, FIELD_NAME, TIME_COL); + + } + + @Test + public void testFactory() + { + Assert.assertTrue(floatFirstAggregatorFactory.canVectorize(selectorFactory)); + VectorAggregator vectorAggregator = floatFirstAggregatorFactory.factorizeVector(selectorFactory); + Assert.assertNotNull(vectorAggregator); + Assert.assertEquals(FloatFirstVectorAggregator.class, vectorAggregator.getClass()); + } + + @Test + public void initValueShouldBeZero() + { + target.initValue(buf, 0); + float initVal = buf.getFloat(0); + Assert.assertEquals(0.0f, initVal, EPSILON); + } + + @Test + public void aggregate() + { + target.init(buf, 0); + target.aggregate(buf, 0, 0, VALUES.length); + Pair result = (Pair) target.get(buf, 0); + Assert.assertEquals(times[0], result.lhs.longValue()); + Assert.assertEquals(VALUES[0], result.rhs, EPSILON); + } + + @Test + public void aggregateWithNulls() + { + target.aggregate(buf, 0, 0, VALUES.length); + Pair result = (Pair) target.get(buf, 0); + Assert.assertEquals(times[0], result.lhs.longValue()); + Assert.assertEquals(VALUES[0], result.rhs, EPSILON); + } + + @Test + public void aggregateBatchWithoutRows() + { + int[] positions = new int[]{0, 43, 70}; + int positionOffset = 2; + clearBufferForPositions(positionOffset, positions); + target.aggregate(buf, 3, positions, null, positionOffset); + for (int i = 0; i < positions.length; i++) { + Pair result = (Pair) target.get(buf, positions[i] + positionOffset); + Assert.assertEquals(times[i], result.lhs.longValue()); + if (!NullHandling.replaceWithDefault() && NULLS[i]) { + Assert.assertNull(result.rhs); + } else { + Assert.assertEquals(VALUES[i], result.rhs, EPSILON); + } + } + } + + @Test + public void aggregateBatchWithRows() + { + int[] positions = new int[]{0, 43, 70}; + int[] rows = new int[]{3, 2, 0}; + int positionOffset = 2; + clearBufferForPositions(positionOffset, positions); + target.aggregate(buf, 3, positions, rows, positionOffset); + for (int i = 0; i < positions.length; i++) { + Pair result = (Pair) target.get(buf, positions[i] + positionOffset); + Assert.assertEquals(times[rows[i]], result.lhs.longValue()); + if (!NullHandling.replaceWithDefault() && NULLS[rows[i]]) { + Assert.assertNull(result.rhs); + } else { + Assert.assertEquals(VALUES[rows[i]], result.rhs, EPSILON); + } + } + } + + private void clearBufferForPositions(int offset, int... positions) + { + for (int position : positions) { + target.init(buf, offset + position); + } + } +} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregationTest.java new file mode 100644 index 000000000000..ec4017600628 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregationTest.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.first; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.dimension.DimensionSpec; +import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.vector.BaseLongVectorValueSelector; +import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector; +import org.apache.druid.segment.vector.NoFilterVectorOffset; +import org.apache.druid.segment.vector.ReadableVectorInspector; +import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; +import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.testing.InitializedNullHandlingTest; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.util.concurrent.ThreadLocalRandom; + + +public class LongFirstVectorAggregationTest extends InitializedNullHandlingTest +{ + private static final double EPSILON = 1e-5; + private static final long[] VALUES = new long[]{7, 15, 2, 150}; + private static final boolean[] NULLS = new boolean[]{false, false, true, false}; + private static final String NAME = "NAME"; + private static final String FIELD_NAME = "FIELD_NAME"; + private static final String TIME_COL = "__time"; + private long[] times = {2436, 6879, 7888, 8224}; + private VectorValueSelector selector; + private BaseLongVectorValueSelector timeSelector; + private ByteBuffer buf; + private LongFirstVectorAggregator target; + + private LongFirstAggregatorFactory longFirstAggregatorFactory; + private VectorColumnSelectorFactory selectorFactory; + + @Before + public void setup() + { + byte[] randomBytes = new byte[1024]; + ThreadLocalRandom.current().nextBytes(randomBytes); + buf = ByteBuffer.wrap(randomBytes); + timeSelector = new BaseLongVectorValueSelector(new NoFilterVectorOffset(times.length, 0, times.length) + { + }) + { + @Override + public long[] getLongVector() + { + return times; + } + + @Nullable + @Override + public boolean[] getNullVector() + { + return NULLS; + } + }; + selector = new BaseLongVectorValueSelector(new NoFilterVectorOffset(VALUES.length, 0, VALUES.length) + { + + }) + { + @Override + public long[] getLongVector() + { + return VALUES; + } + + @Nullable + @Override + public boolean[] getNullVector() + { + if (!NullHandling.replaceWithDefault()) { + return NULLS; + } + return null; + } + }; + + target = new LongFirstVectorAggregator(timeSelector, selector); + clearBufferForPositions(0, 0); + + selectorFactory = new VectorColumnSelectorFactory() + { + @Override + public ReadableVectorInspector getReadableVectorInspector() + { + return null; + } + + @Override + public SingleValueDimensionVectorSelector makeSingleValueDimensionSelector(DimensionSpec dimensionSpec) + { + return null; + } + + @Override + public MultiValueDimensionVectorSelector makeMultiValueDimensionSelector(DimensionSpec dimensionSpec) + { + return null; + } + + @Override + public VectorValueSelector makeValueSelector(String column) + { + if (TIME_COL.equals(column)) { + return timeSelector; + } else if (FIELD_NAME.equals(column)) { + return selector; + } else { + return null; + } + } + + @Override + public VectorObjectSelector makeObjectSelector(String column) + { + return null; + } + + @Nullable + @Override + public ColumnCapabilities getColumnCapabilities(String column) + { + if (FIELD_NAME.equals(column)) { + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.LONG); + } + return null; + } + }; + longFirstAggregatorFactory = new LongFirstAggregatorFactory(NAME, FIELD_NAME, TIME_COL); + } + + @Test + public void testFactory() + { + Assert.assertTrue(longFirstAggregatorFactory.canVectorize(selectorFactory)); + VectorAggregator vectorAggregator = longFirstAggregatorFactory.factorizeVector(selectorFactory); + Assert.assertNotNull(vectorAggregator); + Assert.assertEquals(LongFirstVectorAggregator.class, vectorAggregator.getClass()); + } + + @Test + public void initValueShouldInitZero() + { + target.initValue(buf, 0); + long initVal = buf.getLong(0); + Assert.assertEquals(0, initVal); + } + + @Test + public void aggregate() + { + target.aggregate(buf, 0, 0, VALUES.length); + Pair result = (Pair) target.get(buf, 0); + Assert.assertEquals(times[0], result.lhs.longValue()); + Assert.assertEquals(VALUES[0], result.rhs, EPSILON); + } + + @Test + public void aggregateWithNulls() + { + target.aggregate(buf, 0, 0, VALUES.length); + Pair result = (Pair) target.get(buf, 0); + Assert.assertEquals(times[0], result.lhs.longValue()); + Assert.assertEquals(VALUES[0], result.rhs, EPSILON); + } + + @Test + public void aggregateBatchWithoutRows() + { + int[] positions = new int[]{0, 43, 70}; + int positionOffset = 2; + clearBufferForPositions(positionOffset, positions); + target.aggregate(buf, 3, positions, null, positionOffset); + for (int i = 0; i < positions.length; i++) { + Pair result = (Pair) target.get(buf, positions[i] + positionOffset); + Assert.assertEquals(times[i], result.lhs.longValue()); + if (!NullHandling.replaceWithDefault() && NULLS[i]) { + Assert.assertNull(result.rhs); + } else { + Assert.assertEquals(VALUES[i], result.rhs, EPSILON); + } + } + } + + @Test + public void aggregateBatchWithRows() + { + int[] positions = new int[]{0, 43, 70}; + int[] rows = new int[]{3, 2, 0}; + int positionOffset = 2; + clearBufferForPositions(positionOffset, positions); + target.aggregate(buf, 3, positions, rows, positionOffset); + for (int i = 0; i < positions.length; i++) { + Pair result = (Pair) target.get(buf, positions[i] + positionOffset); + Assert.assertEquals(times[rows[i]], result.lhs.longValue()); + if (!NullHandling.replaceWithDefault() && NULLS[rows[i]]) { + Assert.assertNull(result.rhs); + } else { + Assert.assertEquals(VALUES[rows[i]], result.rhs, EPSILON); + } + } + } + + private void clearBufferForPositions(int offset, int... positions) + { + for (int position : positions) { + target.init(buf, offset + position); + } + } +} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregatorTest.java new file mode 100644 index 000000000000..148f4f95937a --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregatorTest.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.first; + +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.SerializablePairLongString; +import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.segment.vector.BaseLongVectorValueSelector; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; +import org.apache.druid.testing.InitializedNullHandlingTest; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.nio.ByteBuffer; +import java.util.concurrent.ThreadLocalRandom; + + +@RunWith(MockitoJUnitRunner.class) +public class StringFirstVectorAggregatorTest extends InitializedNullHandlingTest +{ + private static final double EPSILON = 1e-5; + private static final String[] VALUES = new String[]{"a", "b", null, "c"}; + private static final boolean[] NULLS = new boolean[]{false, false, true, false}; + private static final String NAME = "NAME"; + private static final String FIELD_NAME = "FIELD_NAME"; + private static final String TIME_COL = "__time"; + private long[] times = {2436, 6879, 7888, 8224}; + private long[] timesSame = {2436, 2436}; + private SerializablePairLongString[] pairs = { + new SerializablePairLongString(2345001L, "first"), + new SerializablePairLongString(2345100L, "notFirst") + }; + + @Mock + private VectorObjectSelector selector; + @Mock + private VectorObjectSelector selectorForPairs; + @Mock + private BaseLongVectorValueSelector timeSelector; + @Mock + private BaseLongVectorValueSelector timeSelectorForPairs; + private ByteBuffer buf; + private StringFirstVectorAggregator target; + private StringFirstVectorAggregator targetWithPairs; + + private StringFirstAggregatorFactory stringFirstAggregatorFactory; + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private VectorColumnSelectorFactory selectorFactory; + + @Before + public void setup() + { + byte[] randomBytes = new byte[1024]; + ThreadLocalRandom.current().nextBytes(randomBytes); + buf = ByteBuffer.wrap(randomBytes); + Mockito.doReturn(VALUES).when(selector).getObjectVector(); + Mockito.doReturn(times).when(timeSelector).getLongVector(); + Mockito.doReturn(timesSame).when(timeSelectorForPairs).getLongVector(); + Mockito.doReturn(pairs).when(selectorForPairs).getObjectVector(); + target = new StringFirstVectorAggregator(timeSelector, selector, 10); + targetWithPairs = new StringFirstVectorAggregator(timeSelectorForPairs, selectorForPairs, 10); + clearBufferForPositions(0, 0); + + + Mockito.doReturn(selector).when(selectorFactory).makeObjectSelector(FIELD_NAME); + Mockito.doReturn(timeSelector).when(selectorFactory).makeValueSelector(TIME_COL); + stringFirstAggregatorFactory = new StringFirstAggregatorFactory(NAME, FIELD_NAME, TIME_COL, 10); + + } + + @Test + public void testAggregateWithPairs() + { + targetWithPairs.aggregate(buf, 0, 0, pairs.length); + Pair result = (Pair) targetWithPairs.get(buf, 0); + //Should come 0 as the last value as the left of the pair is greater + Assert.assertEquals(pairs[0].lhs.longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[0].rhs, result.rhs); + } + + @Test + public void testFactory() + { + Assert.assertTrue(stringFirstAggregatorFactory.canVectorize(selectorFactory)); + VectorAggregator vectorAggregator = stringFirstAggregatorFactory.factorizeVector(selectorFactory); + Assert.assertNotNull(vectorAggregator); + Assert.assertEquals(StringFirstVectorAggregator.class, vectorAggregator.getClass()); + } + + @Test + public void initValueShouldBeMaxDate() + { + target.init(buf, 0); + long initVal = buf.getLong(0); + Assert.assertEquals(DateTimes.MAX.getMillis(), initVal); + } + + @Test + public void aggregate() + { + target.aggregate(buf, 0, 0, VALUES.length); + Pair result = (Pair) target.get(buf, 0); + Assert.assertEquals(times[0], result.lhs.longValue()); + Assert.assertEquals(VALUES[0], result.rhs); + } + + @Test + public void aggregateBatchWithoutRows() + { + int[] positions = new int[]{0, 43, 70}; + int positionOffset = 2; + clearBufferForPositions(positionOffset, positions); + target.aggregate(buf, 3, positions, null, positionOffset); + for (int i = 0; i < positions.length; i++) { + Pair result = (Pair) target.get(buf, positions[i] + positionOffset); + Assert.assertEquals(times[i], result.lhs.longValue()); + Assert.assertEquals(VALUES[i], result.rhs); + } + } + + @Test + public void aggregateBatchWithRows() + { + int[] positions = new int[]{0, 43, 70}; + int[] rows = new int[]{3, 2, 0}; + int positionOffset = 2; + clearBufferForPositions(positionOffset, positions); + target.aggregate(buf, 3, positions, rows, positionOffset); + for (int i = 0; i < positions.length; i++) { + Pair result = (Pair) target.get(buf, positions[i] + positionOffset); + Assert.assertEquals(times[rows[i]], result.lhs.longValue()); + Assert.assertEquals(VALUES[rows[i]], result.rhs); + } + } + + private void clearBufferForPositions(int offset, int... positions) + { + for (int position : positions) { + target.init(buf, offset + position); + } + } +} diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java index 269067ff1f9b..033f64bcb9c8 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java @@ -3280,9 +3280,6 @@ public void testGroupByWithCardinality() @Test public void testGroupByWithFirstLast() { - // Cannot vectorize due to "first", "last" aggregators. - cannotVectorize(); - GroupByQuery query = makeQueryBuilder() .setDataSource(QueryRunnerTestHelper.DATA_SOURCE) .setQuerySegmentSpec(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC) @@ -3370,9 +3367,6 @@ public void testGroupByWithFirstLast() @Test public void testGroupByWithNoResult() { - // Cannot vectorize due to first, last aggregators. - cannotVectorize(); - GroupByQuery query = makeQueryBuilder() .setDataSource(QueryRunnerTestHelper.DATA_SOURCE) .setQuerySegmentSpec(QueryRunnerTestHelper.EMPTY_INTERVAL) @@ -7198,9 +7192,6 @@ public void testSubqueryWithHyperUniquesPostAggregator() @Test public void testSubqueryWithFirstLast() { - // Cannot vectorize due to "first", "last" aggregators. - cannotVectorize(); - GroupByQuery subquery = makeQueryBuilder() .setDataSource(QueryRunnerTestHelper.DATA_SOURCE) .setQuerySegmentSpec(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC) diff --git a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java index 8fee0e213fc5..a89103ac44a9 100644 --- a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java @@ -169,9 +169,6 @@ public TimeseriesQueryRunnerTest( @Test public void testEmptyTimeseries() { - // Cannot vectorize due to "doubleFirst" aggregator. - cannotVectorize(); - TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.DATA_SOURCE) .granularity(QueryRunnerTestHelper.ALL_GRAN) @@ -1960,9 +1957,6 @@ public void testTimeseriesWithMultiValueFilteringJavascriptAggregatorAndAlsoRegu @Test public void testTimeseriesWithFirstLastAggregator() { - // Cannot vectorize due to "doubleFirst", "doubleLast" aggregators. - cannotVectorize(); - TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.DATA_SOURCE) .granularity(QueryRunnerTestHelper.MONTH_GRAN) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index e1aab8de41c7..582408003719 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -630,8 +630,6 @@ public void testGroupBySingleColumnDescendingNoTopN() public void testEarliestAggregators() { notMsqCompatible(); - // Cannot vectorize EARLIEST aggregator. - skipVectorize(); testQuery( "SELECT " @@ -1096,8 +1094,6 @@ public void testStringLatestByGroupByWithAlwaysFalseCondition() public void testPrimitiveEarliestInSubquery() { notMsqCompatible(); - // Cannot vectorize EARLIEST aggregator. - skipVectorize(); testQuery( "SELECT SUM(val1), SUM(val2), SUM(val3) FROM (SELECT dim2, EARLIEST(m1) AS val1, EARLIEST(cnt) AS val2, EARLIEST(m2) AS val3 FROM foo GROUP BY dim2)", @@ -1195,11 +1191,8 @@ public void testStringLatestInSubquery() @Test public void testStringEarliestInSubquery() { - // Cannot vectorize EARLIEST aggregator. - skipVectorize(); - testQuery( - "SELECT SUM(val) FROM (SELECT dim2, EARLIEST(dim1, 10) AS val FROM foo GROUP BY dim2)", + "SELECT SUM(val) FROM (SELECT dim2, EARLIEST(dim1,10) AS val FROM foo GROUP BY dim2)", ImmutableList.of( GroupByQuery.builder() .setDataSource( @@ -1305,6 +1298,75 @@ public void testPrimitiveAnyInSubquery() ); } + @Test + public void testStringEarliestSingleStringDim() + { + notMsqCompatible(); + testQuery( + "SELECT dim2, EARLIEST(dim1,10) AS val FROM foo GROUP BY dim2", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE1) + .setDimensions(dimensions(new DefaultDimensionSpec("dim2", "d0"))) + .setAggregatorSpecs(aggregators(new StringFirstAggregatorFactory( + "a0", + "dim1", + null, + 10 + ))) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + NullHandling.sqlCompatible() ? + ImmutableList.of( + new Object[]{null, "10.1"}, + new Object[]{"", "2"}, + new Object[]{"a", ""}, + new Object[]{"abc", "def"} + ) : ImmutableList.of( + new Object[]{"", "10.1"}, + new Object[]{"a", ""}, + new Object[]{"abc", "def"} + ) + ); + } + + @Test + public void testStringEarliestMultiStringDim() + { + testQuery( + "SELECT dim2, EARLIEST(dim3,10) AS val FROM foo GROUP BY dim2", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE1) + .setDimensions(dimensions(new DefaultDimensionSpec("dim2", "d0"))) + .setAggregatorSpecs(aggregators(new StringFirstAggregatorFactory( + "a0", + "dim3", + null, + 10 + ))) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + NullHandling.sqlCompatible() ? + ImmutableList.of( + new Object[]{null, "[b, c]"}, + new Object[]{"", "d"}, + new Object[]{"a", "[a, b]"}, + new Object[]{"abc", null} + ) : ImmutableList.of( + new Object[]{"", "[b, c]"}, + new Object[]{"a", "[a, b]"}, + new Object[]{"abc", ""} + ) + ); + } + // This test the off-heap (buffer) version of the AnyAggregator (String) @Test public void testStringAnyInSubquery() @@ -1356,8 +1418,6 @@ public void testStringAnyInSubquery() public void testEarliestAggregatorsNumericNulls() { notMsqCompatible(); - // Cannot vectorize EARLIEST aggregator. - skipVectorize(); testQuery( "SELECT EARLIEST(l1), EARLIEST(d1), EARLIEST(f1) FROM druid.numfoo", @@ -1417,8 +1477,6 @@ public void testLatestAggregatorsNumericNull() public void testFirstLatestAggregatorsSkipNulls() { notMsqCompatible(); - // Cannot vectorize EARLIEST aggregator. - skipVectorize(); final DimFilter filter; if (useDefault) { @@ -1533,8 +1591,7 @@ public void testAnyAggregatorsSkipNullsWithFilter() public void testOrderByEarliestFloat() { notMsqCompatible(); - // Cannot vectorize EARLIEST aggregator. - skipVectorize(); + List expected; if (NullHandling.replaceWithDefault()) { expected = ImmutableList.of( @@ -1581,8 +1638,7 @@ public void testOrderByEarliestFloat() public void testOrderByEarliestDouble() { notMsqCompatible(); - // Cannot vectorize EARLIEST aggregator. - skipVectorize(); + List expected; if (NullHandling.replaceWithDefault()) { expected = ImmutableList.of( @@ -1629,8 +1685,7 @@ public void testOrderByEarliestDouble() public void testOrderByEarliestLong() { notMsqCompatible(); - // Cannot vectorize EARLIEST aggregator. - skipVectorize(); + List expected; if (NullHandling.replaceWithDefault()) { expected = ImmutableList.of( From 425ebaa387ce27fd46a54e23ce360abe82aa17dd Mon Sep 17 00:00:00 2001 From: Jill Osborne Date: Tue, 5 Sep 2023 22:16:01 +0100 Subject: [PATCH 021/258] Query tips doc (#14922) Co-authored-by: Katya Macedo <38017980+ektravel@users.noreply.github.com> Co-authored-by: Victoria Lim Co-authored-by: Katya Macedo <38017980+ektravel@users.noreply.github.com> --- docs/querying/tips-good-queries.md | 200 +++++++++++++++++++++++++++++ website/sidebars.json | 1 + 2 files changed, 201 insertions(+) create mode 100644 docs/querying/tips-good-queries.md diff --git a/docs/querying/tips-good-queries.md b/docs/querying/tips-good-queries.md new file mode 100644 index 000000000000..8b718d9b76ff --- /dev/null +++ b/docs/querying/tips-good-queries.md @@ -0,0 +1,200 @@ +--- +id: tips-good-queries +title: "Tips for writing good queries in Druid" +sidebar_label: "Tips for writing good queries" +--- + + + +This topic includes tips and examples that can help you investigate and improve query performance and accuracy using [Apache Druid SQL](./sql.md). Use this topic as a companion to the Jupyter Notebook tutorial [Learn the basics of Druid SQL](https://github.com/apache/druid/blob/master/examples/quickstart/jupyter-notebooks/notebooks/03-query/00-using-sql-with-druidapi.ipynb). + +Your ability to effectively query your data depends in large part on the way you've ingested and stored the data in Apache Druid. This document assumes that you've followed the best practices described in [Schema design tips and best practices](../ingestion/schema-design.md#general-tips-and-best-practices) when modeling your data. + +## Investigate query performance + +If your queries run slower than anticipated, you can use the following tools to investigate query performance issues. + +### Analyze query metrics + +You can configure Druid processes to emit metrics that are essential for monitoring query execution. See [Query metrics](../operations/metrics.md#query-metrics) for more information. + +### Generate an explain plan + +An explain plan shows the full query details and all of the operations Druid performs to execute it. You can use the information in the plan to identify possible areas of query improvement. + +See [Explain plan](./sql.md#explain-plan) and [Interpreting explain plan output](./sql-translation.md#interpreting-explain-plan-output) for more information. + +You can follow the [Get to know Query view tutorial](../tutorials/tutorial-sql-query-view.md) to create an example explain plan in the Druid console. + +## Improve query performance + +In most cases, you can improve query performance by adjusting Druid settings and by manually tuning your queries. + +### Adjust Druid settings + +This section outlines Druid settings that can help to improve query performance. + +#### Turn on query caching + +You can enable caching in Druid to improve query times for frequently accessed data. Caching enables increased concurrency on the same system, leading to noticeable performance improvements for queries handling throughput for concurrent, mixed workloads. + +The largest performance gains from caching tend to apply to TopN and timeseries queries. For GroupBy queries, if the bottleneck is in the merging phase on the Broker, enabling caching results in little noticeable query improvement. See [Performance considerations for caching](./caching.md#performance-considerations-for-caching) for more information. + +#### Use approximation + +When possible, design your SQL queries in such a way that they match the rules for TopN approximation, so that Druid enables TopN by default. For Druid to automatically optimize for TopN, your SQL query must include the following: + +- GROUP BY on one dimension, and +- ORDER BY on one aggregate. + + See [TopN queries](./topnquery.md) for more information. + +Note that TopN queries are approximate in that each data process ranks its top K results and only returns those top K results to the Broker. + +You can follow the tutorial [Using TopN approximation in Druid queries](https://github.com/apache/druid/blob/master/examples/quickstart/jupyter-notebooks/notebooks/03-query/02-approxRanking.ipynb) to work through some examples with approximation turned on and off. The tutorial [Get to know Query view](../tutorials/tutorial-sql-query-view.md) demonstrates running aggregate queries in the Druid console. + +### Manually tune your queries + +This section outlines techniques you can use to improve your query accuracy and performance. + +#### Query one table at a time + +Query a single table at a time to minimize the load on the Druid processor. + +#### Select specific columns + +Only select the columns needed for the query instead of retrieving all columns from the table. This reduces the amount of data retrieved from the database, which improves query performance. + +#### Use filters + +Use filters, for example the WHERE clause, and filter on time. Try to minimize the use of inequality filters, because they're very resource-intensive. + +The following example query filters on `__time` and `product`: + +``` +SELECT + FLOOR(__time to day), + product, + sum(quantity * price) as revenue +FROM "orders" +WHERE + __time > '2023-08-20' and product = 'product 1' +GROUP BY 1, 2 +``` + +The following example uses a wildcard filter on the `diffUrl` column: + +``` +SELECT * from Wikipedia +WHERE diffUrl LIKE 'https://en.wikipedia%' +AND TIME_IN_INTERVAL(__time, '2016-06-27T01:00:00/2016-06-27T02:00:00') +``` + +#### Shorten your queries + +Make your queries shorter where possible—Druid processes shorter queries faster. You might also be able to divide a single query into multiple queries. + +For example, the following query aggregates over multiple datasources using UNION ALL: + +``` +SELECT id, SUM(revenue) FROM + (SELECT id, revenue from datasource_1 +UNION ALL + SELECT id, revenue FROM datasource_2) +... +UNION ALL + SELECT id, revenue FROM datasource_n) +GROUP BY id +``` + +To simplify this query, you could split it into several queries, for example: + +``` +SELECT id, SUM(revenue) FROM datasource_1 + +SELECT id, SUM(revenue) FROM datasource_2 +... +SELECT id, SUM(revenue) FROM datasource_n +``` + +You could then manually aggregate the results of the individual queries. + +#### Minimize or remove subqueries + +Consider whether you can pre-compute a subquery task and store it as a join or make it a part of the datasource. See [Datasources: join](./datasource.md#join) and [SQL query translation: Joins](./sql-translation.md#joins) for more information and examples. + +#### Consider alternatives to GroupBy + +Consider using Timeseries and TopN as alternatives to GroupBy. See [GroupBy queries: alternatives](./groupbyquery.md#alternatives) for more information. + +Avoid grouping on high cardinality columns, for example user ID. Investigate whether you can apply a filter first, to reduce the number of results for grouping. + +#### Query over smaller intervals + +Consider whether you can query a smaller time interval to return a smaller results set. + +For example, the following query doesn't limit on time and could be resource-intensive: + +``` +SELECT cust_id, sum(revenue) FROM myDatasource +GROUP BY cust_id +``` + +This query could be split into multiple queries over smaller time spans, with the results combined client-side. For example: + +``` +SELECT cust_id, sum(revenue) FROM myDatasource +GROUP BY cust_id +WHERE __time BETWEEN '2023-07-01' AND '2023-07-31' + +SELECT cust_id, sum(revenue) FROM myDatasource +GROUP BY cust_id +WHERE __time BETWEEN '2023-08-01' AND '2023-08-31' +``` + +#### Reduce the computation in your queries + +Examine your query to see if it uses a lot of transformations, functions, and expressions. Consider whether you could rewrite the query to reduce the level of computation. + +## Druid SQL query example + +The following example query demonstrates many of the tips outlined in this topic. +The query: + +- selects specific dimensions and metrics +- uses approximation +- selects from a single table +- groups by low cardinality columns +- filters on both dimensions and time +- orders by a dimension and a measure +- includes a limit + +``` +SELECT + FLOOR() AS month, + country, + SUM(price), + APPROX_COUNT_DISTINCT_DS_HLL(userid) +FROM sales +GROUP BY month, country +WHERE artist = 'Madonna' AND TIME_IN_INTERVAL(__time, '2023-08-01/P1M') +ORDER BY country, SUM(price) DESC +LIMIT 100 +``` diff --git a/website/sidebars.json b/website/sidebars.json index b777d957e77c..1062b3dfee97 100644 --- a/website/sidebars.json +++ b/website/sidebars.json @@ -99,6 +99,7 @@ "label": "Druid SQL", "items": [ "querying/sql", + "querying/tips-good-queries", "querying/query-deep-storage", "querying/sql-data-types", "querying/sql-operators", From 706b57c0b2a80d7278a1456da605ec998f6e3b0b Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 5 Sep 2023 16:17:00 -0700 Subject: [PATCH 022/258] fixup array and mvd sql docs (#14928) --- docs/querying/sql-array-functions.md | 4 ++-- docs/querying/sql-multivalue-string-functions.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/querying/sql-array-functions.md b/docs/querying/sql-array-functions.md index b39c5d526bc3..89607e9f0852 100644 --- a/docs/querying/sql-array-functions.md +++ b/docs/querying/sql-array-functions.md @@ -56,8 +56,8 @@ The following table describes array functions. To learn more about array aggrega |`ARRAY_OVERLAP(arr1, arr2)`|Returns 1 if `arr1` and `arr2` have any elements in common, else 0.| |`ARRAY_OFFSET_OF(arr, expr)`|Returns the 0-based index of the first occurrence of `expr` in the array. If no matching elements exist in the array, returns `null` or `-1` if `druid.generic.useDefaultValueForNull=true` (legacy mode).| |`ARRAY_ORDINAL_OF(arr, expr)`|Returns the 1-based index of the first occurrence of `expr` in the array. If no matching elements exist in the array, returns `null` or `-1` if `druid.generic.useDefaultValueForNull=true` (legacy mode).| -|`ARRAY_PREPEND(expr, arr)`|Prepends `expr` to `arr` at the beginning, the resulting array type determined by the type of `arr`.| -|`ARRAY_APPEND(arr1, expr)`|Appends `expr` to `arr`, the resulting array type determined by the type of `arr1`.| +|`ARRAY_PREPEND(expr, arr)`|Adds `expr` to the beginning of `arr`, the resulting array type determined by the type of `arr`.| +|`ARRAY_APPEND(arr, expr)`|Appends `expr` to `arr`, the resulting array type determined by the type of `arr`.| |`ARRAY_CONCAT(arr1, arr2)`|Concatenates `arr2` to `arr1`. The resulting array type is determined by the type of `arr1`.| |`ARRAY_SLICE(arr, start, end)`|Returns the subarray of `arr` from the 0-based index `start` (inclusive) to `end` (exclusive). Returns `null`, if `start` is less than 0, greater than length of `arr`, or greater than `end`.| |`ARRAY_TO_STRING(arr, str)`|Joins all elements of `arr` by the delimiter specified by `str`.| diff --git a/docs/querying/sql-multivalue-string-functions.md b/docs/querying/sql-multivalue-string-functions.md index 9688ca083f3e..8b4a17c7b5ec 100644 --- a/docs/querying/sql-multivalue-string-functions.md +++ b/docs/querying/sql-multivalue-string-functions.md @@ -57,8 +57,8 @@ All array references in the multi-value string function documentation can refer |`MV_OVERLAP(arr1, arr2)`|Returns 1 if `arr1` and `arr2` have any elements in common, else 0.| |`MV_OFFSET_OF(arr, expr)`|Returns the 0-based index of the first occurrence of `expr` in the array. If no matching elements exist in the array, returns `null` or -1 if `druid.generic.useDefaultValueForNull=true` (legacy mode).| |`MV_ORDINAL_OF(arr, expr)`|Returns the 1-based index of the first occurrence of `expr` in the array. If no matching elements exist in the array, returns `null` or `-1` if `druid.generic.useDefaultValueForNull=true` (legacy mode).| -|`MV_PREPEND(expr, arr)`|Adds `expr` to `arr` at the beginning, the resulting array type determined by the type of the array.| -|`MV_APPEND(arr1, expr)`|Appends `expr` to `arr`, the resulting array type determined by the type of the first array.| +|`MV_PREPEND(expr, arr)`|Adds `expr` to the beginning of `arr`, the resulting array type determined by the type `arr`.| +|`MV_APPEND(arr, expr)`|Appends `expr` to `arr`, the resulting array type determined by the type of `arr`.| |`MV_CONCAT(arr1, arr2)`|Concatenates `arr2` to `arr1`. The resulting array type is determined by the type of `arr1`.| |`MV_SLICE(arr, start, end)`|Returns the subarray of `arr` from the 0-based index start(inclusive) to end(exclusive), or `null`, if start is less than 0, greater than length of arr or greater than end.| |`MV_TO_STRING(arr, str)`|Joins all elements of `arr` by the delimiter specified by `str`.| From 959148ad370f4133c59f24f8afc3f297b0911e41 Mon Sep 17 00:00:00 2001 From: Adarsh Sanjeev Date: Wed, 6 Sep 2023 10:35:57 +0530 Subject: [PATCH 023/258] Add code to wait for segments generated to be loaded on historicals (#14322) Currently, after an MSQ query, the web console is responsible for waiting for the segments to load. It does so by checking if there are any segments loading into the datasource ingested into, which can cause some issues, like in cases where the segments would never be loaded, or would end up waiting for other ingests as well. This PR shifts this responsibility to the controller, which would have the list of segments created. --- docs/api-reference/sql-ingestion-api.md | 24 +- .../apache/druid/msq/exec/ControllerImpl.java | 118 +++-- .../druid/msq/exec/SegmentLoadWaiter.java | 422 ++++++++++++++++++ .../msq/indexing/MSQWorkerTaskLauncher.java | 24 +- .../msq/indexing/report/MSQStatusReport.java | 16 +- .../druid/msq/exec/SegmentLoadWaiterTest.java | 95 ++++ .../indexing/report/MSQTaskReportTest.java | 44 +- .../resources/SqlStatementResourceTest.java | 6 +- .../apache/druid/msq/test/MSQTestBase.java | 14 +- .../apache/druid/discovery/BrokerClient.java | 125 ++++++ .../apache/druid/discovery/ClientUtils.java | 62 +++ .../druid/discovery/DruidLeaderClient.java | 36 +- .../druid/discovery/BrokerClientTest.java | 154 +++++++ .../discovery/DruidLeaderClientTest.java | 2 +- website/.spelling | 2 + 15 files changed, 1070 insertions(+), 74 deletions(-) create mode 100644 extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/SegmentLoadWaiter.java create mode 100644 extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/SegmentLoadWaiterTest.java create mode 100644 server/src/main/java/org/apache/druid/discovery/BrokerClient.java create mode 100644 server/src/main/java/org/apache/druid/discovery/ClientUtils.java create mode 100644 server/src/test/java/org/apache/druid/discovery/BrokerClientTest.java diff --git a/docs/api-reference/sql-ingestion-api.md b/docs/api-reference/sql-ingestion-api.md index 3109537c4a2a..3daadfa5085f 100644 --- a/docs/api-reference/sql-ingestion-api.md +++ b/docs/api-reference/sql-ingestion-api.md @@ -288,7 +288,19 @@ The response shows an example report for a query. "startTime": "2022-09-14T22:12:09.266Z", "durationMs": 28227, "pendingTasks": 0, - "runningTasks": 2 + "runningTasks": 2, + "segmentLoadStatus": { + "state": "SUCCESS", + "dataSource": "kttm_simple", + "startTime": "2022-09-14T23:12:09.266Z", + "duration": 15, + "totalSegments": 1, + "usedSegments": 1, + "precachedSegments": 0, + "onDemandSegments": 0, + "pendingSegments": 0, + "unknownSegments": 0 + } }, "stages": [ { @@ -593,6 +605,16 @@ The following table describes the response fields when you retrieve a report for | `multiStageQuery.payload.status.durationMs` | Milliseconds elapsed after the query has started running. -1 denotes that the query hasn't started running yet. | | `multiStageQuery.payload.status.pendingTasks` | Number of tasks that are not fully started. -1 denotes that the number is currently unknown. | | `multiStageQuery.payload.status.runningTasks` | Number of currently running tasks. Should be at least 1 since the controller is included. | +| `multiStageQuery.payload.status.segmentLoadStatus` | Segment loading container. Only present after the segments have been published. | +| `multiStageQuery.payload.status.segmentLoadStatus.state` | Either INIT, WAITING, SUCCESS, FAILED or TIMED_OUT. | +| `multiStageQuery.payload.status.segmentLoadStatus.startTime` | Time since which the controller has been waiting for the segments to finish loading. | +| `multiStageQuery.payload.status.segmentLoadStatus.duration` | The duration in milliseconds that the controller has been waiting for the segments to load. | +| `multiStageQuery.payload.status.segmentLoadStatus.totalSegments` | The total number of segments generated by the job. This includes tombstone segments (if any). | +| `multiStageQuery.payload.status.segmentLoadStatus.usedSegments` | The number of segments which are marked as used based on the load rules. Unused segments can be cleaned up at any time. | +| `multiStageQuery.payload.status.segmentLoadStatus.precachedSegments` | The number of segments which are marked as precached and served by historicals, as per the load rules. | +| `multiStageQuery.payload.status.segmentLoadStatus.onDemandSegments` | The number of segments which are not loaded on any historical, as per the load rules. | +| `multiStageQuery.payload.status.segmentLoadStatus.pendingSegments` | The number of segments remaining to be loaded. | +| `multiStageQuery.payload.status.segmentLoadStatus.unknownSegments` | The number of segments whose status is unknown. | | `multiStageQuery.payload.status.errorReport` | Error object. Only present if there was an error. | | `multiStageQuery.payload.status.errorReport.taskId` | The task that reported the error, if known. May be a controller task or a worker task. | | `multiStageQuery.payload.status.errorReport.host` | The hostname and port of the task that reported the error, if known. | diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java index d883a587e9b5..20033480f106 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java @@ -44,6 +44,7 @@ import org.apache.druid.data.input.impl.DimensionSchema; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.discovery.BrokerClient; import org.apache.druid.frame.allocation.ArenaMemoryAllocator; import org.apache.druid.frame.channel.FrameChannelSequence; import org.apache.druid.frame.key.ClusterBy; @@ -63,6 +64,7 @@ import org.apache.druid.indexing.common.TaskLockType; import org.apache.druid.indexing.common.TaskReport; import org.apache.druid.indexing.common.actions.LockListAction; +import org.apache.druid.indexing.common.actions.LockReleaseAction; import org.apache.druid.indexing.common.actions.MarkSegmentsAsUnusedAction; import org.apache.druid.indexing.common.actions.SegmentAllocateAction; import org.apache.druid.indexing.common.actions.SegmentTransactionalInsertAction; @@ -292,6 +294,7 @@ public class ControllerImpl implements Controller private WorkerMemoryParameters workerMemoryParameters; private boolean isDurableStorageEnabled; private boolean isFaultToleranceEnabled; + private volatile SegmentLoadWaiter segmentLoadWaiter; public ControllerImpl( final MSQControllerTask task, @@ -437,6 +440,45 @@ public TaskStatus runTask(final Closer closer) } } + if (queryKernel != null && queryKernel.isSuccess()) { + // If successful, encourage the tasks to exit successfully. + postFinishToAllTasks(); + workerTaskLauncher.stop(false); + } else { + // If not successful, cancel running tasks. + if (workerTaskLauncher != null) { + workerTaskLauncher.stop(true); + } + } + + // Wait for worker tasks to exit. Ignore their return status. At this point, we've done everything we need to do, + // so we don't care about the task exit status. + if (workerTaskRunnerFuture != null) { + try { + workerTaskRunnerFuture.get(); + } + catch (Exception ignored) { + // Suppress. + } + } + + + try { + releaseTaskLocks(); + + cleanUpDurableStorageIfNeeded(); + + if (queryKernel != null && queryKernel.isSuccess()) { + if (segmentLoadWaiter != null) { + // If successful and there are segments created, segmentLoadWaiter should wait for them to become available. + segmentLoadWaiter.waitForSegmentsToLoad(); + } + } + } + catch (Exception e) { + log.warn(e, "Exception thrown during cleanup. Ignoring it and writing task report."); + } + try { // Write report even if something went wrong. final MSQStagesReport stagesReport; @@ -488,7 +530,8 @@ public TaskStatus runTask(final Closer closer) workerWarnings, queryStartTime, new Interval(queryStartTime, DateTimes.nowUtc()).toDurationMillis(), - workerTaskLauncher + workerTaskLauncher, + segmentLoadWaiter ), stagesReport, countersSnapshot, @@ -504,30 +547,6 @@ public TaskStatus runTask(final Closer closer) log.warn(e, "Error encountered while writing task report. Skipping."); } - if (queryKernel != null && queryKernel.isSuccess()) { - // If successful, encourage the tasks to exit successfully. - postFinishToAllTasks(); - workerTaskLauncher.stop(false); - } else { - // If not successful, cancel running tasks. - if (workerTaskLauncher != null) { - workerTaskLauncher.stop(true); - } - } - - // Wait for worker tasks to exit. Ignore their return status. At this point, we've done everything we need to do, - // so we don't care about the task exit status. - if (workerTaskRunnerFuture != null) { - try { - workerTaskRunnerFuture.get(); - } - catch (Exception ignored) { - // Suppress. - } - } - - cleanUpDurableStorageIfNeeded(); - if (taskStateForReport == TaskState.SUCCESS) { return TaskStatus.success(id()); } else { @@ -536,6 +555,23 @@ public TaskStatus runTask(final Closer closer) } } + /** + * Releases the locks obtained by the task. + */ + private void releaseTaskLocks() throws IOException + { + final List locks; + try { + locks = context.taskActionClient().submit(new LockListAction()); + for (final TaskLock lock : locks) { + context.taskActionClient().submit(new LockReleaseAction(lock.getInterval())); + } + } + catch (IOException e) { + throw new IOException("Failed to release locks", e); + } + } + /** * Adds some logic to {@link #kernelManipulationQueue}, where it will, in due time, be executed by the main * controller loop in {@link RunQueryUntilDone#run()}. @@ -875,7 +911,8 @@ public Map liveReports() workerWarnings, queryStartTime, queryStartTime == null ? -1L : new Interval(queryStartTime, DateTimes.nowUtc()).toDurationMillis(), - workerTaskLauncher + workerTaskLauncher, + segmentLoadWaiter ), makeStageReport( queryDef, @@ -1316,17 +1353,36 @@ private void publishAllSegments(final Set segments) throws IOExcept if (segmentsWithTombstones.isEmpty()) { // Nothing to publish, only drop. We already validated that the intervalsToDrop do not have any // partially-overlapping segments, so it's safe to drop them as intervals instead of as specific segments. + // This should not need a segment load wait as segments are marked as unused immediately. for (final Interval interval : intervalsToDrop) { context.taskActionClient() .submit(new MarkSegmentsAsUnusedAction(task.getDataSource(), interval)); } } else { + Set versionsToAwait = segmentsWithTombstones.stream().map(DataSegment::getVersion).collect(Collectors.toSet()); + segmentLoadWaiter = new SegmentLoadWaiter( + context.injector().getInstance(BrokerClient.class), + context.jsonMapper(), + task.getDataSource(), + versionsToAwait, + segmentsWithTombstones.size(), + true + ); performSegmentPublish( context.taskActionClient(), SegmentTransactionalInsertAction.overwriteAction(null, segmentsWithTombstones) ); } } else if (!segments.isEmpty()) { + Set versionsToAwait = segments.stream().map(DataSegment::getVersion).collect(Collectors.toSet()); + segmentLoadWaiter = new SegmentLoadWaiter( + context.injector().getInstance(BrokerClient.class), + context.jsonMapper(), + task.getDataSource(), + versionsToAwait, + segments.size(), + true + ); // Append mode. performSegmentPublish( context.taskActionClient(), @@ -2072,7 +2128,8 @@ private static MSQStatusReport makeStatusReport( final Queue errorReports, @Nullable final DateTime queryStartTime, final long queryDuration, - MSQWorkerTaskLauncher taskLauncher + MSQWorkerTaskLauncher taskLauncher, + final SegmentLoadWaiter segmentLoadWaiter ) { int pendingTasks = -1; @@ -2083,6 +2140,9 @@ private static MSQStatusReport makeStatusReport( pendingTasks = workerTaskCount.getPendingWorkerCount(); runningTasks = workerTaskCount.getRunningWorkerCount() + 1; // To account for controller. } + + SegmentLoadWaiter.SegmentLoadWaiterStatus status = segmentLoadWaiter == null ? null : segmentLoadWaiter.status(); + return new MSQStatusReport( taskState, errorReport, @@ -2090,7 +2150,8 @@ private static MSQStatusReport makeStatusReport( queryStartTime, queryDuration, pendingTasks, - runningTasks + runningTasks, + status ); } @@ -2259,6 +2320,7 @@ private Pair> run() throws IOExceptio throwKernelExceptionIfNotUnknown(); } + updateLiveReportMaps(); cleanUpEffectivelyFinishedStages(); return Pair.of(queryKernel, workerTaskLauncherFuture); } diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/SegmentLoadWaiter.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/SegmentLoadWaiter.java new file mode 100644 index 000000000000..3a54c41e4104 --- /dev/null +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/SegmentLoadWaiter.java @@ -0,0 +1,422 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.msq.exec; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.discovery.BrokerClient; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.http.client.Request; +import org.apache.druid.sql.http.ResultFormat; +import org.apache.druid.sql.http.SqlQuery; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.joda.time.DateTime; +import org.joda.time.Interval; + +import javax.annotation.Nullable; +import javax.ws.rs.core.MediaType; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Class that periodically checks with the broker if all the segments generated are loaded by querying the sys table + * and blocks till it is complete. This will account for and not wait for segments that would never be loaded due to + * load rules. Should only be called if the query generates new segments or tombstones. + *
+ * If an exception is thrown during operation, this will simply log the exception and exit without failing the task, + * since the segments have already been published successfully, and should be loaded eventually. + *
+ * If the segments are not loaded within {@link #TIMEOUT_DURATION_MILLIS} milliseconds, this logs a warning and exits + * for the same reason. + */ +public class SegmentLoadWaiter +{ + private static final Logger log = new Logger(SegmentLoadWaiter.class); + private static final long SLEEP_DURATION_MILLIS = TimeUnit.SECONDS.toMillis(5); + private static final long TIMEOUT_DURATION_MILLIS = TimeUnit.MINUTES.toMillis(10); + + /** + * The query sent to the broker. This query uses replication_factor to determine how many copies of a segment has to be + * loaded as per the load rules. + * - If a segment is not used, the broker will not have any information about it, hence, a COUNT(*) should return the used count only. + * - If replication_factor is more than 0, the segment will be loaded on historicals and needs to be waited for. + * - If replication_factor is 0, that means that the segment will never be loaded on a historical and does not need to + * be waited for. + * - If replication_factor is -1, the replication factor is not known currently and will become known after a load rule + * evaluation. + *
+ * See https://github.com/apache/druid/pull/14403 for more details about replication_factor + */ + private static final String LOAD_QUERY = "SELECT COUNT(*) AS usedSegments,\n" + + "COUNT(*) FILTER (WHERE is_published = 1 AND replication_factor > 0) AS precachedSegments,\n" + + "COUNT(*) FILTER (WHERE is_published = 1 AND replication_factor = 0) AS onDemandSegments,\n" + + "COUNT(*) FILTER (WHERE is_available = 0 AND is_published = 1 AND replication_factor != 0) AS pendingSegments,\n" + + "COUNT(*) FILTER (WHERE replication_factor = -1) AS unknownSegments\n" + + "FROM sys.segments\n" + + "WHERE datasource = '%s' AND is_overshadowed = 0 AND version = '%s'"; + + private final BrokerClient brokerClient; + private final ObjectMapper objectMapper; + // Map of version vs latest load status. + private final Map versionToLoadStatusMap; + private final String datasource; + private final Set versionsToAwait; + private final int totalSegmentsGenerated; + private final boolean doWait; + private final AtomicReference status; + + public SegmentLoadWaiter( + BrokerClient brokerClient, + ObjectMapper objectMapper, + String datasource, + Set versionsToAwait, + int totalSegmentsGenerated, + boolean doWait + ) + { + this.brokerClient = brokerClient; + this.objectMapper = objectMapper; + this.datasource = datasource; + this.versionsToAwait = new TreeSet<>(versionsToAwait); + this.versionToLoadStatusMap = new HashMap<>(); + this.totalSegmentsGenerated = totalSegmentsGenerated; + this.status = new AtomicReference<>(new SegmentLoadWaiterStatus(State.INIT, null, 0, totalSegmentsGenerated, 0, 0, 0, 0, totalSegmentsGenerated)); + this.doWait = doWait; + } + + /** + * Uses broker client to check if all segments created by the ingestion have been loaded and updates the {@link #status)} + * periodically. + *
+ * If an exception is thrown during operation, this will log the exception and return without failing the task, + * since the segments have already been published successfully, and should be loaded eventually. + *
+ * Only expected to be called from the main controller thread. + */ + public void waitForSegmentsToLoad() + { + DateTime startTime = DateTimes.nowUtc(); + boolean hasAnySegmentBeenLoaded = false; + + try { + while (!versionsToAwait.isEmpty()) { + // Check the timeout and exit if exceeded. + long runningMillis = new Interval(startTime, DateTimes.nowUtc()).toDurationMillis(); + if (runningMillis > TIMEOUT_DURATION_MILLIS) { + log.warn("Runtime [%s] exceeded timeout [%s] while waiting for segments to load. Exiting.", runningMillis, TIMEOUT_DURATION_MILLIS); + updateStatus(State.TIMED_OUT, startTime); + return; + } + + Iterator iterator = versionsToAwait.iterator(); + + // Query the broker for all pending versions + while (iterator.hasNext()) { + String version = iterator.next(); + + // Fetch the load status for this version from the broker + VersionLoadStatus loadStatus = fetchLoadStatusForVersion(version); + versionToLoadStatusMap.put(version, loadStatus); + + hasAnySegmentBeenLoaded = hasAnySegmentBeenLoaded || loadStatus.getUsedSegments() > 0; + + // If loading is done for this stage, remove it from future loops. + if (hasAnySegmentBeenLoaded && loadStatus.isLoadingComplete()) { + iterator.remove(); + } + } + + if (!versionsToAwait.isEmpty()) { + // Update the status. + updateStatus(State.WAITING, startTime); + + // Sleep for a while before retrying. + waitIfNeeded(SLEEP_DURATION_MILLIS); + } + } + } + catch (Exception e) { + log.warn(e, "Exception occurred while waiting for segments to load. Exiting."); + + // Update the status and return. + updateStatus(State.FAILED, startTime); + return; + } + // Update the status. + updateStatus(State.SUCCESS, startTime); + } + + private void waitIfNeeded(long waitTimeMillis) throws Exception + { + if (doWait) { + Thread.sleep(waitTimeMillis); + } + } + + /** + * Updates the {@link #status} with the latest details based on {@link #versionToLoadStatusMap} + */ + private void updateStatus(State state, DateTime startTime) + { + int pendingSegmentCount = 0, usedSegmentsCount = 0, precachedSegmentCount = 0, onDemandSegmentCount = 0, unknownSegmentCount = 0; + for (Map.Entry entry : versionToLoadStatusMap.entrySet()) { + usedSegmentsCount += entry.getValue().getUsedSegments(); + precachedSegmentCount += entry.getValue().getPrecachedSegments(); + onDemandSegmentCount += entry.getValue().getOnDemandSegments(); + unknownSegmentCount += entry.getValue().getUnknownSegments(); + pendingSegmentCount += entry.getValue().getPendingSegments(); + } + + long runningMillis = new Interval(startTime, DateTimes.nowUtc()).toDurationMillis(); + status.set( + new SegmentLoadWaiterStatus( + state, + startTime, + runningMillis, + totalSegmentsGenerated, + usedSegmentsCount, + precachedSegmentCount, + onDemandSegmentCount, + pendingSegmentCount, + unknownSegmentCount + ) + ); + } + + /** + * Uses {@link #brokerClient} to fetch latest load status for a given version. Converts the response into a + * {@link VersionLoadStatus} and returns it. + */ + private VersionLoadStatus fetchLoadStatusForVersion(String version) throws Exception + { + Request request = brokerClient.makeRequest(HttpMethod.POST, "/druid/v2/sql/"); + SqlQuery sqlQuery = new SqlQuery(StringUtils.format(LOAD_QUERY, datasource, version), + ResultFormat.OBJECTLINES, + false, false, false, null, null); + request.setContent(MediaType.APPLICATION_JSON, objectMapper.writeValueAsBytes(sqlQuery)); + + String response = brokerClient.sendQuery(request); + + if (response.trim().isEmpty()) { + // If no segments are returned for a version, all segments have been dropped by a drop rule. + return new VersionLoadStatus(0, 0, 0, 0, 0); + } else { + return objectMapper.readValue(response, VersionLoadStatus.class); + } + } + + /** + * Returns the current status of the load. + */ + public SegmentLoadWaiterStatus status() + { + return status.get(); + } + + public static class SegmentLoadWaiterStatus + { + private final State state; + private final DateTime startTime; + private final long duration; + private final int totalSegments; + private final int usedSegments; + private final int precachedSegments; + private final int onDemandSegments; + private final int pendingSegments; + private final int unknownSegments; + + @JsonCreator + public SegmentLoadWaiterStatus( + @JsonProperty("state") SegmentLoadWaiter.State state, + @JsonProperty("startTime") @Nullable DateTime startTime, + @JsonProperty("duration") long duration, + @JsonProperty("totalSegments") int totalSegments, + @JsonProperty("usedSegments") int usedSegments, + @JsonProperty("precachedSegments") int precachedSegments, + @JsonProperty("onDemandSegments") int onDemandSegments, + @JsonProperty("pendingSegments") int pendingSegments, + @JsonProperty("unknownSegments") int unknownSegments + ) + { + this.state = state; + this.startTime = startTime; + this.duration = duration; + this.totalSegments = totalSegments; + this.usedSegments = usedSegments; + this.precachedSegments = precachedSegments; + this.onDemandSegments = onDemandSegments; + this.pendingSegments = pendingSegments; + this.unknownSegments = unknownSegments; + } + + @JsonProperty + public SegmentLoadWaiter.State getState() + { + return state; + } + + @Nullable + @JsonProperty + @JsonInclude(JsonInclude.Include.NON_NULL) + public DateTime getStartTime() + { + return startTime; + } + + @JsonProperty + public long getDuration() + { + return duration; + } + + @JsonProperty + public long getTotalSegments() + { + return totalSegments; + } + + @JsonProperty + public int getUsedSegments() + { + return usedSegments; + } + + @JsonProperty + public int getPrecachedSegments() + { + return precachedSegments; + } + + @JsonProperty + public int getOnDemandSegments() + { + return onDemandSegments; + } + + @JsonProperty + public int getPendingSegments() + { + return pendingSegments; + } + + @JsonProperty + public int getUnknownSegments() + { + return unknownSegments; + } + } + + public enum State + { + /** + * Initial state after being initialised with the segment versions and before #waitForSegmentsToLoad has been called. + */ + INIT, + /** + * All segments that need to be loaded have not yet been loaded. The load status is perodically being queried from + * the broker. + */ + WAITING, + /** + * All segments which need to be loaded have been loaded, and the SegmentLoadWaiter exited successfully. + */ + SUCCESS, + /** + * An exception occurred while checking load status. The SegmentLoadWaiter exited without failing the task. + */ + FAILED, + /** + * The time spent waiting for segments to load exceeded org.apache.druid.msq.exec.SegmentLoadWaiter#TIMEOUT_DURATION_MILLIS. + * The SegmentLoadWaiter exited without failing the task. + */ + TIMED_OUT + } + + public static class VersionLoadStatus + { + private final int usedSegments; + private final int precachedSegments; + private final int onDemandSegments; + private final int pendingSegments; + private final int unknownSegments; + + @JsonCreator + public VersionLoadStatus( + @JsonProperty("usedSegments") int usedSegments, + @JsonProperty("precachedSegments") int precachedSegments, + @JsonProperty("onDemandSegments") int onDemandSegments, + @JsonProperty("pendingSegments") int pendingSegments, + @JsonProperty("unknownSegments") int unknownSegments + ) + { + this.usedSegments = usedSegments; + this.precachedSegments = precachedSegments; + this.onDemandSegments = onDemandSegments; + this.pendingSegments = pendingSegments; + this.unknownSegments = unknownSegments; + } + + @JsonProperty + public int getUsedSegments() + { + return usedSegments; + } + + @JsonProperty + public int getPrecachedSegments() + { + return precachedSegments; + } + + @JsonProperty + public int getOnDemandSegments() + { + return onDemandSegments; + } + + @JsonProperty + public int getPendingSegments() + { + return pendingSegments; + } + + @JsonProperty + public int getUnknownSegments() + { + return unknownSegments; + } + + @JsonIgnore + public boolean isLoadingComplete() + { + return pendingSegments == 0 && (usedSegments == precachedSegments + onDemandSegments); + } + } +} diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQWorkerTaskLauncher.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQWorkerTaskLauncher.java index f391b08b671f..dcc81d868644 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQWorkerTaskLauncher.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQWorkerTaskLauncher.java @@ -182,7 +182,19 @@ public ListenableFuture start() */ public void stop(final boolean interrupt) { - if (state.compareAndSet(State.STARTED, State.STOPPED)) { + if (state.compareAndSet(State.NEW, State.STOPPED)) { + state.set(State.STOPPED); + if (interrupt) { + cancelTasksOnStop.set(true); + } + + synchronized (taskIds) { + // Wake up sleeping mainLoop. + taskIds.notifyAll(); + } + exec.shutdown(); + stopFuture.set(null); + } else if (state.compareAndSet(State.STARTED, State.STOPPED)) { if (interrupt) { cancelTasksOnStop.set(true); } @@ -466,9 +478,13 @@ private void runNewTasks() public WorkerCount getWorkerTaskCount() { synchronized (taskIds) { - int runningTasks = fullyStartedTasks.size(); - int pendingTasks = desiredTaskCount - runningTasks; - return new WorkerCount(runningTasks, pendingTasks); + if (stopFuture.isDone()) { + return new WorkerCount(0, 0); + } else { + int runningTasks = fullyStartedTasks.size(); + int pendingTasks = desiredTaskCount - runningTasks; + return new WorkerCount(runningTasks, pendingTasks); + } } } diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQStatusReport.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQStatusReport.java index 3791bc82e16a..ea721d84f474 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQStatusReport.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQStatusReport.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import org.apache.druid.indexer.TaskState; +import org.apache.druid.msq.exec.SegmentLoadWaiter; import org.apache.druid.msq.indexing.error.MSQErrorReport; import org.joda.time.DateTime; @@ -50,6 +51,9 @@ public class MSQStatusReport private final int runningTasks; + @Nullable + private final SegmentLoadWaiter.SegmentLoadWaiterStatus segmentLoadWaiterStatus; + @JsonCreator public MSQStatusReport( @JsonProperty("status") TaskState status, @@ -58,7 +62,8 @@ public MSQStatusReport( @JsonProperty("startTime") @Nullable DateTime startTime, @JsonProperty("durationMs") long durationMs, @JsonProperty("pendingTasks") int pendingTasks, - @JsonProperty("runningTasks") int runningTasks + @JsonProperty("runningTasks") int runningTasks, + @JsonProperty("segmentLoadWaiterStatus") @Nullable SegmentLoadWaiter.SegmentLoadWaiterStatus segmentLoadWaiterStatus ) { this.status = Preconditions.checkNotNull(status, "status"); @@ -68,6 +73,7 @@ public MSQStatusReport( this.durationMs = durationMs; this.pendingTasks = pendingTasks; this.runningTasks = runningTasks; + this.segmentLoadWaiterStatus = segmentLoadWaiterStatus; } @JsonProperty @@ -117,6 +123,14 @@ public long getDurationMs() return durationMs; } + @Nullable + @JsonProperty + @JsonInclude(JsonInclude.Include.NON_NULL) + public SegmentLoadWaiter.SegmentLoadWaiterStatus getSegmentLoadWaiterStatus() + { + return segmentLoadWaiterStatus; + } + @Override public boolean equals(Object o) { diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/SegmentLoadWaiterTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/SegmentLoadWaiterTest.java new file mode 100644 index 000000000000..e14fa5faec26 --- /dev/null +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/SegmentLoadWaiterTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.msq.exec; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableSet; +import org.apache.druid.discovery.BrokerClient; +import org.apache.druid.java.util.http.client.Request; +import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class SegmentLoadWaiterTest +{ + private static final String TEST_DATASOURCE = "testDatasource"; + + private SegmentLoadWaiter segmentLoadWaiter; + + private BrokerClient brokerClient; + + /** + * Single version created, loaded after 3 attempts + */ + @Test + public void testSingleVersionWaitsForLoadCorrectly() throws Exception + { + brokerClient = mock(BrokerClient.class); + + doReturn(mock(Request.class)).when(brokerClient).makeRequest(any(), anyString()); + doAnswer(new Answer() + { + int timesInvoked = 0; + @Override + public String answer(InvocationOnMock invocation) throws Throwable + { + timesInvoked += 1; + SegmentLoadWaiter.VersionLoadStatus loadStatus = new SegmentLoadWaiter.VersionLoadStatus(5, timesInvoked, 0, 5 - timesInvoked, 0); + return new ObjectMapper().writeValueAsString(loadStatus); + } + }).when(brokerClient).sendQuery(any()); + segmentLoadWaiter = new SegmentLoadWaiter(brokerClient, new ObjectMapper(), TEST_DATASOURCE, ImmutableSet.of("version1"), 5, false); + segmentLoadWaiter.waitForSegmentsToLoad(); + + verify(brokerClient, times(5)).sendQuery(any()); + } + + @Test + public void testMultipleVersionWaitsForLoadCorrectly() throws Exception + { + brokerClient = mock(BrokerClient.class); + + doReturn(mock(Request.class)).when(brokerClient).makeRequest(any(), anyString()); + doAnswer(new Answer() + { + int timesInvoked = 0; + @Override + public String answer(InvocationOnMock invocation) throws Throwable + { + timesInvoked += 1; + SegmentLoadWaiter.VersionLoadStatus loadStatus = new SegmentLoadWaiter.VersionLoadStatus(5, timesInvoked, 0, 5 - timesInvoked, 0); + return new ObjectMapper().writeValueAsString(loadStatus); + } + }).when(brokerClient).sendQuery(any()); + segmentLoadWaiter = new SegmentLoadWaiter(brokerClient, new ObjectMapper(), TEST_DATASOURCE, ImmutableSet.of("version1"), 5, false); + segmentLoadWaiter.waitForSegmentsToLoad(); + + verify(brokerClient, times(5)).sendQuery(any()); + } + +} diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/report/MSQTaskReportTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/report/MSQTaskReportTest.java index eeac9486dc9f..ef50008d48e2 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/report/MSQTaskReportTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/report/MSQTaskReportTest.java @@ -30,10 +30,12 @@ import org.apache.druid.indexer.TaskState; import org.apache.druid.indexing.common.SingleFileTaskReportFileWriter; import org.apache.druid.indexing.common.TaskReport; +import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.common.guava.Yielder; import org.apache.druid.java.util.common.guava.Yielders; import org.apache.druid.msq.counters.CounterSnapshotsTree; +import org.apache.druid.msq.exec.SegmentLoadWaiter; import org.apache.druid.msq.guice.MSQIndexingModule; import org.apache.druid.msq.indexing.error.MSQErrorReport; import org.apache.druid.msq.indexing.error.TooManyColumnsFault; @@ -90,10 +92,22 @@ public void testSerdeResultsReport() throws Exception new Object[]{"bar"} ); + SegmentLoadWaiter.SegmentLoadWaiterStatus status = new SegmentLoadWaiter.SegmentLoadWaiterStatus( + SegmentLoadWaiter.State.WAITING, + DateTimes.nowUtc(), + 200L, + 100, + 80, + 30, + 50, + 10, + 0 + ); + final MSQTaskReport report = new MSQTaskReport( TASK_ID, new MSQTaskReportPayload( - new MSQStatusReport(TaskState.SUCCESS, null, new ArrayDeque<>(), null, 0, 1, 2), + new MSQStatusReport(TaskState.SUCCESS, null, new ArrayDeque<>(), null, 0, 1, 2, status), MSQStagesReport.create( QUERY_DEFINITION, ImmutableMap.of(), @@ -142,11 +156,23 @@ public void testSerdeResultsReport() throws Exception @Test public void testSerdeErrorReport() throws Exception { + SegmentLoadWaiter.SegmentLoadWaiterStatus status = new SegmentLoadWaiter.SegmentLoadWaiterStatus( + SegmentLoadWaiter.State.FAILED, + DateTimes.nowUtc(), + 200L, + 100, + 80, + 30, + 50, + 10, + 0 + ); + final MSQErrorReport errorReport = MSQErrorReport.fromFault(TASK_ID, HOST, 0, new TooManyColumnsFault(10, 5)); final MSQTaskReport report = new MSQTaskReport( TASK_ID, new MSQTaskReportPayload( - new MSQStatusReport(TaskState.FAILED, errorReport, new ArrayDeque<>(), null, 0, 1, 2), + new MSQStatusReport(TaskState.FAILED, errorReport, new ArrayDeque<>(), null, 0, 1, 2, status), MSQStagesReport.create( QUERY_DEFINITION, ImmutableMap.of(), @@ -179,10 +205,22 @@ public void testSerdeErrorReport() throws Exception @Test public void testWriteTaskReport() throws Exception { + SegmentLoadWaiter.SegmentLoadWaiterStatus status = new SegmentLoadWaiter.SegmentLoadWaiterStatus( + SegmentLoadWaiter.State.SUCCESS, + DateTimes.nowUtc(), + 200L, + 100, + 80, + 30, + 50, + 10, + 0 + ); + final MSQTaskReport report = new MSQTaskReport( TASK_ID, new MSQTaskReportPayload( - new MSQStatusReport(TaskState.SUCCESS, null, new ArrayDeque<>(), null, 0, 1, 2), + new MSQStatusReport(TaskState.SUCCESS, null, new ArrayDeque<>(), null, 0, 1, 2, status), MSQStagesReport.create( QUERY_DEFINITION, ImmutableMap.of(), diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java index a603dcb91738..ec986a93159e 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java @@ -240,7 +240,8 @@ public class SqlStatementResourceTest extends MSQTestBase null, 0, 1, - 2 + 2, + null ), MSQStagesReport.create( MSQTaskReportTest.QUERY_DEFINITION, @@ -305,7 +306,8 @@ public class SqlStatementResourceTest extends MSQTestBase null, 0, 1, - 2 + 2, + null ), MSQStagesReport.create( MSQTaskReportTest.QUERY_DEFINITION, diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java index 3ce2d18e40de..9ebcb2ec53e3 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java @@ -42,6 +42,7 @@ import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.LongDimensionSchema; import org.apache.druid.data.input.impl.StringDimensionSchema; +import org.apache.druid.discovery.BrokerClient; import org.apache.druid.discovery.NodeRole; import org.apache.druid.frame.channel.FrameChannelSequence; import org.apache.druid.frame.testutil.FrameTestUtil; @@ -73,6 +74,7 @@ import org.apache.druid.java.util.common.guava.Yielder; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.http.client.Request; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.metadata.input.InputSourceModule; import org.apache.druid.msq.counters.CounterNames; @@ -210,6 +212,10 @@ import static org.apache.druid.sql.calcite.util.CalciteTests.DATASOURCE2; import static org.apache.druid.sql.calcite.util.TestDataBuilder.ROWS1; import static org.apache.druid.sql.calcite.util.TestDataBuilder.ROWS2; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; /** * Base test runner for running MSQ unit tests. It sets up multi stage query execution environment @@ -352,7 +358,7 @@ public void tearDown2() // which depends on the object mapper that the injector will provide, once it // is built, but has not yet been build while we build the SQL engine. @Before - public void setUp2() + public void setUp2() throws Exception { groupByBuffers = TestGroupByBuffers.createDefault(); @@ -380,6 +386,7 @@ public void setUp2() segmentManager = new MSQTestSegmentManager(segmentCacheManager, indexIO); + BrokerClient brokerClient = mock(BrokerClient.class); List modules = ImmutableList.of( binder -> { DruidProcessingConfig druidProcessingConfig = new DruidProcessingConfig() @@ -467,7 +474,8 @@ public String getFormatString() ), new MSQExternalDataSourceModule(), new LookylooModule(), - new SegmentWranglerModule() + new SegmentWranglerModule(), + binder -> binder.bind(BrokerClient.class).toInstance(brokerClient) ); // adding node role injection to the modules, since CliPeon would also do that through run method Injector injector = new CoreInjectorBuilder(new StartupInjectorBuilder().build(), ImmutableSet.of(NodeRole.PEON)) @@ -477,6 +485,8 @@ public String getFormatString() objectMapper = setupObjectMapper(injector); objectMapper.registerModules(sqlModule.getJacksonModules()); + doReturn(mock(Request.class)).when(brokerClient).makeRequest(any(), anyString()); + testTaskActionClient = Mockito.spy(new MSQTestTaskActionClient(objectMapper)); indexingServiceClient = new MSQTestOverlordServiceClient( objectMapper, diff --git a/server/src/main/java/org/apache/druid/discovery/BrokerClient.java b/server/src/main/java/org/apache/druid/discovery/BrokerClient.java new file mode 100644 index 000000000000..bc97c2490ef4 --- /dev/null +++ b/server/src/main/java/org/apache/druid/discovery/BrokerClient.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.discovery; + +import com.google.inject.Inject; +import org.apache.druid.error.DruidException; +import org.apache.druid.guice.annotations.EscalatedGlobal; +import org.apache.druid.java.util.common.IOE; +import org.apache.druid.java.util.common.RetryUtils; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.java.util.http.client.Request; +import org.apache.druid.java.util.http.client.response.StringFullResponseHandler; +import org.apache.druid.java.util.http.client.response.StringFullResponseHolder; +import org.jboss.netty.channel.ChannelException; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.ExecutionException; + +/** + * This class facilitates interaction with Broker. + */ +public class BrokerClient +{ + private static final int MAX_RETRIES = 5; + + private final HttpClient brokerHttpClient; + private final DruidNodeDiscovery druidNodeDiscovery; + + @Inject + public BrokerClient( + @EscalatedGlobal HttpClient brokerHttpClient, + DruidNodeDiscoveryProvider druidNodeDiscoveryProvider + ) + { + this.brokerHttpClient = brokerHttpClient; + this.druidNodeDiscovery = druidNodeDiscoveryProvider.getForNodeRole(NodeRole.BROKER); + } + + /** + * Creates and returns a {@link Request} after choosing a broker. + */ + public Request makeRequest(HttpMethod httpMethod, String urlPath) throws IOException + { + String host = ClientUtils.pickOneHost(druidNodeDiscovery); + + if (host == null) { + throw DruidException.forPersona(DruidException.Persona.ADMIN) + .ofCategory(DruidException.Category.NOT_FOUND) + .build("A leader node could not be found for [%s] service. Check the logs to validate that service is healthy.", NodeRole.BROKER); + } + return new Request(httpMethod, new URL(StringUtils.format("%s%s", host, urlPath))); + } + + public String sendQuery(final Request request) throws Exception + { + return RetryUtils.retry( + () -> { + Request newRequestUrl = getNewRequestUrl(request); + final StringFullResponseHolder fullResponseHolder = brokerHttpClient.go(newRequestUrl, new StringFullResponseHandler(StandardCharsets.UTF_8)).get(); + + HttpResponseStatus responseStatus = fullResponseHolder.getResponse().getStatus(); + if (HttpResponseStatus.SERVICE_UNAVAILABLE.equals(responseStatus) + || HttpResponseStatus.GATEWAY_TIMEOUT.equals(responseStatus)) { + throw DruidException.forPersona(DruidException.Persona.OPERATOR) + .ofCategory(DruidException.Category.RUNTIME_FAILURE) + .build("Request to broker failed due to failed response status: [%s]", responseStatus); + } else if (responseStatus.getCode() != HttpServletResponse.SC_OK) { + throw DruidException.forPersona(DruidException.Persona.OPERATOR) + .ofCategory(DruidException.Category.RUNTIME_FAILURE) + .build("Request to broker failed due to failed response code: [%s]", responseStatus.getCode()); + } + return fullResponseHolder.getContent(); + }, + (throwable) -> { + if (throwable instanceof ExecutionException) { + return throwable.getCause() instanceof IOException || throwable.getCause() instanceof ChannelException; + } + return throwable instanceof IOE; + }, + MAX_RETRIES + ); + } + + private Request getNewRequestUrl(Request oldRequest) + { + try { + return ClientUtils.withUrl( + oldRequest, + new URL(StringUtils.format("%s%s", ClientUtils.pickOneHost(druidNodeDiscovery), oldRequest.getUrl().getPath())) + ); + } + catch (MalformedURLException e) { + // Not an IOException; this is our own fault. + throw DruidException.defensive( + "Failed to build url with path[%] and query string [%s].", + oldRequest.getUrl().getPath(), + oldRequest.getUrl().getQuery() + ); + } + } +} diff --git a/server/src/main/java/org/apache/druid/discovery/ClientUtils.java b/server/src/main/java/org/apache/druid/discovery/ClientUtils.java new file mode 100644 index 000000000000..b9c53343c0d7 --- /dev/null +++ b/server/src/main/java/org/apache/druid/discovery/ClientUtils.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.discovery; + +import com.google.common.collect.Lists; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.http.client.Request; + +import javax.annotation.Nullable; +import java.net.URL; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; + +/** + * Utils class for shared client methods + */ +public class ClientUtils +{ + @Nullable + public static String pickOneHost(DruidNodeDiscovery druidNodeDiscovery) + { + Iterator iter = druidNodeDiscovery.getAllNodes().iterator(); + List discoveryDruidNodeList = Lists.newArrayList(iter); + if (!discoveryDruidNodeList.isEmpty()) { + DiscoveryDruidNode node = discoveryDruidNodeList.get(ThreadLocalRandom.current().nextInt(discoveryDruidNodeList.size())); + return StringUtils.format( + "%s://%s", + node.getDruidNode().getServiceScheme(), + node.getDruidNode().getHostAndPortToUse() + ); + } + return null; + } + + public static Request withUrl(Request old, URL url) + { + Request req = new Request(old.getMethod(), url); + req.addHeaderValues(old.getHeaders()); + if (old.hasContent()) { + req.setContent(old.getContent().copy()); + } + return req; + } +} diff --git a/server/src/main/java/org/apache/druid/discovery/DruidLeaderClient.java b/server/src/main/java/org/apache/druid/discovery/DruidLeaderClient.java index d681004326c7..4ca1441f6f23 100644 --- a/server/src/main/java/org/apache/druid/discovery/DruidLeaderClient.java +++ b/server/src/main/java/org/apache/druid/discovery/DruidLeaderClient.java @@ -39,12 +39,10 @@ import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpResponseStatus; -import javax.annotation.Nullable; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.StandardCharsets; -import java.util.Iterator; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -202,7 +200,7 @@ public > H go( redirectUrl.getPort() )); - request = withUrl(request, redirectUrl); + request = ClientUtils.withUrl(request, redirectUrl); } else if (HttpResponseStatus.SERVICE_UNAVAILABLE.equals(responseStatus) || HttpResponseStatus.GATEWAY_TIMEOUT.equals(responseStatus)) { log.warn( @@ -260,7 +258,7 @@ private String getCurrentKnownLeader(final boolean cached) throws IOException { final String leader = currentKnownLeader.accumulateAndGet( null, - (current, given) -> current == null || !cached ? pickOneHost() : current + (current, given) -> current == null || !cached ? ClientUtils.pickOneHost(druidNodeDiscovery) : current ); if (leader == null) { @@ -274,43 +272,17 @@ private String getCurrentKnownLeader(final boolean cached) throws IOException } } - @Nullable - private String pickOneHost() - { - Iterator iter = druidNodeDiscovery.getAllNodes().iterator(); - if (iter.hasNext()) { - DiscoveryDruidNode node = iter.next(); - return StringUtils.format( - "%s://%s", - node.getDruidNode().getServiceScheme(), - node.getDruidNode().getHostAndPortToUse() - ); - } - - return null; - } - - private Request withUrl(Request old, URL url) - { - Request req = new Request(old.getMethod(), url); - req.addHeaderValues(old.getHeaders()); - if (old.hasContent()) { - req.setContent(old.getContent()); - } - return req; - } - private Request getNewRequestUrlInvalidatingCache(Request oldRequest) throws IOException { try { Request newRequest; if (oldRequest.getUrl().getQuery() == null) { - newRequest = withUrl( + newRequest = ClientUtils.withUrl( oldRequest, new URL(StringUtils.format("%s%s", getCurrentKnownLeader(false), oldRequest.getUrl().getPath())) ); } else { - newRequest = withUrl( + newRequest = ClientUtils.withUrl( oldRequest, new URL(StringUtils.format( "%s%s?%s", diff --git a/server/src/test/java/org/apache/druid/discovery/BrokerClientTest.java b/server/src/test/java/org/apache/druid/discovery/BrokerClientTest.java new file mode 100644 index 000000000000..333882a43051 --- /dev/null +++ b/server/src/test/java/org/apache/druid/discovery/BrokerClientTest.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.discovery; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.name.Names; +import org.apache.druid.guice.GuiceInjectors; +import org.apache.druid.guice.Jerseys; +import org.apache.druid.guice.JsonConfigProvider; +import org.apache.druid.guice.LazySingleton; +import org.apache.druid.guice.LifecycleModule; +import org.apache.druid.guice.annotations.Self; +import org.apache.druid.initialization.Initialization; +import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.java.util.http.client.Request; +import org.apache.druid.server.DruidNode; +import org.apache.druid.server.initialization.BaseJettyTest; +import org.apache.druid.server.initialization.jetty.JettyServerInitializer; +import org.easymock.EasyMock; +import org.eclipse.jetty.server.Server; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.junit.Assert; +import org.junit.Test; + +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.nio.charset.StandardCharsets; + +public class BrokerClientTest extends BaseJettyTest +{ + private DiscoveryDruidNode discoveryDruidNode; + private HttpClient httpClient; + + @Override + protected Injector setupInjector() + { + final DruidNode node = new DruidNode("test", "localhost", false, null, null, true, false); + discoveryDruidNode = new DiscoveryDruidNode(node, NodeRole.BROKER, ImmutableMap.of()); + + Injector injector = Initialization.makeInjectorWithModules( + GuiceInjectors.makeStartupInjector(), ImmutableList.of( + binder -> { + JsonConfigProvider.bindInstance( + binder, + Key.get(DruidNode.class, Self.class), + node + ); + binder.bind(Integer.class).annotatedWith(Names.named("port")).toInstance(node.getPlaintextPort()); + binder.bind(JettyServerInitializer.class).to(DruidLeaderClientTest.TestJettyServerInitializer.class).in(LazySingleton.class); + Jerseys.addResource(binder, SimpleResource.class); + LifecycleModule.register(binder, Server.class); + } + ) + ); + httpClient = injector.getInstance(BaseJettyTest.ClientHolder.class).getClient(); + return injector; + } + + @Test + public void testSimple() throws Exception + { + DruidNodeDiscovery druidNodeDiscovery = EasyMock.createMock(DruidNodeDiscovery.class); + EasyMock.expect(druidNodeDiscovery.getAllNodes()).andReturn(ImmutableList.of(discoveryDruidNode)).anyTimes(); + + DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class); + EasyMock.expect(druidNodeDiscoveryProvider.getForNodeRole(NodeRole.BROKER)).andReturn(druidNodeDiscovery); + + EasyMock.replay(druidNodeDiscovery, druidNodeDiscoveryProvider); + + BrokerClient brokerClient = new BrokerClient( + httpClient, + druidNodeDiscoveryProvider + ); + + Request request = brokerClient.makeRequest(HttpMethod.POST, "/simple/direct"); + request.setContent("hello".getBytes(StandardCharsets.UTF_8)); + Assert.assertEquals("hello", brokerClient.sendQuery(request)); + } + + @Test + public void testError() throws Exception + { + DruidNodeDiscovery druidNodeDiscovery = EasyMock.createMock(DruidNodeDiscovery.class); + EasyMock.expect(druidNodeDiscovery.getAllNodes()).andReturn(ImmutableList.of(discoveryDruidNode)).anyTimes(); + + DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class); + EasyMock.expect(druidNodeDiscoveryProvider.getForNodeRole(NodeRole.BROKER)).andReturn(druidNodeDiscovery); + + EasyMock.replay(druidNodeDiscovery, druidNodeDiscoveryProvider); + + BrokerClient brokerClient = new BrokerClient( + httpClient, + druidNodeDiscoveryProvider + ); + + Request request = brokerClient.makeRequest(HttpMethod.POST, "/simple/flakey"); + request.setContent("hello".getBytes(StandardCharsets.UTF_8)); + Assert.assertEquals("hello", brokerClient.sendQuery(request)); + } + + @Path("/simple") + public static class SimpleResource + { + private static int attempt = 0; + + @POST + @Path("/direct") + @Produces(MediaType.APPLICATION_JSON) + public Response direct(String input) + { + if ("hello".equals(input)) { + return Response.ok("hello").build(); + } else { + return Response.serverError().build(); + } + } + + @POST + @Path("/flakey") + @Produces(MediaType.APPLICATION_JSON) + public Response redirecting() + { + if (attempt > 2) { + return Response.ok("hello").build(); + } else { + attempt += 1; + return Response.status(504).build(); + } + } + } +} diff --git a/server/src/test/java/org/apache/druid/discovery/DruidLeaderClientTest.java b/server/src/test/java/org/apache/druid/discovery/DruidLeaderClientTest.java index f0f91469dc86..4d9870c6e6e1 100644 --- a/server/src/test/java/org/apache/druid/discovery/DruidLeaderClientTest.java +++ b/server/src/test/java/org/apache/druid/discovery/DruidLeaderClientTest.java @@ -281,7 +281,7 @@ public void testFindCurrentLeader() Assert.assertEquals("http://localhost:1234/", druidLeaderClient.findCurrentLeader()); } - private static class TestJettyServerInitializer implements JettyServerInitializer + static class TestJettyServerInitializer implements JettyServerInitializer { @Override public void initialize(Server server, Injector injector) diff --git a/website/.spelling b/website/.spelling index cc4e02fcf293..d08c87c12bde 100644 --- a/website/.spelling +++ b/website/.spelling @@ -440,6 +440,7 @@ preemptible prefetch prefetched prefetching +precached prepend prepended prepending @@ -747,6 +748,7 @@ TooManyWorkers NotEnoughMemory WorkerFailed WorkerRpcFailed +TIMED_OUT # MSQ context parameters maxNumTasks taskAssignment From 6ee0b06e38d9512c9240f55344ed185d2f62a3a6 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Wed, 6 Sep 2023 05:47:19 +0000 Subject: [PATCH 024/258] Auto configuration for maxSubqueryBytes (#14808) A new monitor SubqueryCountStatsMonitor which emits the metrics corresponding to the subqueries and their execution is now introduced. Moreover, the user can now also use the auto mode to automatically set the number of bytes available per query for the inlining of its subquery's results. --- docs/configuration/index.md | 1 + docs/operations/metrics.md | 7 + .../movingaverage/MovingAverageQueryTest.java | 24 ++- .../org/apache/druid/query/QueryContext.java | 9 +- .../apache/druid/query/QueryContextTest.java | 17 ++ .../query/lookup/LookupReferencesManager.java | 4 + .../server/ClientQuerySegmentWalker.java | 79 +++++++-- .../druid/server/SubqueryGuardrailHelper.java | 160 ++++++++++++++++++ .../server/initialization/ServerConfig.java | 10 +- .../metrics/SubqueryCountStatsMonitor.java | 89 ++++++++++ .../metrics/SubqueryCountStatsProvider.java | 133 +++++++++++++++ .../server/ClientQuerySegmentWalkerTest.java | 3 +- .../apache/druid/server/QueryStackTests.java | 17 +- .../server/SubqueryGuardrailHelperTest.java | 160 ++++++++++++++++++ .../java/org/apache/druid/cli/CliBroker.java | 2 + .../SpecificSegmentsQuerySegmentWalker.java | 3 +- 16 files changed, 682 insertions(+), 36 deletions(-) create mode 100644 server/src/main/java/org/apache/druid/server/SubqueryGuardrailHelper.java create mode 100644 server/src/main/java/org/apache/druid/server/metrics/SubqueryCountStatsMonitor.java create mode 100644 server/src/main/java/org/apache/druid/server/metrics/SubqueryCountStatsProvider.java create mode 100644 server/src/test/java/org/apache/druid/server/SubqueryGuardrailHelperTest.java diff --git a/docs/configuration/index.md b/docs/configuration/index.md index b8d4d6a4a502..287f6872d3b7 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -395,6 +395,7 @@ Metric monitoring is an essential part of Druid operations. The following monit |`org.apache.druid.server.metrics.HistoricalMetricsMonitor`|Reports statistics on Historical processes. Available only on Historical processes.| |`org.apache.druid.server.metrics.SegmentStatsMonitor` | **EXPERIMENTAL** Reports statistics about segments on Historical processes. Available only on Historical processes. Not to be used when lazy loading is configured. | |`org.apache.druid.server.metrics.QueryCountStatsMonitor`|Reports how many queries have been successful/failed/interrupted.| +|`org.apache.druid.server.metrics.SubqueryCountStatsMonitor`|Reports how many subqueries have been materialized as rows or bytes and various other statistics related to the subquery execution| |`org.apache.druid.server.emitter.HttpEmittingMonitor`|Reports internal metrics of `http` or `parametrized` emitter (see below). Must not be used with another emitter type. See the description of the metrics here: https://github.com/apache/druid/pull/4973.| |`org.apache.druid.server.metrics.TaskCountStatsMonitor`|Reports how many ingestion tasks are currently running/pending/waiting and also the number of successful/failed tasks per emission period.| |`org.apache.druid.server.metrics.TaskSlotCountStatsMonitor`|Reports metrics about task slot usage per emission period.| diff --git a/docs/operations/metrics.md b/docs/operations/metrics.md index e4a144e4822a..38e68e81c76b 100644 --- a/docs/operations/metrics.md +++ b/docs/operations/metrics.md @@ -72,6 +72,13 @@ Metrics may have additional dimensions beyond those listed above. |`metadatacache/refresh/time`|Time taken to refresh segments in broker segment metadata cache.|`dataSource`| |`serverview/sync/healthy`|Sync status of the Broker with a segment-loading server such as a Historical or Peon. Emitted only when [HTTP-based server view](../configuration/index.md#segment-management) is enabled. This metric can be used in conjunction with `serverview/sync/unstableTime` to debug slow startup of Brokers.|`server`, `tier`|1 for fully synced servers, 0 otherwise| |`serverview/sync/unstableTime`|Time in milliseconds for which the Broker has been failing to sync with a segment-loading server. Emitted only when [HTTP-based server view](../configuration/index.md#segment-management) is enabled.|`server`, `tier`|Not emitted for synced servers.| +|`subquery/rowLimit/count`|Number of subqueries whose results are materialized as rows (Java objects on heap).|This metric is only available if the `SubqueryCountStatsMonitor` module is included.| | +|`subquery/byteLimit/count`|Number of subqueries whose results are materialized as frames (Druid's internal byte representation of rows).|This metric is only available if the `SubqueryCountStatsMonitor` module is included.| | +|`subquery/fallback/count`|Number of subqueries which cannot be materialized as frames|This metric is only available if the `SubqueryCountStatsMonitor` module is included.| | +|`subquery/fallback/insufficientType/count`|Number of subqueries which cannot be materialized as frames due to insufficient type information in the row signature.|This metric is only available if the `SubqueryCountStatsMonitor` module is included.| | +|`subquery/fallback/unknownReason/count`|Number of subqueries which cannot be materialized as frames due other reasons.|This metric is only available if the `SubqueryCountStatsMonitor` module is included.| | +|`query/rowLimit/exceeded/count`|Number of queries whose inlined subquery results exceeded the given row limit|This metric is only available if the `SubqueryCountStatsMonitor` module is included.| | +|`query/byteLimit/exceeded/count`|Number of queries whose inlined subquery results exceeded the given byte limit|This metric is only available if the `SubqueryCountStatsMonitor` module is included.| | ### Historical diff --git a/extensions-contrib/moving-average-query/src/test/java/org/apache/druid/query/movingaverage/MovingAverageQueryTest.java b/extensions-contrib/moving-average-query/src/test/java/org/apache/druid/query/movingaverage/MovingAverageQueryTest.java index 223048200f59..f5fda5f4e483 100644 --- a/extensions-contrib/moving-average-query/src/test/java/org/apache/druid/query/movingaverage/MovingAverageQueryTest.java +++ b/extensions-contrib/moving-average-query/src/test/java/org/apache/druid/query/movingaverage/MovingAverageQueryTest.java @@ -74,6 +74,7 @@ import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.initialization.ServerConfig; import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.server.metrics.SubqueryCountStatsProvider; import org.apache.druid.testing.InitializedNullHandlingTest; import org.apache.druid.timeline.TimelineLookup; import org.hamcrest.core.IsInstanceOf; @@ -310,6 +311,15 @@ public void testQuery() throws IOException Assert.assertNotNull(expectedResults); Assert.assertThat(expectedResults, IsInstanceOf.instanceOf(List.class)); + DruidHttpClientConfig httpClientConfig = new DruidHttpClientConfig() + { + @Override + public long getMaxQueuedBytes() + { + return 0L; + } + }; + CachingClusteredClient baseClient = new CachingClusteredClient( warehouse, new TimelineServerView() @@ -354,14 +364,7 @@ public void registerServerRemovedCallback(Executor exec, ServerRemovedCallback c jsonMapper, new ForegroundCachePopulator(jsonMapper, new CachePopulatorStats(), -1), new CacheConfig(), - new DruidHttpClientConfig() - { - @Override - public long getMaxQueuedBytes() - { - return 0L; - } - }, + httpClientConfig, new BrokerParallelMergeConfig(), ForkJoinPool.commonPool(), QueryStackTests.DEFAULT_NOOP_SCHEDULER, @@ -384,7 +387,10 @@ public void emit(Event event) jsonMapper, serverConfig, null, - new CacheConfig() + new CacheConfig(), + null, + httpClientConfig, + new SubqueryCountStatsProvider() ); defineMocks(); diff --git a/processing/src/main/java/org/apache/druid/query/QueryContext.java b/processing/src/main/java/org/apache/druid/query/QueryContext.java index 403cef1fa4ff..c247ab0e6399 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryContext.java +++ b/processing/src/main/java/org/apache/druid/query/QueryContext.java @@ -347,9 +347,14 @@ public int getMaxSubqueryRows(int defaultSize) return getInt(QueryContexts.MAX_SUBQUERY_ROWS_KEY, defaultSize); } - public long getMaxSubqueryMemoryBytes(long defaultMemoryBytes) + public String getMaxSubqueryMemoryBytes(String defaultMemoryBytes) { - return getLong(QueryContexts.MAX_SUBQUERY_BYTES_KEY, defaultMemoryBytes); + // Generic to allow for both strings and numbers to be passed as values in the query context + Object maxSubqueryBytesObject = get(QueryContexts.MAX_SUBQUERY_BYTES_KEY); + if (maxSubqueryBytesObject == null) { + maxSubqueryBytesObject = defaultMemoryBytes; + } + return String.valueOf(maxSubqueryBytesObject); } public boolean isUseNestedForUnknownTypeInSubquery(boolean defaultUseNestedForUnkownTypeInSubquery) diff --git a/processing/src/test/java/org/apache/druid/query/QueryContextTest.java b/processing/src/test/java/org/apache/druid/query/QueryContextTest.java index ebdbded3a724..54acab0a3f87 100644 --- a/processing/src/test/java/org/apache/druid/query/QueryContextTest.java +++ b/processing/src/test/java/org/apache/druid/query/QueryContextTest.java @@ -325,6 +325,23 @@ public void testGetHumanReadableBytes() assertThrows(BadQueryContextException.class, () -> context.getHumanReadableBytes("m6", HumanReadableBytes.ZERO)); } + @Test + public void testGetMaxSubqueryBytes() + { + final QueryContext context1 = new QueryContext( + ImmutableMap.of(QueryContexts.MAX_SUBQUERY_BYTES_KEY, 500_000_000) + ); + assertEquals("500000000", context1.getMaxSubqueryMemoryBytes(null)); + + final QueryContext context2 = new QueryContext( + ImmutableMap.of(QueryContexts.MAX_SUBQUERY_BYTES_KEY, "auto") + ); + assertEquals("auto", context2.getMaxSubqueryMemoryBytes(null)); + + final QueryContext context3 = new QueryContext(ImmutableMap.of()); + assertEquals("unlimited", context3.getMaxSubqueryMemoryBytes("unlimited")); + } + @Test public void testDefaultEnableQueryDebugging() { diff --git a/server/src/main/java/org/apache/druid/query/lookup/LookupReferencesManager.java b/server/src/main/java/org/apache/druid/query/lookup/LookupReferencesManager.java index 792c52f00320..3cdaec0d4a34 100644 --- a/server/src/main/java/org/apache/druid/query/lookup/LookupReferencesManager.java +++ b/server/src/main/java/org/apache/druid/query/lookup/LookupReferencesManager.java @@ -53,6 +53,7 @@ import java.io.IOException; import java.util.AbstractMap; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -310,6 +311,9 @@ public Optional get(String lookupName) @Override public Set getAllLookupNames() { + if (stateRef.get() == null) { + return Collections.emptySet(); + } return stateRef.get().lookupMap.keySet(); } diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 4054a7e64863..9f21fe8e6874 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -28,8 +28,11 @@ import org.apache.druid.client.DirectDruidClient; import org.apache.druid.client.cache.Cache; import org.apache.druid.client.cache.CacheConfig; +import org.apache.druid.error.DruidException; import org.apache.druid.frame.allocation.ArenaMemoryAllocatorFactory; -import org.apache.druid.java.util.common.IAE; +import org.apache.druid.frame.write.UnsupportedColumnTypeException; +import org.apache.druid.guice.annotations.Client; +import org.apache.druid.guice.http.DruidHttpClientConfig; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.guava.Sequence; @@ -57,10 +60,12 @@ import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.context.ResponseContext; +import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.query.planning.DataSourceAnalysis; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.server.initialization.ServerConfig; +import org.apache.druid.server.metrics.SubqueryCountStatsProvider; import org.joda.time.Interval; import javax.annotation.Nullable; @@ -103,6 +108,8 @@ public class ClientQuerySegmentWalker implements QuerySegmentWalker private final ServerConfig serverConfig; private final Cache cache; private final CacheConfig cacheConfig; + private final SubqueryGuardrailHelper subqueryGuardrailHelper; + private final SubqueryCountStatsProvider subqueryStatsProvider; public ClientQuerySegmentWalker( ServiceEmitter emitter, @@ -114,7 +121,10 @@ public ClientQuerySegmentWalker( ObjectMapper objectMapper, ServerConfig serverConfig, Cache cache, - CacheConfig cacheConfig + CacheConfig cacheConfig, + LookupExtractorFactoryContainerProvider lookupManager, + DruidHttpClientConfig httpClientConfig, + SubqueryCountStatsProvider subqueryStatsProvider ) { this.emitter = emitter; @@ -127,6 +137,12 @@ public ClientQuerySegmentWalker( this.serverConfig = serverConfig; this.cache = cache; this.cacheConfig = cacheConfig; + this.subqueryGuardrailHelper = new SubqueryGuardrailHelper( + lookupManager, + Runtime.getRuntime().maxMemory(), + httpClientConfig.getNumConnections() + ); + this.subqueryStatsProvider = subqueryStatsProvider; } @Inject @@ -140,7 +156,10 @@ public ClientQuerySegmentWalker( ObjectMapper objectMapper, ServerConfig serverConfig, Cache cache, - CacheConfig cacheConfig + CacheConfig cacheConfig, + LookupExtractorFactoryContainerProvider lookupManager, + @Client DruidHttpClientConfig httpClientConfig, + SubqueryCountStatsProvider subqueryStatsProvider ) { this( @@ -153,7 +172,10 @@ public ClientQuerySegmentWalker( objectMapper, serverConfig, cache, - cacheConfig + cacheConfig, + lookupManager, + httpClientConfig, + subqueryStatsProvider ); } @@ -175,10 +197,13 @@ public QueryRunner getQueryRunnerForIntervals(Query query, Iterable> DataSource toInlineDataSource( final AtomicBoolean cannotMaterializeToFrames, final int limit, long memoryLimit, - boolean useNestedForUnknownTypeInSubquery + boolean useNestedForUnknownTypeInSubquery, + SubqueryCountStatsProvider subqueryStatsProvider ) { final int rowLimitToUse = limit < 0 ? Integer.MAX_VALUE : limit; @@ -651,21 +678,25 @@ private static > DataSource toInlineDataSource( switch (ClientQuerySegmentWalkerUtils.getLimitType(memoryLimit, cannotMaterializeToFrames.get())) { case ROW_LIMIT: if (limitAccumulator.get() >= rowLimitToUse) { + subqueryStatsProvider.incrementQueriesExceedingRowLimit(); throw ResourceLimitExceededException.withMessage( "Cannot issue the query, subqueries generated results beyond maximum[%d] rows", rowLimitToUse ); } + subqueryStatsProvider.incrementSubqueriesWithRowLimit(); dataSource = materializeResultsAsArray( query, results, toolChest, limitAccumulator, - limit + limit, + subqueryStatsProvider ); break; case MEMORY_LIMIT: if (memoryLimitAccumulator.get() >= memoryLimit) { + subqueryStatsProvider.incrementQueriesExceedingByteLimit(); throw ResourceLimitExceededException.withMessage( "Cannot issue the query, subqueries generated results beyond maximum[%d] bytes", memoryLimit @@ -678,30 +709,36 @@ private static > DataSource toInlineDataSource( limitAccumulator, memoryLimitAccumulator, memoryLimit, - useNestedForUnknownTypeInSubquery + useNestedForUnknownTypeInSubquery, + subqueryStatsProvider ); if (!maybeDataSource.isPresent()) { cannotMaterializeToFrames.set(true); // Check if the previous row limit accumulator has exceeded the memory results - if (memoryLimitAccumulator.get() >= memoryLimit) { + if (limitAccumulator.get() >= rowLimitToUse) { + subqueryStatsProvider.incrementQueriesExceedingRowLimit(); throw ResourceLimitExceededException.withMessage( - "Cannot issue the query, subqueries generated results beyond maximum[%d] bytes", - memoryLimit + "Cannot issue the query, subqueries generated results beyond maximum[%d] rows", + rowLimitToUse ); } + subqueryStatsProvider.incrementSubqueriesWithRowLimit(); + subqueryStatsProvider.incrementSubqueriesFallingBackToRowLimit(); dataSource = materializeResultsAsArray( query, results, toolChest, limitAccumulator, - limit + limit, + subqueryStatsProvider ); } else { + subqueryStatsProvider.incrementSubqueriesWithByteLimit(); dataSource = maybeDataSource.get(); } break; default: - throw new IAE("Only row based and memory based limiting is supported"); + throw DruidException.defensive("Only row based and memory based limiting is supported"); } return dataSource; } @@ -717,7 +754,8 @@ private static > Optional materializeR final AtomicInteger limitAccumulator, final AtomicLong memoryLimitAccumulator, long memoryLimit, - boolean useNestedForUnknownTypeInSubquery + boolean useNestedForUnknownTypeInSubquery, + final SubqueryCountStatsProvider subqueryStatsProvider ) { Optional> framesOptional; @@ -730,7 +768,13 @@ private static > Optional materializeR useNestedForUnknownTypeInSubquery ); } + catch (UnsupportedColumnTypeException e) { + subqueryStatsProvider.incrementSubqueriesFallingBackDueToUnsufficientTypeInfo(); + log.debug(e, "Type info in signature insufficient to materialize rows as frames."); + return Optional.empty(); + } catch (Exception e) { + subqueryStatsProvider.incrementSubqueriesFallingBackDueToUnknownReason(); log.debug(e, "Unable to materialize the results as frames due to an unhandleable exception " + "while conversion. Defaulting to materializing the results as rows"); return Optional.empty(); @@ -747,6 +791,7 @@ private static > Optional materializeR frame -> { limitAccumulator.addAndGet(frame.getFrame().numRows()); if (memoryLimitAccumulator.addAndGet(frame.getFrame().numBytes()) >= memoryLimit) { + subqueryStatsProvider.incrementQueriesExceedingByteLimit(); throw ResourceLimitExceededException.withMessage( "Subquery generated results beyond maximum[%d] bytes", memoryLimit @@ -767,7 +812,8 @@ private static > DataSource materializeResultsAsAr final Sequence results, final QueryToolChest toolChest, final AtomicInteger limitAccumulator, - final int limit + final int limit, + final SubqueryCountStatsProvider subqueryStatsProvider ) { final int rowLimitToUse = limit < 0 ? Integer.MAX_VALUE : limit; @@ -779,6 +825,7 @@ private static > DataSource materializeResultsAsAr resultList, (acc, in) -> { if (limitAccumulator.getAndIncrement() >= rowLimitToUse) { + subqueryStatsProvider.incrementQueriesExceedingRowLimit(); throw ResourceLimitExceededException.withMessage( "Subquery generated results beyond maximum[%d] rows", rowLimitToUse diff --git a/server/src/main/java/org/apache/druid/server/SubqueryGuardrailHelper.java b/server/src/main/java/org/apache/druid/server/SubqueryGuardrailHelper.java new file mode 100644 index 000000000000..541f44744810 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/SubqueryGuardrailHelper.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server; + +import org.apache.druid.error.InvalidInput; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.query.lookup.LookupExtractor; +import org.apache.druid.query.lookup.LookupExtractorFactoryContainer; +import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; +import org.joda.time.DateTime; +import org.joda.time.Interval; + +/** + * Aids the {@link ClientQuerySegmentWalker} compute the available heap size per query for materializing the inline + * results from the subqueries + */ +public class SubqueryGuardrailHelper +{ + private static final double SUBQUERY_MEMORY_BYTES_FRACTION = 0.5; + private static final Logger log = new Logger(SubqueryGuardrailHelper.class); + + + public static final String UNLIMITED_LIMIT_VALUE = "unlimited"; + public static final String AUTO_LIMIT_VALUE = "auto"; + + public static final Long UNLIMITED_LIMIT_REPRESENTATION = -1L; + + private final long autoLimitBytes; + + public SubqueryGuardrailHelper( + final LookupExtractorFactoryContainerProvider lookupManager, + final long maxMemoryInJvm, + final int brokerNumHttpConnections + ) + { + final DateTime start = DateTimes.nowUtc(); + autoLimitBytes = computeLimitBytesForAuto(lookupManager, maxMemoryInJvm, brokerNumHttpConnections); + final long startupTimeMs = new Interval(start, DateTimes.nowUtc()).toDurationMillis(); + + log.info("Took [%d] ms to initialize the SubqueryGuardrailHelper.", startupTimeMs); + + if (startupTimeMs >= 10_000) { + log.warn("Took more than 10 seconds to initialize the SubqueryGuardrailHelper. " + + "This happens when the lookup sizes are very large. " + + "Consider lowering the size of the lookups to reduce the initialization time." + ); + } + + log.info("Each query has a memory limit of [%d] bytes to materialize its subqueries' results if auto " + + "limit is used", autoLimitBytes); + } + + public long convertSubqueryLimitStringToLong(final String maxSubqueryLimit) + { + if (UNLIMITED_LIMIT_VALUE.equalsIgnoreCase(maxSubqueryLimit)) { + return UNLIMITED_LIMIT_REPRESENTATION; + } + if (AUTO_LIMIT_VALUE.equalsIgnoreCase(maxSubqueryLimit)) { + return autoLimitBytes; + } + + long retVal; + try { + retVal = Long.parseLong(maxSubqueryLimit); + } + catch (NumberFormatException e) { + throw InvalidInput.exception( + e, + "Unable to parse the provided maxSubqueryLimit [%s] to a valid number. Valid values for the " + + "maxSubqueryLimits can be 'auto', 'unlimited' or a positive number representing bytes to reserve.", + maxSubqueryLimit + ); + } + + // This can happen if the provided number is greater than Longs.MAX_VALUE + if (retVal < 0) { + throw InvalidInput.exception("Limit too large"); + } + + return retVal; + } + + /** + * Computes the byte limit when 'auto' is passed as a parameter. This computes the total heap space available + * for the subquery inlining by getting a fraction of the total heap space in JVM, removing the size of the lookups, + * and dividing it by the maximum concurrent queries that can run. Maximum concurrent queries that Druid can + * run is usually limited by its broker's http threadpool size + * + * Consider a JVM running locally with 4 GB heap size, 20 Broker threads and 100 MB space required by the lookups. + * Each query under 'auto' would then get 97.5 MB to materialize the results. Considering the default of 100,000 rows + * and each row consuming 300 - 700 bytes of heap space, the subqueries would approximately consume between 30 MB to + * 70 MB of data, which looks approximately equivalent to what we reserved with auto. This would wildly vary as we have + * larger rows, but that's where the "auto" factor of subquery bytes come into play, where we would estimate by size + * the number of rows that can be materialized based on the memory they consume. + */ + private static long computeLimitBytesForAuto( + final LookupExtractorFactoryContainerProvider lookupManager, + final long maxMemoryInJvm, + final int brokerNumHttpConnections + ) + { + long memoryInJvmWithoutLookups = maxMemoryInJvm - computeLookupFootprint(lookupManager); + long memoryInJvmForSubqueryResultsInlining = (long) (memoryInJvmWithoutLookups * SUBQUERY_MEMORY_BYTES_FRACTION); + long memoryInJvmForSubqueryResultsInliningPerQuery = memoryInJvmForSubqueryResultsInlining + / brokerNumHttpConnections; + return Math.max(memoryInJvmForSubqueryResultsInliningPerQuery, 1L); + } + + /** + * Computes the size occupied by the lookups. If the size of the lookup cannot be computed, it skips over the lookup + */ + private static long computeLookupFootprint(final LookupExtractorFactoryContainerProvider lookupManager) + { + + if (lookupManager == null || lookupManager.getAllLookupNames() == null) { + log.warn("Failed to get the lookupManager for estimating lookup size. Skipping."); + return 0; + } + + int lookupCount = 0; + long lookupFootprint = 0; + + for (final String lookupName : lookupManager.getAllLookupNames()) { + final LookupExtractorFactoryContainer container = lookupManager.get(lookupName).orElse(null); + + if (container != null) { + try { + final LookupExtractor extractor = container.getLookupExtractorFactory().get(); + lookupFootprint += extractor.estimateHeapFootprint(); + lookupCount++; + } + catch (Exception e) { + log.noStackTrace().warn(e, "Failed to load lookup [%s] for size estimation. Skipping.", lookupName); + } + } + } + + log.debug("Lookup footprint: [%d] lookups with [%,d] total bytes.", lookupCount, lookupFootprint); + + return lookupFootprint; + } +} diff --git a/server/src/main/java/org/apache/druid/server/initialization/ServerConfig.java b/server/src/main/java/org/apache/druid/server/initialization/ServerConfig.java index 276a0030af48..04366d30dcdb 100644 --- a/server/src/main/java/org/apache/druid/server/initialization/ServerConfig.java +++ b/server/src/main/java/org/apache/druid/server/initialization/ServerConfig.java @@ -44,7 +44,7 @@ public class ServerConfig { public static final int DEFAULT_GZIP_INFLATE_BUFFER_SIZE = 4096; - public static final long DEFAULT_MAX_SUBQUERY_BYTES = -1L; + public static final String DEFAULT_MAX_SUBQUERY_BYTES = "unlimited"; private static final boolean DEFAULT_USE_NESTED_FOR_UNKNOWN_TYPE_IN_SUBQUERY = false; @@ -61,7 +61,7 @@ public ServerConfig( long defaultQueryTimeout, long maxScatterGatherBytes, int maxSubqueryRows, - long maxSubqueryBytes, + String maxSubqueryBytes, boolean useNestedForUnknownTypeInSubquery, long maxQueryTimeout, int maxRequestHeaderSize, @@ -140,7 +140,7 @@ public ServerConfig(boolean enableQueryRequestsQueuing) private int maxSubqueryRows = 100000; @JsonProperty - private long maxSubqueryBytes = DEFAULT_MAX_SUBQUERY_BYTES; + private String maxSubqueryBytes = DEFAULT_MAX_SUBQUERY_BYTES; @JsonProperty private boolean useNestedForUnknownTypeInSubquery = DEFAULT_USE_NESTED_FOR_UNKNOWN_TYPE_IN_SUBQUERY; @@ -231,7 +231,7 @@ public int getMaxSubqueryRows() return maxSubqueryRows; } - public long getMaxSubqueryBytes() + public String getMaxSubqueryBytes() { return maxSubqueryBytes; } @@ -322,7 +322,7 @@ public boolean equals(Object o) enableRequestLimit == that.enableRequestLimit && defaultQueryTimeout == that.defaultQueryTimeout && maxSubqueryRows == that.maxSubqueryRows && - maxSubqueryBytes == that.maxSubqueryBytes && + Objects.equals(maxSubqueryBytes, that.maxSubqueryBytes) && useNestedForUnknownTypeInSubquery == that.useNestedForUnknownTypeInSubquery && maxQueryTimeout == that.maxQueryTimeout && maxRequestHeaderSize == that.maxRequestHeaderSize && diff --git a/server/src/main/java/org/apache/druid/server/metrics/SubqueryCountStatsMonitor.java b/server/src/main/java/org/apache/druid/server/metrics/SubqueryCountStatsMonitor.java new file mode 100644 index 000000000000..57da36ac1546 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/metrics/SubqueryCountStatsMonitor.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.metrics; + +import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; +import org.apache.druid.java.util.metrics.AbstractMonitor; +import org.apache.druid.java.util.metrics.KeyedDiff; + +import java.util.Map; + +/** + * Monitors and emits the metrics corresponding to the subqueries and their materialization. + */ +public class SubqueryCountStatsMonitor extends AbstractMonitor +{ + + private static final String KEY = "subqueryCountStats"; + + private static final String ROW_LIMIT_COUNT = "subquery/rowLimit/count"; + private static final String BYTE_LIMIT_COUNT = "subquery/byteLimit/count"; + private static final String FALLBACK_COUNT = "subquery/fallback/count"; + private static final String INSUFFICIENT_TYPE_COUNT = "subquery/fallback/insufficientType/count"; + private static final String UNKNOWN_REASON_COUNT = "subquery/fallback/unknownReason/count"; + private static final String ROW_LIMIT_EXCEEDED_COUNT = "query/rowLimit/exceeded/count"; + private static final String BYTE_LIMIT_EXCEEDED_COUNT = "query/byteLimit/exceeded/count"; + + private final KeyedDiff keyedDiff = new KeyedDiff(); + private final SubqueryCountStatsProvider statsProvider; + + @Inject + public SubqueryCountStatsMonitor(SubqueryCountStatsProvider statsProvider) + { + this.statsProvider = statsProvider; + } + + @Override + public boolean doMonitor(ServiceEmitter emitter) + { + final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder(); + final long subqueriesWithRowBasedLimit = statsProvider.subqueriesWithRowLimit(); + final long subqueriesWithByteBasedLimit = statsProvider.subqueriesWithByteLimit(); + final long subqueriesFallingBackToRowBasedLimit = statsProvider.subqueriesFallingBackToRowLimit(); + final long subqueriesFallingBackDueToUnsufficientTypeInfo = statsProvider.subqueriesFallingBackDueToUnsufficientTypeInfo(); + final long subqueriesFallingBackDueToUnknownReason = statsProvider.subqueriesFallingBackDueUnknownReason(); + final long queriesExceedingRowLimit = statsProvider.queriesExceedingRowLimit(); + final long queriesExceedingByteLimit = statsProvider.queriesExceedingByteLimit(); + + Map diff = keyedDiff.to( + KEY, + ImmutableMap.of( + ROW_LIMIT_COUNT, subqueriesWithRowBasedLimit, + BYTE_LIMIT_COUNT, subqueriesWithByteBasedLimit, + FALLBACK_COUNT, subqueriesFallingBackToRowBasedLimit, + INSUFFICIENT_TYPE_COUNT, subqueriesFallingBackDueToUnsufficientTypeInfo, + UNKNOWN_REASON_COUNT, subqueriesFallingBackDueToUnknownReason, + ROW_LIMIT_EXCEEDED_COUNT, queriesExceedingRowLimit, + BYTE_LIMIT_EXCEEDED_COUNT, queriesExceedingByteLimit + ) + ); + + if (diff != null) { + for (Map.Entry diffEntry : diff.entrySet()) { + emitter.emit(builder.setMetric(diffEntry.getKey(), diffEntry.getValue())); + } + } + + return true; + } +} diff --git a/server/src/main/java/org/apache/druid/server/metrics/SubqueryCountStatsProvider.java b/server/src/main/java/org/apache/druid/server/metrics/SubqueryCountStatsProvider.java new file mode 100644 index 000000000000..c7007765496d --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/metrics/SubqueryCountStatsProvider.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.metrics; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * Collects the metrics corresponding to the subqueries and their materialization. + */ +public class SubqueryCountStatsProvider +{ + + private final AtomicLong successfulSubqueriesWithRowLimit = new AtomicLong(); + private final AtomicLong successfulSubqueriesWithByteLimit = new AtomicLong(); + private final AtomicLong subqueriesFallingBackToRowLimit = new AtomicLong(); + private final AtomicLong subqueriesFallingBackDueToUnsufficientTypeInfo = new AtomicLong(); + private final AtomicLong subqueriesFallingBackDueToUnknownReason = new AtomicLong(); + private final AtomicLong queriesExceedingRowLimit = new AtomicLong(); + private final AtomicLong queriesExceedingByteLimit = new AtomicLong(); + + /** + * @return Count of subqueries where the results are materialized as rows i.e. {@code List} + */ + public long subqueriesWithRowLimit() + { + return successfulSubqueriesWithRowLimit.get(); + } + + /** + * @return Count of subqueries where the results are materialized as {@link org.apache.druid.frame.Frame} + */ + public long subqueriesWithByteLimit() + { + return successfulSubqueriesWithByteLimit.get(); + } + + /** + * @return Count of subqueries where the results are + */ + public long subqueriesFallingBackToRowLimit() + { + return subqueriesFallingBackToRowLimit.get(); + } + + /** + * @return Count of the subset of subqueries that are falling back due to insufficient type information in the + * {@link org.apache.druid.segment.column.RowSignature}. This is expected to be the most common and already known + * cause of fallback, therefore this is added as a separate metric + */ + public long subqueriesFallingBackDueToUnsufficientTypeInfo() + { + return subqueriesFallingBackDueToUnsufficientTypeInfo.get(); + } + + /** + * @return Count of the subset of subqueries that are falling back due to insufficient an unknown error. This can be due to a + * few known reasons like columnar frames not supporting the array types right now, or due to unknown errors while + * performing the materialization + */ + public long subqueriesFallingBackDueUnknownReason() + { + return subqueriesFallingBackDueToUnknownReason.get(); + } + + /** + * @return Number of queries that fail due to their subqueries exceeding the prescribed row limit + */ + public long queriesExceedingRowLimit() + { + return queriesExceedingRowLimit.get(); + } + + /** + * @return Number of subqueries that fail due to their subqueries exceeding the prescribed byte limit + */ + public long queriesExceedingByteLimit() + { + return queriesExceedingByteLimit.get(); + } + + + public void incrementSubqueriesWithRowLimit() + { + successfulSubqueriesWithRowLimit.incrementAndGet(); + } + + public void incrementSubqueriesWithByteLimit() + { + successfulSubqueriesWithByteLimit.incrementAndGet(); + } + + public void incrementSubqueriesFallingBackToRowLimit() + { + subqueriesFallingBackToRowLimit.incrementAndGet(); + } + + public void incrementSubqueriesFallingBackDueToUnsufficientTypeInfo() + { + subqueriesFallingBackDueToUnsufficientTypeInfo.incrementAndGet(); + } + + public void incrementSubqueriesFallingBackDueToUnknownReason() + { + subqueriesFallingBackDueToUnknownReason.incrementAndGet(); + } + + public void incrementQueriesExceedingRowLimit() + { + queriesExceedingRowLimit.incrementAndGet(); + } + + public void incrementQueriesExceedingByteLimit() + { + queriesExceedingByteLimit.incrementAndGet(); + } +} diff --git a/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java b/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java index 7180b03983fa..fcdc894f61fa 100644 --- a/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java +++ b/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java @@ -1439,7 +1439,8 @@ public QueryRunner getQueryRunnerForSegments(Query query, Iterable getModules() binder.bind(BrokerQueryResource.class).in(LazySingleton.class); Jerseys.addResource(binder, BrokerQueryResource.class); binder.bind(QueryCountStatsProvider.class).to(BrokerQueryResource.class).in(LazySingleton.class); + binder.bind(SubqueryCountStatsProvider.class).toInstance(new SubqueryCountStatsProvider()); Jerseys.addResource(binder, BrokerResource.class); Jerseys.addResource(binder, ClientInfoResource.class); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java index b43a65159567..a47bbcf95779 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java @@ -116,7 +116,8 @@ public SpecificSegmentsQuerySegmentWalker( ), conglomerate, joinableFactoryWrapper.getJoinableFactory(), - new ServerConfig() + new ServerConfig(), + LOOKUP_EXTRACTOR_FACTORY_CONTAINER_PROVIDER ); } From 23308c050da41769dc0e4ac00645a83483160e27 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 6 Sep 2023 15:41:58 +0200 Subject: [PATCH 025/258] Remove DruidAggregateCaseToFilterRule (#14940) The issue due to which the custom rule was added has been fixed as a part of https://issues.apache.org/jira/browse/CALCITE-3763 and accommodated during Calcite upgrade --- .../calcite/planner/CalciteRulesManager.java | 2 - .../DruidAggregateCaseToFilterRule.java | 339 ------------------ 2 files changed, 341 deletions(-) delete mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidAggregateCaseToFilterRule.java diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java index 7fd7cb49b21f..8d2f1103922b 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java @@ -56,7 +56,6 @@ import org.apache.druid.sql.calcite.rule.FilterJoinExcludePushToChildRule; import org.apache.druid.sql.calcite.rule.ProjectAggregatePruneUnusedCallRule; import org.apache.druid.sql.calcite.rule.SortCollapseRule; -import org.apache.druid.sql.calcite.rule.logical.DruidAggregateCaseToFilterRule; import org.apache.druid.sql.calcite.rule.logical.DruidLogicalRules; import org.apache.druid.sql.calcite.run.EngineFeature; @@ -318,7 +317,6 @@ public List logicalConventionRuleSet(final PlannerContext plannerCon final ImmutableList.Builder retVal = ImmutableList .builder() .addAll(baseRuleSet(plannerContext)) - .add(DruidAggregateCaseToFilterRule.INSTANCE) .add(new DruidLogicalRules(plannerContext).rules().toArray(new RelOptRule[0])); return retVal.build(); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidAggregateCaseToFilterRule.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidAggregateCaseToFilterRule.java deleted file mode 100644 index 700f108f2902..000000000000 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidAggregateCaseToFilterRule.java +++ /dev/null @@ -1,339 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.sql.calcite.rule.logical; - -import com.google.common.collect.ImmutableList; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.RelCollations; -import org.apache.calcite.rel.core.Aggregate; -import org.apache.calcite.rel.core.AggregateCall; -import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.RelFactories; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexLiteral; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlPostfixOperator; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.tools.RelBuilderFactory; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * A copy of {@link org.apache.calcite.rel.rules.AggregateCaseToFilterRule} except that it fixes a bug to eliminate - * left-over projects for converted aggregates to filter-aggregates. The elimination of left-over projects is necessary - * with the new planning since it determines the cost of the plan and hence determines which plan is going to get picked - * as the cheapest one. - * This fix will also be contributed upstream to Calcite project, and we can remove this rule once the fix is a part of - * the Calcite version we use. - */ -public class DruidAggregateCaseToFilterRule extends RelOptRule -{ - public static final DruidAggregateCaseToFilterRule INSTANCE = - new DruidAggregateCaseToFilterRule(RelFactories.LOGICAL_BUILDER, null); - - /** - * Creates an AggregateCaseToFilterRule. - */ - protected DruidAggregateCaseToFilterRule( - RelBuilderFactory relBuilderFactory, - String description - ) - { - super(operand(Aggregate.class, operand(Project.class, any())), - relBuilderFactory, description - ); - } - - @Override - public boolean matches(final RelOptRuleCall call) - { - final Aggregate aggregate = call.rel(0); - final Project project = call.rel(1); - - for (AggregateCall aggregateCall : aggregate.getAggCallList()) { - final int singleArg = soleArgument(aggregateCall); - if (singleArg >= 0 - && isThreeArgCase(project.getProjects().get(singleArg))) { - return true; - } - } - - return false; - } - - @Override - public void onMatch(RelOptRuleCall call) - { - final Aggregate aggregate = call.rel(0); - final Project project = call.rel(1); - final List newCalls = - new ArrayList<>(aggregate.getAggCallList().size()); - List newProjects; - - // TODO : fix grouping columns - Set groupUsedFields = new HashSet<>(); - for (int fieldNumber : aggregate.getGroupSet()) { - groupUsedFields.add(fieldNumber); - } - - List updatedProjects = new ArrayList<>(); - for (int i = 0; i < project.getProjects().size(); i++) { - if (groupUsedFields.contains(i)) { - updatedProjects.add(project.getProjects().get(i)); - } - } - newProjects = updatedProjects; - - for (AggregateCall aggregateCall : aggregate.getAggCallList()) { - AggregateCall newCall = - transform(aggregateCall, project, newProjects); - - // Possibly CAST the new aggregator to an appropriate type. - newCalls.add(newCall); - } - final RelBuilder relBuilder = call.builder() - .push(project.getInput()) - .project(newProjects); - - final RelBuilder.GroupKey groupKey = - relBuilder.groupKey( - aggregate.getGroupSet(), - aggregate.getGroupSets() - ); - - relBuilder.aggregate(groupKey, newCalls) - .convert(aggregate.getRowType(), false); - - call.transformTo(relBuilder.build()); - call.getPlanner().prune(aggregate); - } - - private AggregateCall transform(AggregateCall aggregateCall, Project project, List newProjects) - { - final int singleArg = soleArgument(aggregateCall); - if (singleArg < 0) { - Set newFields = new HashSet<>(); - for (int fieldNumber : aggregateCall.getArgList()) { - newProjects.add(project.getProjects().get(fieldNumber)); - newFields.add(newProjects.size() - 1); - } - int newFilterArg = -1; - if (aggregateCall.hasFilter()) { - newProjects.add(project.getProjects().get(aggregateCall.filterArg)); - newFilterArg = newProjects.size() - 1; - } - return AggregateCall.create(aggregateCall.getAggregation(), - aggregateCall.isDistinct(), - aggregateCall.isApproximate(), - aggregateCall.ignoreNulls(), - new ArrayList<>(newFields), - newFilterArg, - aggregateCall.getCollation(), - aggregateCall.getType(), - aggregateCall.getName() - ); - } - - final RexNode rexNode = project.getProjects().get(singleArg); - if (!isThreeArgCase(rexNode)) { - newProjects.add(rexNode); - int callArg = newProjects.size() - 1; - int newFilterArg = -1; - if (aggregateCall.hasFilter()) { - newProjects.add(project.getProjects().get(aggregateCall.filterArg)); - newFilterArg = newProjects.size() - 1; - } - return AggregateCall.create(aggregateCall.getAggregation(), - aggregateCall.isDistinct(), - aggregateCall.isApproximate(), - aggregateCall.ignoreNulls(), - ImmutableList.of(callArg), - newFilterArg, - aggregateCall.getCollation(), - aggregateCall.getType(), - aggregateCall.getName() - ); - } - - final RelOptCluster cluster = project.getCluster(); - final RexBuilder rexBuilder = cluster.getRexBuilder(); - final RexCall caseCall = (RexCall) rexNode; - - // If one arg is null and the other is not, reverse them and set "flip", - // which negates the filter. - final boolean flip = RexLiteral.isNullLiteral(caseCall.operands.get(1)) - && !RexLiteral.isNullLiteral(caseCall.operands.get(2)); - final RexNode arg1 = caseCall.operands.get(flip ? 2 : 1); - final RexNode arg2 = caseCall.operands.get(flip ? 1 : 2); - - // Operand 1: Filter - final SqlPostfixOperator op = - flip ? SqlStdOperatorTable.IS_FALSE : SqlStdOperatorTable.IS_TRUE; - final RexNode filterFromCase = - rexBuilder.makeCall(op, caseCall.operands.get(0)); - - // Combine the CASE filter with an honest-to-goodness SQL FILTER, if the - // latter is present. - final RexNode filter; - if (aggregateCall.filterArg >= 0) { - filter = rexBuilder.makeCall(SqlStdOperatorTable.AND, - project.getProjects().get(aggregateCall.filterArg), filterFromCase - ); - } else { - filter = filterFromCase; - } - - final SqlKind kind = aggregateCall.getAggregation().getKind(); - if (aggregateCall.isDistinct()) { - // Just one style supported: - // COUNT(DISTINCT CASE WHEN x = 'foo' THEN y END) - // => - // COUNT(DISTINCT y) FILTER(WHERE x = 'foo') - - if (kind == SqlKind.COUNT - && RexLiteral.isNullLiteral(arg2)) { - newProjects.add(arg1); - newProjects.add(filter); - return AggregateCall.create(SqlStdOperatorTable.COUNT, true, false, - false, ImmutableList.of(newProjects.size() - 2), - newProjects.size() - 1, RelCollations.EMPTY, - aggregateCall.getType(), aggregateCall.getName() - ); - } - newProjects.add(rexNode); - int callArg = newProjects.size() - 1; - int newFilterArg = -1; - if (aggregateCall.hasFilter()) { - newProjects.add(project.getProjects().get(aggregateCall.filterArg)); - newFilterArg = newProjects.size() - 1; - } - return AggregateCall.create(aggregateCall.getAggregation(), - aggregateCall.isDistinct(), - aggregateCall.isApproximate(), - aggregateCall.ignoreNulls(), - ImmutableList.of(callArg), - newFilterArg, - aggregateCall.getCollation(), - aggregateCall.getType(), - aggregateCall.getName() - ); - } - - // Four styles supported: - // - // A1: AGG(CASE WHEN x = 'foo' THEN cnt END) - // => operands (x = 'foo', cnt, null) - // A2: SUM(CASE WHEN x = 'foo' THEN cnt ELSE 0 END) - // => operands (x = 'foo', cnt, 0); must be SUM - // B: SUM(CASE WHEN x = 'foo' THEN 1 ELSE 0 END) - // => operands (x = 'foo', 1, 0); must be SUM - // C: COUNT(CASE WHEN x = 'foo' THEN 'dummy' END) - // => operands (x = 'foo', 'dummy', null) - - if (kind == SqlKind.COUNT // Case C - && arg1.isA(SqlKind.LITERAL) - && !RexLiteral.isNullLiteral(arg1) - && RexLiteral.isNullLiteral(arg2)) { - newProjects.add(filter); - return AggregateCall.create(SqlStdOperatorTable.COUNT, false, false, - false, ImmutableList.of(), newProjects.size() - 1, - RelCollations.EMPTY, aggregateCall.getType(), - aggregateCall.getName() - ); - } else if (kind == SqlKind.SUM // Case B - && isIntLiteral(arg1) && RexLiteral.intValue(arg1) == 1 - && isIntLiteral(arg2) && RexLiteral.intValue(arg2) == 0) { - - newProjects.add(filter); - final RelDataTypeFactory typeFactory = cluster.getTypeFactory(); - final RelDataType dataType = - typeFactory.createTypeWithNullability( - typeFactory.createSqlType(SqlTypeName.BIGINT), false); - return AggregateCall.create(SqlStdOperatorTable.COUNT, false, false, - false, ImmutableList.of(), newProjects.size() - 1, - RelCollations.EMPTY, dataType, aggregateCall.getName() - ); - } else if ((RexLiteral.isNullLiteral(arg2) // Case A1 - && aggregateCall.getAggregation().allowsFilter()) - || (kind == SqlKind.SUM // Case A2 - && isIntLiteral(arg2) - && RexLiteral.intValue(arg2) == 0)) { - newProjects.add(arg1); - newProjects.add(filter); - return AggregateCall.create(aggregateCall.getAggregation(), false, - false, false, ImmutableList.of(newProjects.size() - 2), - newProjects.size() - 1, RelCollations.EMPTY, - aggregateCall.getType(), aggregateCall.getName() - ); - } else { - newProjects.add(rexNode); - int callArg = newProjects.size() - 1; - int newFilterArg = -1; - if (aggregateCall.hasFilter()) { - newProjects.add(project.getProjects().get(aggregateCall.filterArg)); - newFilterArg = newProjects.size() - 1; - } - return AggregateCall.create(aggregateCall.getAggregation(), - aggregateCall.isDistinct(), - aggregateCall.isApproximate(), - aggregateCall.ignoreNulls(), - ImmutableList.of(callArg), - newFilterArg, - aggregateCall.getCollation(), - aggregateCall.getType(), - aggregateCall.getName() - ); - } - } - - /** - * Returns the argument, if an aggregate call has a single argument, - * otherwise -1. - */ - private static int soleArgument(AggregateCall aggregateCall) - { - return aggregateCall.getArgList().size() == 1 - ? aggregateCall.getArgList().get(0) - : -1; - } - - private static boolean isThreeArgCase(final RexNode rexNode) - { - return rexNode.getKind() == SqlKind.CASE - && ((RexCall) rexNode).operands.size() == 3; - } - - private static boolean isIntLiteral(final RexNode rexNode) - { - return rexNode instanceof RexLiteral - && SqlTypeName.INT_TYPES.contains(rexNode.getType().getSqlTypeName()); - } -} From a8fa979115a404e9cc418bb9e604873921e3c4d1 Mon Sep 17 00:00:00 2001 From: Soumyava <93540295+somu-imply@users.noreply.github.com> Date: Wed, 6 Sep 2023 08:57:03 -0700 Subject: [PATCH 026/258] Unnest dont push down not (#14942) * Not pushing down not filters * New test case * Updating tests * Removing a stale comment --- .../druid/segment/UnnestStorageAdapter.java | 4 +- .../segment/UnnestStorageAdapterTest.java | 46 +++++++++++++++++-- .../sql/calcite/CalciteArraysQueryTest.java | 13 +----- 3 files changed, 47 insertions(+), 16 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java index 71a1809a885c..83694d9618d5 100644 --- a/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java +++ b/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java @@ -495,6 +495,8 @@ private List recursiveRewriteOnUnnestFilters( false ); preFilterList.add(new OrFilter(orChildFilters)); + } else if (filter instanceof NotFilter) { + continue; } else { final Filter newFilter = rewriteFilterOnUnnestColumnIfPossible( filter, @@ -579,7 +581,7 @@ static boolean filterMapsOverMultiValueStrings(final Filter filter) } return true; } else if (filter instanceof NotFilter) { - return filterMapsOverMultiValueStrings(((NotFilter) filter).getBaseFilter()); + return false; } else { return filter instanceof SelectorFilter || filter instanceof InDimFilter diff --git a/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java index 4257ac4c0723..2139335b594a 100644 --- a/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java @@ -59,6 +59,7 @@ import java.util.Arrays; import java.util.List; +import static org.apache.druid.segment.filter.FilterTestUtils.not; import static org.apache.druid.segment.filter.FilterTestUtils.selector; import static org.apache.druid.segment.filter.Filters.and; import static org.apache.druid.segment.filter.Filters.or; @@ -467,15 +468,54 @@ public void test_nested_filters_unnested_and_topLevelAND3filtersInORWithNestedOr public void test_nested_filters_unnested_and_topLevelAND2sdf() { final Filter testQueryFilter = and(ImmutableList.of( - selector(OUTPUT_COLUMN_NAME, "3"), + not(selector(OUTPUT_COLUMN_NAME, "3")), + selector(COLUMNNAME, "2") + )); + testComputeBaseAndPostUnnestFilters( + testQueryFilter, + "multi-string1 = 2", + "(~(unnested-multi-string1 = 3) && multi-string1 = 2)" + ); + } + + @Test + public void test_nested_filters_unnested_and_topLevelOR2sdf() + { + final Filter testQueryFilter = or(ImmutableList.of( + not(selector(OUTPUT_COLUMN_NAME, "3")), selector(COLUMNNAME, "2") )); testComputeBaseAndPostUnnestFilters( testQueryFilter, - "(multi-string1 = 3 && multi-string1 = 2)", - "(unnested-multi-string1 = 3 && multi-string1 = 2)" + "(multi-string1 = 2)", + "(~(unnested-multi-string1 = 3) || multi-string1 = 2)" + ); + } + + @Test + public void test_not_pushdown_not_filter() + { + final Filter testQueryFilter = and(ImmutableList.of( + not(selector(OUTPUT_COLUMN_NAME, "3")), + or(ImmutableList.of( + or(ImmutableList.of( + selector("newcol", "2"), + selector(COLUMNNAME, "2"), + and(ImmutableList.of( + selector("newcol", "3"), + selector(COLUMNNAME, "7") + )) + )), + selector(OUTPUT_COLUMN_NAME, "1") + )) + )); + testComputeBaseAndPostUnnestFilters( + testQueryFilter, + "(newcol = 2 || multi-string1 = 2 || (newcol = 3 && multi-string1 = 7) || multi-string1 = 1)", + "(~(unnested-multi-string1 = 3) && (newcol = 2 || multi-string1 = 2 || (newcol = 3 && multi-string1 = 7) || unnested-multi-string1 = 1))" ); } + @Test public void test_pushdown_filters_unnested_dimension_with_unnest_adapters() { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index 9f25f4cd00f8..4842044892dc 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -3686,19 +3686,8 @@ public void testUnnestWithFiltersInsideAndOutside() .columns(ImmutableList.of("j0.unnest")) .build() ), - // The result as incorrect because of pushing the not filter - // into the base. I would expect this result to be 'a' - // Putting the not on the direct mapped column from unnest - // makes it eatup the values from MVD. - // For example select dim3 from numFoo where dim2='a' AND dim1 <> 'foo' - // Has 2 rows - // ["a","b"] - // empty - // if dim3 <> 'b' is pushed down to base it eats up the first row and the - // result only has empty. - // Future developer should ensure not filters involving direct mapping of unnested - // column should not be pushed to base but should onluy appear in the post filter ImmutableList.of( + new Object[]{"a"}, new Object[]{""} ) ); From 88f3c9baed5f76bdca66c3284b4368894d7d8462 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Thu, 7 Sep 2023 01:14:05 +0530 Subject: [PATCH 027/258] Fix bug in computed value of balancerComputeThreads (#14947) In smartSegmentLoading mode, use computed value of balancerComputeThreads rather than configured value. --- .../coordinator/duty/PrepareBalancerAndLoadQueues.java | 2 +- .../server/coordinator/loading/SegmentLoadingConfig.java | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/PrepareBalancerAndLoadQueues.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/PrepareBalancerAndLoadQueues.java index f066235efc95..6af49e15ee91 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/PrepareBalancerAndLoadQueues.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/PrepareBalancerAndLoadQueues.java @@ -94,7 +94,7 @@ public DruidCoordinatorRuntimeParams run(DruidCoordinatorRuntimeParams params) collectHistoricalStats(cluster, stats); collectUsedSegmentStats(params, stats); - int numBalancerThreads = params.getCoordinatorDynamicConfig().getBalancerComputeThreads(); + final int numBalancerThreads = segmentLoadingConfig.getBalancerComputeThreads(); final BalancerStrategy balancerStrategy = balancerStrategyFactory.createBalancerStrategy(numBalancerThreads); log.info( "Using balancer strategy [%s] with [%d] threads.", diff --git a/server/src/main/java/org/apache/druid/server/coordinator/loading/SegmentLoadingConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/loading/SegmentLoadingConfig.java index 6b15c3c01a84..ca0305eaa099 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/loading/SegmentLoadingConfig.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/loading/SegmentLoadingConfig.java @@ -49,9 +49,11 @@ public static SegmentLoadingConfig create(CoordinatorDynamicConfig dynamicConfig // Compute replicationThrottleLimit with a lower bound of 100 final int throttlePercentage = 5; final int replicationThrottleLimit = Math.max(100, numUsedSegments * throttlePercentage / 100); + final int numBalancerThreads = CoordinatorDynamicConfig.getDefaultBalancerComputeThreads(); log.info( - "Smart segment loading is enabled. Calculated replicationThrottleLimit[%,d] (%d%% of used segments[%,d]).", - replicationThrottleLimit, throttlePercentage, numUsedSegments + "Smart segment loading is enabled. Calculated replicationThrottleLimit[%,d]" + + " (%d%% of used segments[%,d]) and numBalancerThreads[%d].", + replicationThrottleLimit, throttlePercentage, numUsedSegments, numBalancerThreads ); return new SegmentLoadingConfig( @@ -59,7 +61,7 @@ public static SegmentLoadingConfig create(CoordinatorDynamicConfig dynamicConfig replicationThrottleLimit, 60, true, - CoordinatorDynamicConfig.getDefaultBalancerComputeThreads() + numBalancerThreads ); } else { // Use the configured values From e100b18e86fb1cabf43cf21e9904107335850e6e Mon Sep 17 00:00:00 2001 From: Hardik Bajaj <58038410+hardikbajaj@users.noreply.github.com> Date: Thu, 7 Sep 2023 16:54:33 +0530 Subject: [PATCH 028/258] Updated documentation for OshiSysMonitor (#14912) --- dev/druid_intellij_formatting.xml | 4 ++ docs/operations/metrics.md | 66 +++++++++++++++++++++++++++---- website/.spelling | 3 ++ 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/dev/druid_intellij_formatting.xml b/dev/druid_intellij_formatting.xml index 6e0a33c9fd21..7771b2654721 100644 --- a/dev/druid_intellij_formatting.xml +++ b/dev/druid_intellij_formatting.xml @@ -74,6 +74,10 @@