applicationLevelExceptions() {
+ return Stream.of(
+ Arguments.of(new AccessControlException("denied"),
+ "AccessControlException"),
+ Arguments.of(new IllegalArgumentException("bad request"),
+ "IllegalArgumentException"),
+ Arguments.of(new IOException("application error: not leader"),
+ "plain IOException without connection-class cause"),
+ Arguments.of(new RuntimeException("retry, please"),
+ "plain RuntimeException")
+ );
+ }
+
+ @ParameterizedTest(name = "isConnectionFailure rejects {1}")
+ @MethodSource("applicationLevelExceptions")
+ public void testRejectsApplicationLevel(Throwable t, String label) {
+ assertFalse(ConnectionFailureUtils.isConnectionFailure(t),
+ label + " is an application error, refresh must NOT trigger");
+ }
+
+ @Test
+ public void testNullIsNotAConnectionFailure() {
+ assertFalse(ConnectionFailureUtils.isConnectionFailure(null));
+ }
+
+ /**
+ * {@code Throwable.initCause} contractually rejects setting cause to
+ * the throwable itself, but cycles of length 2+ have appeared in
+ * practice (proxy frameworks and faulty initCause callers can
+ * construct them). The walk must terminate within the configured
+ * depth bound rather than looping forever.
+ *
+ * We build the length-2 cycle through {@link Throwable#initCause}
+ * (no reflection) -- the no-arg ctor leaves cause uninitialized
+ * (cause==this sentinel), so a single initCause call on each side
+ * is permitted and lets us close the cycle.
+ */
+ @Test
+ public void testCycleOfLengthTwoTerminates() {
+ Throwable a = new IOException();
+ Throwable b = new IOException();
+ a.initCause(b);
+ b.initCause(a);
+ // Neither a nor b is a connection-class type. The walk must return
+ // false (not loop forever and not throw).
+ assertFalse(ConnectionFailureUtils.isConnectionFailure(a),
+ "length-2 cycle must terminate cleanly");
+ }
+
+ /**
+ * Defense against an unbounded chain of non-connection-class
+ * exceptions: the depth bound must kick in.
+ *
+ * Built using {@link Throwable#initCause} on freshly-constructed
+ * exceptions (no-arg ctor leaves cause uninitialized) so the test
+ * does not depend on JDK-internal reflective access to the
+ * {@code cause} field, which fails on JDK 16+ without
+ * {@code --add-opens java.base/java.lang=ALL-UNNAMED}.
+ */
+ @Test
+ public void testUnboundedChainOfNonMatchingTerminates() {
+ Throwable head = new RuntimeException();
+ Throwable cursor = head;
+ for (int i = 1; i < 1024; i++) {
+ Throwable next = new RuntimeException();
+ cursor.initCause(next);
+ cursor = next;
+ }
+ assertFalse(ConnectionFailureUtils.isConnectionFailure(head));
+ }
+}
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestSlidingWindow.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestSlidingWindow.java
index 369426bcfd08..ba891e044b75 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestSlidingWindow.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestSlidingWindow.java
@@ -23,7 +23,7 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.time.Duration;
-import org.apache.ozone.test.TestClock;
+import org.apache.ozone.test.MockClock;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -32,11 +32,11 @@
*/
class TestSlidingWindow {
- private TestClock testClock;
+ private MockClock testClock;
@BeforeEach
void setup() {
- testClock = TestClock.newInstance();
+ testClock = MockClock.newInstance();
}
@Test
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/io_/retry/TestRetryProxy.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/io_/retry/TestRetryProxy.java
new file mode 100644
index 000000000000..4239d49a02c7
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/io_/retry/TestRetryProxy.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io_.retry;
+
+import static org.apache.hadoop.io_.retry.RetryPolicies.RETRY_FOREVER;
+import static org.apache.hadoop.io_.retry.RetryPolicies.TRY_ONCE_THEN_FAIL;
+import static org.apache.hadoop.io_.retry.RetryPolicies.exponentialBackoffRetry;
+import static org.apache.hadoop.io_.retry.RetryPolicies.retryForeverWithFixedSleep;
+import static org.apache.hadoop.io_.retry.RetryPolicies.retryUpToMaximumCountWithFixedSleep;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyBoolean;
+import static org.mockito.Mockito.anyInt;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import javax.security.sasl.SaslException;
+import org.apache.hadoop.io.retry.Idempotent;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryPolicy.RetryAction;
+import org.apache.hadoop.io.retry.RetryPolicy.RetryAction.RetryDecision;
+import org.apache.hadoop.io_.retry.UnreliableInterface.UnreliableException;
+import org.apache.hadoop.ipc_.ProtocolTranslator;
+import org.apache.hadoop.security.AccessControlException;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * TestRetryProxy tests the behaviour of the {@link RetryPolicy} class using
+ * a certain method of {@link UnreliableInterface} implemented by
+ * {@link UnreliableImplementation}.
+ *
+ * Some methods may be sensitive to the {@link Idempotent} annotation
+ * (annotated in {@link UnreliableInterface}).
+ */
+public class TestRetryProxy {
+
+ private UnreliableImplementation unreliableImpl;
+ private RetryAction caughtRetryAction = null;
+
+ @BeforeEach
+ public void setUp() throws Exception {
+ unreliableImpl = new UnreliableImplementation();
+ }
+
+ // answer mockPolicy's method with realPolicy, caught method's return value
+ private void setupMockPolicy(RetryPolicy mockPolicy,
+ final RetryPolicy realPolicy) throws Exception {
+ when(mockPolicy.shouldRetry(any(Exception.class), anyInt(), anyInt(), anyBoolean()))
+ .thenAnswer(invocation -> {
+ Object[] args = invocation.getArguments();
+ Exception e = (Exception) args[0];
+ int retries = (int) args[1];
+ int failovers = (int) args[2];
+ boolean isIdempotentOrAtMostOnce = (boolean) args[3];
+ caughtRetryAction = realPolicy.shouldRetry(e, retries, failovers,
+ isIdempotentOrAtMostOnce);
+ return caughtRetryAction;
+ });
+ }
+
+ @Test
+ public void testTryOnceThenFail() throws Exception {
+ RetryPolicy policy = mock(RetryPolicies.TryOnceThenFail.class);
+ RetryPolicy realPolicy = TRY_ONCE_THEN_FAIL;
+ setupMockPolicy(policy, realPolicy);
+
+ UnreliableInterface unreliable = (UnreliableInterface)
+ RetryProxy.create(UnreliableInterface.class, unreliableImpl, policy);
+ unreliable.alwaysSucceeds();
+ try {
+ unreliable.failsOnceThenSucceeds();
+ fail("Should fail");
+ } catch (UnreliableException e) {
+ // expected
+ verify(policy, times(1)).shouldRetry(any(Exception.class), anyInt(),
+ anyInt(), anyBoolean());
+ assertEquals(RetryDecision.FAIL, caughtRetryAction.action);
+ assertEquals("try once and fail.", caughtRetryAction.reason);
+ } catch (Exception e) {
+ fail("Other exception other than UnreliableException should also get " +
+ "failed.");
+ }
+ }
+
+ /**
+ * Test for {@link RetryInvocationHandler#isRpcInvocation(Object)}.
+ */
+ @Test
+ public void testRpcInvocation() throws Exception {
+ // For a proxy method should return true
+ final UnreliableInterface unreliable = (UnreliableInterface)
+ RetryProxy.create(UnreliableInterface.class, unreliableImpl, RETRY_FOREVER);
+ assertTrue(RetryInvocationHandler.isRpcInvocation(unreliable));
+
+ final AtomicInteger count = new AtomicInteger();
+ // Embed the proxy in ProtocolTranslator
+ ProtocolTranslator xlator = new ProtocolTranslator() {
+ @Override
+ public Object getUnderlyingProxyObject() {
+ count.getAndIncrement();
+ return unreliable;
+ }
+ };
+
+ // For a proxy wrapped in ProtocolTranslator method should return true
+ assertTrue(RetryInvocationHandler.isRpcInvocation(xlator));
+ // Ensure underlying proxy was looked at
+ assertEquals(1, count.get());
+
+ // For non-proxy the method must return false
+ assertFalse(RetryInvocationHandler.isRpcInvocation(new Object()));
+ }
+
+ @Test
+ public void testRetryForever() throws UnreliableException {
+ UnreliableInterface unreliable = (UnreliableInterface)
+ RetryProxy.create(UnreliableInterface.class, unreliableImpl, RETRY_FOREVER);
+ unreliable.alwaysSucceeds();
+ unreliable.failsOnceThenSucceeds();
+ unreliable.failsTenTimesThenSucceeds();
+ }
+
+ @Test
+ public void testRetryForeverWithFixedSleep() throws UnreliableException {
+ UnreliableInterface unreliable = (UnreliableInterface) RetryProxy.create(
+ UnreliableInterface.class, unreliableImpl,
+ retryForeverWithFixedSleep(1, TimeUnit.MILLISECONDS));
+ unreliable.alwaysSucceeds();
+ unreliable.failsOnceThenSucceeds();
+ unreliable.failsTenTimesThenSucceeds();
+ }
+
+ @Test
+ public void testRetryUpToMaximumCountWithFixedSleep() throws
+ Exception {
+
+ RetryPolicy policy = mock(RetryPolicies.RetryUpToMaximumCountWithFixedSleep.class);
+ int maxRetries = 8;
+ RetryPolicy realPolicy = retryUpToMaximumCountWithFixedSleep(maxRetries, 1, TimeUnit.NANOSECONDS);
+ setupMockPolicy(policy, realPolicy);
+
+ UnreliableInterface unreliable = (UnreliableInterface)
+ RetryProxy.create(UnreliableInterface.class, unreliableImpl, policy);
+ // shouldRetry += 1
+ unreliable.alwaysSucceeds();
+ // shouldRetry += 2
+ unreliable.failsOnceThenSucceeds();
+ try {
+ // shouldRetry += (maxRetries -1) (just failed once above)
+ unreliable.failsTenTimesThenSucceeds();
+ fail("Should fail");
+ } catch (UnreliableException e) {
+ // expected
+ verify(policy, times(maxRetries + 2)).shouldRetry(any(Exception.class),
+ anyInt(), anyInt(), anyBoolean());
+ assertEquals(RetryDecision.FAIL, caughtRetryAction.action);
+ assertEquals(RetryPolicies.RetryUpToMaximumCountWithFixedSleep.constructReasonString(
+ maxRetries), caughtRetryAction.reason);
+ } catch (Exception e) {
+ fail("Other exception other than UnreliableException should also get " +
+ "failed.");
+ }
+ }
+
+ @Test
+ public void testExponentialRetry() throws UnreliableException {
+ UnreliableInterface unreliable = (UnreliableInterface) RetryProxy.create(UnreliableInterface.class, unreliableImpl,
+ exponentialBackoffRetry(5, 1L, TimeUnit.NANOSECONDS));
+ unreliable.alwaysSucceeds();
+ unreliable.failsOnceThenSucceeds();
+ try {
+ unreliable.failsTenTimesThenSucceeds();
+ fail("Should fail");
+ } catch (UnreliableException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testRetryInterruptible() throws Throwable {
+ final UnreliableInterface unreliable = (UnreliableInterface)
+ RetryProxy.create(UnreliableInterface.class, unreliableImpl,
+ retryUpToMaximumCountWithFixedSleep(10, 10, TimeUnit.SECONDS));
+
+ final CountDownLatch latch = new CountDownLatch(1);
+ final AtomicReference futureThread = new AtomicReference();
+ ExecutorService exec = Executors.newSingleThreadExecutor();
+ try {
+ Future future = exec.submit(() -> {
+ futureThread.set(Thread.currentThread());
+ latch.countDown();
+ try {
+ unreliable.alwaysFailsWithFatalException();
+ } catch (UndeclaredThrowableException ute) {
+ return ute.getCause();
+ }
+ return null;
+ });
+ latch.await();
+ Thread.sleep(1000); // time to fail and sleep
+ assertTrue(futureThread.get().isAlive());
+ futureThread.get().interrupt();
+ Throwable e = future.get(1, TimeUnit.SECONDS); // should return immediately
+ assertNotNull(e);
+ assertEquals(InterruptedIOException.class, e.getClass());
+ assertEquals("Retry interrupted", e.getMessage());
+ assertEquals(InterruptedException.class, e.getCause().getClass());
+ assertEquals("sleep interrupted", e.getCause().getMessage());
+ } finally {
+ exec.shutdown();
+ }
+ }
+
+ @Test
+ public void testNoRetryOnSaslError() throws Exception {
+ RetryPolicy policy = mock(RetryPolicy.class);
+ RetryPolicy realPolicy = RetryPolicies.failoverOnNetworkException(5);
+ setupMockPolicy(policy, realPolicy);
+
+ UnreliableInterface unreliable = (UnreliableInterface) RetryProxy.create(
+ UnreliableInterface.class, unreliableImpl, policy);
+
+ try {
+ unreliable.failsWithSASLExceptionTenTimes();
+ fail("Should fail");
+ } catch (SaslException e) {
+ // expected
+ verify(policy, times(1)).shouldRetry(any(Exception.class), anyInt(),
+ anyInt(), anyBoolean());
+ assertEquals(RetryDecision.FAIL, caughtRetryAction.action);
+ }
+ }
+
+ @Test
+ public void testNoRetryOnAccessControlException() throws Exception {
+ RetryPolicy policy = mock(RetryPolicy.class);
+ RetryPolicy realPolicy = RetryPolicies.failoverOnNetworkException(5);
+ setupMockPolicy(policy, realPolicy);
+
+ UnreliableInterface unreliable = (UnreliableInterface) RetryProxy.create(
+ UnreliableInterface.class, unreliableImpl, policy);
+
+ try {
+ unreliable.failsWithAccessControlExceptionEightTimes();
+ fail("Should fail");
+ } catch (AccessControlException e) {
+ // expected
+ verify(policy, times(1)).shouldRetry(any(Exception.class), anyInt(),
+ anyInt(), anyBoolean());
+ assertEquals(RetryDecision.FAIL, caughtRetryAction.action);
+ }
+ }
+
+ @Test
+ public void testWrappedAccessControlException() throws Exception {
+ RetryPolicy policy = mock(RetryPolicy.class);
+ RetryPolicy realPolicy = RetryPolicies.failoverOnNetworkException(5);
+ setupMockPolicy(policy, realPolicy);
+
+ UnreliableInterface unreliable = (UnreliableInterface) RetryProxy.create(
+ UnreliableInterface.class, unreliableImpl, policy);
+
+ try {
+ unreliable.failsWithWrappedAccessControlException();
+ fail("Should fail");
+ } catch (IOException expected) {
+ verify(policy, times(1)).shouldRetry(any(Exception.class), anyInt(),
+ anyInt(), anyBoolean());
+ assertEquals(RetryDecision.FAIL, caughtRetryAction.action);
+ }
+ }
+}
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/io_/retry/UnreliableImplementation.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/io_/retry/UnreliableImplementation.java
new file mode 100644
index 000000000000..71fdc8956b98
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/io_/retry/UnreliableImplementation.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io_.retry;
+
+import java.io.IOException;
+import javax.security.sasl.SaslException;
+import org.apache.hadoop.security.AccessControlException;
+
+/**
+ * For the usage and purpose of this class see {@link UnreliableInterface}
+ * which this class implements.
+ *
+ * @see UnreliableInterface
+ */
+class UnreliableImplementation implements UnreliableInterface {
+
+ private int failsOnceInvocationCount;
+ private int failsTenTimesInvocationCount;
+ private int failsWithSASLExceptionTenTimesInvocationCount;
+ private int failsWithAccessControlExceptionInvocationCount;
+
+ @Override
+ public void alwaysSucceeds() {
+ // do nothing
+ }
+
+ @Override
+ public void alwaysFailsWithFatalException() throws FatalException {
+ throw new FatalException();
+ }
+
+ @Override
+ public void failsOnceThenSucceeds() throws UnreliableException {
+ if (failsOnceInvocationCount++ == 0) {
+ throw new UnreliableException();
+ }
+ }
+
+ @Override
+ public void failsTenTimesThenSucceeds() throws UnreliableException {
+ if (failsTenTimesInvocationCount++ < 10) {
+ throw new UnreliableException();
+ }
+ }
+
+ @Override
+ public void failsWithSASLExceptionTenTimes() throws SaslException {
+ if (failsWithSASLExceptionTenTimesInvocationCount++ < 10) {
+ throw new SaslException();
+ }
+ }
+
+ @Override
+ public void failsWithAccessControlExceptionEightTimes()
+ throws AccessControlException {
+ if (failsWithAccessControlExceptionInvocationCount++ < 8) {
+ throw new AccessControlException();
+ }
+ }
+
+ @Override
+ public void failsWithWrappedAccessControlException()
+ throws IOException {
+ AccessControlException ace = new AccessControlException();
+ IOException ioe = new IOException(ace);
+ throw new IOException(ioe);
+ }
+}
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/io_/retry/UnreliableInterface.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/io_/retry/UnreliableInterface.java
new file mode 100644
index 000000000000..1879250d060f
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/io_/retry/UnreliableInterface.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io_.retry;
+
+import java.io.IOException;
+import javax.security.sasl.SaslException;
+import org.apache.hadoop.io.retry.FailoverProxyProvider;
+import org.apache.hadoop.io.retry.Idempotent;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.security.AccessControlException;
+
+/**
+ * The methods of UnreliableInterface could throw exceptions in a
+ * predefined way. It is currently used for testing {@link RetryPolicy}
+ * and {@link FailoverProxyProvider} classes, but can be potentially used
+ * to test any class's behaviour where an underlying interface or class
+ * may throw exceptions.
+ *
+ * Some methods may be annotated with the {@link Idempotent} annotation.
+ * In order to test those some methods of UnreliableInterface are annotated,
+ * but they are not actually Idempotent functions.
+ *
+ */
+interface UnreliableInterface {
+
+ class UnreliableException extends Exception {
+ // no body
+ }
+
+ class FatalException extends UnreliableException {
+ // no body
+ }
+
+ void alwaysSucceeds() throws UnreliableException;
+
+ void alwaysFailsWithFatalException() throws FatalException;
+
+ void failsOnceThenSucceeds() throws UnreliableException;
+
+ void failsTenTimesThenSucceeds() throws UnreliableException;
+
+ void failsWithSASLExceptionTenTimes() throws SaslException;
+
+ @Idempotent
+ void failsWithAccessControlExceptionEightTimes()
+ throws AccessControlException;
+
+ @Idempotent
+ void failsWithWrappedAccessControlException()
+ throws IOException;
+}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotWithBucketLinkingLegacy.java b/hadoop-hdds/common/src/test/java/org/apache/ratis/util/RatisUtilTestUtil.java
similarity index 69%
rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotWithBucketLinkingLegacy.java
rename to hadoop-hdds/common/src/test/java/org/apache/ratis/util/RatisUtilTestUtil.java
index 62bc2d678871..38ac252c7d07 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotWithBucketLinkingLegacy.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/ratis/util/RatisUtilTestUtil.java
@@ -15,16 +15,16 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.om.snapshot;
+package org.apache.ratis.util;
-import static org.apache.hadoop.ozone.om.helpers.BucketLayout.LEGACY;
+import java.util.List;
-/**
- * Test OmSnapshot for Legacy bucket type.
- */
-public class TestOmSnapshotWithBucketLinkingLegacy extends TestOmSnapshot {
+/** Test util for the {@link org.apache.ratis.util} package. */
+public final class RatisUtilTestUtil {
+
+ private RatisUtilTestUtil() { }
- public TestOmSnapshotWithBucketLinkingLegacy() throws Exception {
- super(LEGACY, false, false, false, true);
+ public static List getValues(WeakValueCache cache) {
+ return cache.getValues();
}
}
diff --git a/hadoop-hdds/config/pom.xml b/hadoop-hdds/config/pom.xml
index 1aa00a4dc45c..8ded8e165535 100644
--- a/hadoop-hdds/config/pom.xml
+++ b/hadoop-hdds/config/pom.xml
@@ -17,10 +17,10 @@
org.apache.ozone
hdds
- 2.2.0-SNAPSHOT
+ 2.3.0-SNAPSHOT
hdds-config
- 2.2.0-SNAPSHOT
+ 2.3.0-SNAPSHOT
jar
Apache Ozone HDDS Config
Apache Ozone Distributed Data Store Config Tools
diff --git a/hadoop-hdds/container-service/pom.xml b/hadoop-hdds/container-service/pom.xml
index a0034eb78f4f..489fd7e21391 100644
--- a/hadoop-hdds/container-service/pom.xml
+++ b/hadoop-hdds/container-service/pom.xml
@@ -17,10 +17,10 @@
org.apache.ozone
hdds
- 2.2.0-SNAPSHOT
+ 2.3.0-SNAPSHOT
hdds-container-service
- 2.2.0-SNAPSHOT
+ 2.3.0-SNAPSHOT
jar
Apache Ozone HDDS Container Service
Apache Ozone Distributed Data Store Container Service
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBean.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBean.java
index 163d1398c949..7cfae9218be6 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBean.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBean.java
@@ -33,6 +33,13 @@ public interface DNMXBean extends ServiceRuntimeInfo {
*/
String getHostname();
+ /**
+ * Gets the datanode UUID.
+ *
+ * @return the datanode UUID for the datanode.
+ */
+ String getDatanodeUuid();
+
/**
* Gets the client rpc port.
*
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBeanImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBeanImpl.java
index 82c59f8f50cf..ecc66121da6c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBeanImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBeanImpl.java
@@ -26,6 +26,7 @@
public class DNMXBeanImpl extends ServiceRuntimeInfoImpl implements DNMXBean {
private String hostName;
+ private String datanodeUuid;
private String clientRpcPort;
private String httpPort;
private String httpsPort;
@@ -39,6 +40,11 @@ public String getHostname() {
return hostName;
}
+ @Override
+ public String getDatanodeUuid() {
+ return datanodeUuid;
+ }
+
@Override
public String getClientRpcPort() {
return clientRpcPort;
@@ -62,6 +68,10 @@ public void setHostName(String hostName) {
this.hostName = hostName;
}
+ public void setDatanodeUuid(String datanodeUuid) {
+ this.datanodeUuid = datanodeUuid;
+ }
+
public void setClientRpcPort(String rpcPort) {
this.clientRpcPort = rpcPort;
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index 1f08dacc90eb..c2a30d97d529 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -39,7 +39,6 @@
import com.google.common.collect.Sets;
import java.io.File;
import java.io.IOException;
-import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
@@ -71,6 +70,7 @@
import org.apache.hadoop.hdds.protocol.DiskBalancerProtocol;
import org.apache.hadoop.hdds.protocol.SecretKeyProtocol;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
+import org.apache.hadoop.hdds.scm.net.HostAndPort;
import org.apache.hadoop.hdds.security.SecurityConfig;
import org.apache.hadoop.hdds.security.symmetric.DefaultSecretKeyClient;
import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
@@ -248,6 +248,7 @@ public String getNamespace() {
datanodeDetails = initializeDatanodeDetails();
datanodeDetails.setHostName(hostname);
serviceRuntimeInfo.setHostName(hostname);
+ serviceRuntimeInfo.setDatanodeUuid(datanodeDetails.getUuidString());
datanodeDetails.validateDatanodeIpAddress();
datanodeDetails.setVersion(
HddsVersionInfo.HDDS_VERSION_INFO.getVersion());
@@ -710,8 +711,11 @@ private String reconfigDeletingServiceWorkers(String value) {
}
private String reconfigReplicationStreamsLimit(String value) {
+ int poolSize = Integer.parseInt(value);
getDatanodeStateMachine().getContainer().getReplicationServer()
- .setPoolSize(Integer.parseInt(value));
+ .setPoolSize(poolSize);
+ getDatanodeStateMachine().getSupervisor()
+ .setReplicationMaxStreams(poolSize);
return value;
}
@@ -760,12 +764,12 @@ private String reconfigScmNodes(String value) {
LOG.info("Reconfiguring SCM nodes for service ID {} with new SCM nodes {} and remove SCM nodes {}",
scmServiceId, scmNodesIdsToAdd, scmNodesIdsToRemove);
- Collection> scmToAdd = HddsServerUtil.getSCMAddressForDatanodes(
+ final Collection> scmToAdd = HddsServerUtil.getSCMAddressForDatanodes(
getConf(), scmServiceId, scmNodesIdsToAdd);
if (scmToAdd == null) {
throw new IllegalStateException("Reconfiguration failed to get SCM address to add due to wrong configuration");
}
- Collection> scmToRemove = HddsServerUtil.getSCMAddressForDatanodes(
+ final Collection> scmToRemove = HddsServerUtil.getSCMAddressForDatanodes(
getConf(), scmServiceId, scmNodesIdsToRemove);
if (scmToRemove == null) {
throw new IllegalArgumentException(
@@ -786,10 +790,10 @@ private String reconfigScmNodes(String value) {
}
// Add the new SCM servers
- for (Pair pair : scmToAdd) {
+ for (Pair pair : scmToAdd) {
String scmNodeId = pair.getLeft();
- InetSocketAddress scmAddress = pair.getRight();
- if (scmAddress.isUnresolved()) {
+ final HostAndPort scmAddress = pair.getRight();
+ if (scmAddress.getAddress().isUnresolved()) {
LOG.warn("Reconfiguration failed to add SCM address {} for SCM service {} since it can't " +
"be resolved, skipping", scmAddress, scmServiceId);
continue;
@@ -805,9 +809,9 @@ private String reconfigScmNodes(String value) {
}
// Remove the old SCM server
- for (Pair pair : scmToRemove) {
+ for (Pair pair : scmToRemove) {
String scmNodeId = pair.getLeft();
- InetSocketAddress scmAddress = pair.getRight();
+ final HostAndPort scmAddress = pair.getRight();
try {
connectionManager.removeSCMServer(scmAddress);
context.removeEndpoint(scmAddress);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/DNContainerOperationClient.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/DNContainerOperationClient.java
index 8556ce5f6d22..d1b5d99a5ecf 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/DNContainerOperationClient.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/DNContainerOperationClient.java
@@ -33,7 +33,6 @@
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
import org.apache.hadoop.hdds.security.SecurityConfig;
import org.apache.hadoop.hdds.security.symmetric.SecretKeySignerClient;
@@ -119,7 +118,7 @@ public ContainerProtos.ContainerChecksumInfo getContainerChecksumInfo(long conta
public static Pipeline createSingleNodePipeline(DatanodeDetails dn) {
return Pipeline.newBuilder()
.setNodes(ImmutableList.of(dn))
- .setId(PipelineID.valueOf(dn.getUuid()))
+ .setId(dn.getID().toPipelineID())
.setState(Pipeline.PipelineState.CLOSED)
.setReplicationConfig(StandaloneReplicationConfig.getInstance(
HddsProtos.ReplicationFactor.ONE)).build();
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockDeletingServiceMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockDeletingServiceMetrics.java
index 91bb8fbc59ac..0151bfaea0fe 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockDeletingServiceMetrics.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockDeletingServiceMetrics.java
@@ -87,7 +87,7 @@ public final class BlockDeletingServiceMetrics {
private BlockDeletingServiceMetrics() {
}
- public static BlockDeletingServiceMetrics create() {
+ public static synchronized BlockDeletingServiceMetrics create() {
if (instance == null) {
MetricsSystem ms = DefaultMetricsSystem.instance();
instance = ms.register(SOURCE_NAME, "BlockDeletingService",
@@ -100,7 +100,7 @@ public static BlockDeletingServiceMetrics create() {
/**
* Unregister the metrics instance.
*/
- public static void unRegister() {
+ public static synchronized void unRegister() {
instance = null;
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/BlockDeletingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/BlockDeletingService.java
index 27b3ec418647..00a46305ee8b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/BlockDeletingService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/BlockDeletingService.java
@@ -123,7 +123,7 @@ public void registerReconfigCallbacks(ReconfigurationHandler handler) {
});
}
- public synchronized void updateAndRestart(OzoneConfiguration ozoneConf) {
+ public void updateAndRestart(OzoneConfiguration ozoneConf) {
long newInterval = ozoneConf.getTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT, TimeUnit.SECONDS);
int newCorePoolSize = ozoneConf.getInt(OZONE_BLOCK_DELETING_SERVICE_WORKERS,
@@ -134,11 +134,15 @@ public synchronized void updateAndRestart(OzoneConfiguration ozoneConf) {
", core pool size {} and timeout {} {}",
newInterval, TimeUnit.SECONDS.name().toLowerCase(), newCorePoolSize, newTimeout,
TimeUnit.NANOSECONDS.name().toLowerCase());
+ // shutdown() awaits the executor; do not hold this monitor (same object as
+ // BackgroundService.PeriodicalTask) or the pool thread can deadlock.
shutdown();
- setInterval(newInterval, TimeUnit.SECONDS);
- setPoolSize(newCorePoolSize);
- setServiceTimeoutInNanos(newTimeout);
- start();
+ synchronized (this) {
+ setInterval(newInterval, TimeUnit.SECONDS);
+ setPoolSize(newCorePoolSize);
+ setServiceTimeoutInNanos(newTimeout);
+ start();
+ }
}
/**
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
index d122dfb587f9..43a3fc0a00bd 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
@@ -73,8 +73,9 @@ public class ContainerSet implements Iterable> {
ConcurrentSkipListMap<>();
private final ConcurrentSkipListSet missingContainerSet =
new ConcurrentSkipListSet<>();
- private final ConcurrentSkipListMap recoveringContainerMap =
- new ConcurrentSkipListMap<>();
+
+ private final ConcurrentSkipListSet recoveringContainerSet =
+ new ConcurrentSkipListSet<>();
private final Clock clock;
private long recoveringTimeout;
@Nullable
@@ -208,8 +209,9 @@ private boolean addContainer(Container container, boolean overwrite) throws
updateContainerIdTable(containerId, container.getContainerData());
missingContainerSet.remove(containerId);
if (container.getContainerData().getState() == RECOVERING) {
- recoveringContainerMap.put(
- clock.millis() + recoveringTimeout, containerId);
+ recoveringContainerSet.add(
+ new RecoveringContainer(clock.millis() + recoveringTimeout,
+ containerId));
}
HddsVolume volume = container.getContainerData().getVolume();
if (volume != null) {
@@ -421,16 +423,16 @@ public boolean removeRecoveringContainer(long containerId) {
Preconditions.checkState(containerId >= 0,
"Container Id cannot be negative.");
//it might take a little long time to iterate all the entries
- // in recoveringContainerMap, but it seems ok here since:
+ // in recoveringContainerSet, but it seems ok here since:
// 1 In the vast majority of cases，there will not be too
// many recovering containers.
// 2 closing container is not a sort of urgent action
//
// we can revisit here if any performance problem happens
- Iterator> it = getRecoveringContainerIterator();
+ Iterator it = getRecoveringContainerIterator();
while (it.hasNext()) {
- Map.Entry entry = it.next();
- if (entry.getValue() == containerId) {
+ RecoveringContainer entry = it.next();
+ if (entry.getContainerId() == containerId) {
it.remove();
return true;
}
@@ -489,11 +491,11 @@ public Iterator> iterator() {
/**
* Return an container Iterator over
- * {@link ContainerSet#recoveringContainerMap}.
- * @return {@literal Iterator>}
+ * {@link ContainerSet#recoveringContainerSet}.
+ * @return {@literal Iterator}
*/
- public Iterator> getRecoveringContainerIterator() {
- return recoveringContainerMap.entrySet().iterator();
+ public Iterator getRecoveringContainerIterator() {
+ return recoveringContainerSet.iterator();
}
/**
@@ -667,4 +669,52 @@ public void buildMissingContainerSetAndValidate(Map container2BCSID
}
});
}
+
+ /**
+ * A class that holds information about a recovering container.
+ */
+ public static class RecoveringContainer
+ implements Comparable {
+ private final long timeout;
+ private final long containerId;
+
+ public RecoveringContainer(long timeout, long containerId) {
+ this.timeout = timeout;
+ this.containerId = containerId;
+ }
+
+ public long getTimeout() {
+ return timeout;
+ }
+
+ public long getContainerId() {
+ return containerId;
+ }
+
+ @Override
+ public int compareTo(RecoveringContainer other) {
+ int timeoutCompare = Long.compare(this.timeout, other.timeout);
+ if (timeoutCompare != 0) {
+ return timeoutCompare;
+ }
+ return Long.compare(this.containerId, other.containerId);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ RecoveringContainer that = (RecoveringContainer) o;
+ return timeout == that.timeout && containerId == that.containerId;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(timeout, containerId);
+ }
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
index 1fd094b55f1c..506dd79c37dc 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
@@ -145,7 +145,7 @@ public class DatanodeConfiguration extends ReconfigurableConfig {
static final int BLOCK_DELETE_THREADS_DEFAULT = 5;
public static final String GRPC_SO_BACKLOG_KEY = "hdds.datanode.grpc.so.backlog";
- public static final int GRPC_SO_BACKLOG_DEFAULT = 4096;
+ public static final int GRPC_SO_BACKLOG_DEFAULT = 256;
public static final String BLOCK_DELETE_COMMAND_WORKER_INTERVAL =
"hdds.datanode.block.delete.command.worker.interval";
@@ -167,7 +167,7 @@ public class DatanodeConfiguration extends ReconfigurableConfig {
*/
@Config(key = "hdds.datanode.grpc.so.backlog",
type = ConfigType.INT,
- defaultValue = "4096",
+ defaultValue = "256",
tags = {DATANODE},
description = "The SO_BACKLOG value for the Datanode gRPC server socket. " +
"This limits the number of pending connections in the kernel's " +
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeQueueMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeQueueMetrics.java
index d442b95285d8..f376d640a3c4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeQueueMetrics.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeQueueMetrics.java
@@ -21,11 +21,11 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.CaseFormat;
-import java.net.InetSocketAddress;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.text.WordUtils;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.scm.net.HostAndPort;
import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsInfo;
@@ -68,9 +68,9 @@ public final class DatanodeQueueMetrics implements MetricsSource {
private Map stateContextCommandQueueMap;
private Map commandDispatcherQueueMap;
- private Map incrementalReportsQueueMap;
- private Map containerActionQueueMap;
- private Map pipelineActionQueueMap;
+ private Map incrementalReportsQueueMap;
+ private Map containerActionQueueMap;
+ private Map pipelineActionQueueMap;
public DatanodeQueueMetrics(DatanodeStateMachine datanodeStateMachine) {
this.registry = new MetricsRegistry(METRICS_SOURCE_NAME);
@@ -132,19 +132,19 @@ public void getMetrics(MetricsCollector collector, boolean b) {
tmpEnum.get(entry.getKey()));
}
- for (Map.Entry entry:
+ for (Map.Entry entry:
incrementalReportsQueueMap.entrySet()) {
builder.addGauge(entry.getValue(),
datanodeStateMachine.getContext()
.getIncrementalReportQueueSize().getOrDefault(entry.getKey(), 0));
}
- for (Map.Entry entry:
+ for (Map.Entry entry:
containerActionQueueMap.entrySet()) {
builder.addGauge(entry.getValue(),
datanodeStateMachine.getContext()
.getContainerActionQueueSize().getOrDefault(entry.getKey(), 0));
}
- for (Map.Entry entry:
+ for (Map.Entry entry:
pipelineActionQueueMap.entrySet()) {
builder.addGauge(entry.getValue(),
datanodeStateMachine.getContext().getPipelineActionQueueSize()
@@ -157,7 +157,7 @@ public static synchronized void unRegister() {
DefaultMetricsSystem.instance().unregisterSource(METRICS_SOURCE_NAME);
}
- public void addEndpoint(InetSocketAddress endpoint) {
+ public void addEndpoint(HostAndPort endpoint) {
incrementalReportsQueueMap.computeIfAbsent(endpoint,
k -> getMetricsInfo(INCREMENTAL_REPORT_QUEUE_PREFIX,
CaseFormat.UPPER_UNDERSCORE
@@ -172,7 +172,7 @@ public void addEndpoint(InetSocketAddress endpoint) {
.to(CaseFormat.UPPER_CAMEL, k.getHostName())));
}
- public void removeEndpoint(InetSocketAddress endpoint) {
+ public void removeEndpoint(HostAndPort endpoint) {
incrementalReportsQueueMap.remove(endpoint);
containerActionQueueMap.remove(endpoint);
pipelineActionQueueMap.remove(endpoint);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index 2f53178e9bf9..00a9bdd1043a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -70,9 +70,7 @@
import org.apache.hadoop.ozone.container.ec.reconstruction.ECReconstructionCoordinator;
import org.apache.hadoop.ozone.container.ec.reconstruction.ECReconstructionMetrics;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.container.replication.ContainerImporter;
import org.apache.hadoop.ozone.container.replication.ContainerReplicator;
-import org.apache.hadoop.ozone.container.replication.DownloadAndImportReplicator;
import org.apache.hadoop.ozone.container.replication.GrpcContainerUploader;
import org.apache.hadoop.ozone.container.replication.MeasuredReplicator;
import org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicationSource;
@@ -80,7 +78,6 @@
import org.apache.hadoop.ozone.container.replication.ReplicationServer.ReplicationConfig;
import org.apache.hadoop.ozone.container.replication.ReplicationSupervisor;
import org.apache.hadoop.ozone.container.replication.ReplicationSupervisorMetrics;
-import org.apache.hadoop.ozone.container.replication.SimpleContainerDownloader;
import org.apache.hadoop.ozone.container.upgrade.DataNodeUpgradeFinalizer;
import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
@@ -125,7 +122,6 @@ public class DatanodeStateMachine implements Closeable {
* constructor in a non-thread-safe way - see HDDS-3116.
*/
private final ReadWriteLock constructionLock = new ReentrantReadWriteLock();
- private final MeasuredReplicator pullReplicatorWithMetrics;
private final MeasuredReplicator pushReplicatorWithMetrics;
private final ReplicationSupervisorMetrics replicationSupervisorMetrics;
private final NettyMetrics nettyMetrics;
@@ -195,21 +191,11 @@ public DatanodeStateMachine(HddsDatanodeService hddsDatanodeService,
}
nextHB = new AtomicLong(Time.monotonicNow());
- ContainerImporter importer = new ContainerImporter(conf,
- container.getContainerSet(),
- container.getController(),
- container.getVolumeSet(),
- volumeChoosingPolicy);
- ContainerReplicator pullReplicator = new DownloadAndImportReplicator(
- conf, container.getContainerSet(),
- importer,
- new SimpleContainerDownloader(conf, certClient));
ContainerReplicator pushReplicator = new PushReplicator(conf,
new OnDemandContainerReplicationSource(container.getController()),
new GrpcContainerUploader(conf, certClient, container.getController())
);
- pullReplicatorWithMetrics = new MeasuredReplicator(pullReplicator, "pull");
pushReplicatorWithMetrics = new MeasuredReplicator(pushReplicator, "push");
ReplicationConfig replicationConfig =
@@ -266,8 +252,7 @@ public DatanodeStateMachine(HddsDatanodeService hddsDatanodeService,
dnConf.getCommandQueueLimit(), threadNamePrefix))
.addHandler(new DeleteBlocksCommandHandler(getContainer(),
conf, dnConf, threadNamePrefix))
- .addHandler(new ReplicateContainerCommandHandler(conf, supervisor,
- pullReplicatorWithMetrics, pushReplicatorWithMetrics))
+ .addHandler(new ReplicateContainerCommandHandler(supervisor, pushReplicatorWithMetrics))
.addHandler(reconstructECContainersCommandHandler)
.addHandler(new DeleteContainerCommandHandler(
dnConf.getContainerDeleteThreads(), clock,
@@ -652,7 +637,7 @@ public EnumCounters getQueuedCommandCount() {
*/
public synchronized void stopDaemon() {
try {
- IOUtils.close(LOG, pushReplicatorWithMetrics, pullReplicatorWithMetrics);
+ IOUtils.close(LOG, pushReplicatorWithMetrics);
supervisor.stop();
context.setShutdownGracefully();
context.setState(DatanodeStates.SHUTDOWN);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java
index 94bc0549e66a..b4a9a6e1a267 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java
@@ -22,7 +22,6 @@
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.io.Closeable;
-import java.net.InetSocketAddress;
import java.time.ZonedDateTime;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -31,6 +30,7 @@
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.scm.net.HostAndPort;
import org.apache.hadoop.ozone.protocol.VersionResponse;
import org.apache.hadoop.ozone.protocolPB.ReconDatanodeProtocolPB;
import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolClientSideTranslatorPB;
@@ -46,7 +46,7 @@ public class EndpointStateMachine
LOG = LoggerFactory.getLogger(EndpointStateMachine.class);
private final StorageContainerDatanodeProtocolClientSideTranslatorPB endPoint;
private final AtomicLong missedCount;
- private final InetSocketAddress address;
+ private final HostAndPort hostAndPort;
private final Lock lock;
private final ConfigurationSource conf;
private EndPointStates state = EndPointStates.FIRST;
@@ -64,18 +64,17 @@ public class EndpointStateMachine
*
* @param endPoint - RPC endPoint.
*/
- public EndpointStateMachine(InetSocketAddress address,
+ public EndpointStateMachine(HostAndPort hostAndPort,
StorageContainerDatanodeProtocolClientSideTranslatorPB endPoint,
ConfigurationSource conf, String threadNamePrefix) {
this.endPoint = endPoint;
this.missedCount = new AtomicLong(0);
- this.address = address;
+ this.hostAndPort = hostAndPort;
lock = new ReentrantLock();
this.conf = conf;
executorService = Executors.newSingleThreadExecutor(
new ThreadFactoryBuilder()
- .setNameFormat(threadNamePrefix + "EndpointStateMachineTaskThread-"
- + this.address + "-%d ")
+ .setNameFormat(threadNamePrefix + "EndpointStateMachineTaskThread-" + hostAndPort + "-%d ")
.build());
}
@@ -180,7 +179,7 @@ public long getMissedCount() {
@Override
public String getAddressString() {
- return getAddress().toString();
+ return hostAndPort.getAddress().toString();
}
public void zeroMissedCount() {
@@ -192,8 +191,8 @@ public void zeroMissedCount() {
*
* @return - EndPoint.
*/
- public InetSocketAddress getAddress() {
- return this.address;
+ public HostAndPort getAddress() {
+ return hostAndPort;
}
/**
@@ -213,7 +212,7 @@ public InetSocketAddress getAddress() {
*/
@Override
public String toString() {
- return address.toString();
+ return hostAndPort.toString();
}
/**
@@ -236,14 +235,8 @@ public void logIfNeeded(Exception ex) {
long missedDurationSeconds = TimeUnit.MILLISECONDS.toSeconds(
this.getMissedCount() * getScmHeartbeatInterval(this.conf)
);
- LOG.warn(
- "Unable to communicate to {} server at {}:{} for past {} seconds.",
- serverName,
- address.getAddress(),
- address.getPort(),
- missedDurationSeconds,
- ex
- );
+ LOG.warn("Unable to communicate to {} server at {} past {} seconds.",
+ serverName, hostAndPort, missedDurationSeconds, ex);
}
if (LOG.isTraceEnabled()) {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
index 27d31c90fe63..25be82cd0414 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
@@ -24,7 +24,6 @@
import java.io.Closeable;
import java.io.IOException;
-import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -36,6 +35,7 @@
import javax.management.ObjectName;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.scm.net.HostAndPort;
import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.retry.RetryPolicy;
@@ -44,7 +44,6 @@
import org.apache.hadoop.ipc_.RPC;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine.EndPointStates;
import org.apache.hadoop.ozone.protocolPB.ReconDatanodeProtocolPB;
import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB;
@@ -62,7 +61,7 @@ public class SCMConnectionManager
LoggerFactory.getLogger(SCMConnectionManager.class);
private final ReadWriteLock mapLock;
- private final Map scmMachines;
+ private final Map scmMachines;
private final int rpcTimeout;
private final ConfigurationSource conf;
@@ -131,7 +130,7 @@ public void writeUnlock() {
* @param address - Address of the SCM machine to send heartbeat to.
* @throws IOException
*/
- public void addSCMServer(InetSocketAddress address,
+ public void addSCMServer(HostAndPort address,
String threadNamePrefix) throws IOException {
writeLock();
try {
@@ -157,7 +156,7 @@ public void addSCMServer(InetSocketAddress address,
StorageContainerDatanodeProtocolPB rpcProxy = RPC.getProtocolProxy(
StorageContainerDatanodeProtocolPB.class, version,
- address, UserGroupInformation.getCurrentUser(), hadoopConfig,
+ address.getAddress(), UserGroupInformation.getCurrentUser(), hadoopConfig,
NetUtils.getDefaultSocketFactory(hadoopConfig), getRpcTimeout(),
retryPolicy).getProxy();
@@ -180,7 +179,7 @@ public void addSCMServer(InetSocketAddress address,
* @param address Recon address.
* @throws IOException
*/
- public void addReconServer(InetSocketAddress address,
+ public void addReconServer(HostAndPort address,
String threadNamePrefix) throws IOException {
LOG.info("Adding Recon Server : {}", address.toString());
writeLock();
@@ -203,7 +202,7 @@ public void addReconServer(InetSocketAddress address,
TimeUnit.MILLISECONDS);
ReconDatanodeProtocolPB rpcProxy = RPC.getProtocolProxy(
ReconDatanodeProtocolPB.class, version,
- address, UserGroupInformation.getCurrentUser(), hadoopConfig,
+ address.getAddress(), UserGroupInformation.getCurrentUser(), hadoopConfig,
NetUtils.getDefaultSocketFactory(hadoopConfig), getRpcTimeout(),
retryPolicy).getProxy();
@@ -225,7 +224,7 @@ public void addReconServer(InetSocketAddress address,
* @param address - Address of the SCM machine to send heartbeat to.
* @throws IOException
*/
- public void removeSCMServer(InetSocketAddress address) throws IOException {
+ public void removeSCMServer(HostAndPort address) throws IOException {
writeLock();
try {
EndpointStateMachine endPoint = scmMachines.remove(address);
@@ -234,7 +233,9 @@ public void removeSCMServer(InetSocketAddress address) throws IOException {
"Ignoring the request.");
return;
}
- endPoint.setState(EndPointStates.SHUTDOWN);
+ // This is a normal reconfiguration removal. Do not set the endpoint to
+ // SHUTDOWN, as an in-flight task may report that state as a DN fatal
+ // shutdown.
endPoint.close();
} finally {
writeUnlock();
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
index 150159eb84ae..4dcb74b40d53 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
@@ -28,7 +28,6 @@
import com.google.protobuf.Descriptors.Descriptor;
import com.google.protobuf.Message;
import java.io.IOException;
-import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -70,6 +69,7 @@
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.scm.net.HostAndPort;
import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.ClosePipelineCommandHandler;
import org.apache.hadoop.ozone.container.common.states.DatanodeState;
@@ -112,16 +112,15 @@ public class StateContext {
private final DatanodeStateMachine parentDatanodeStateMachine;
private final AtomicLong stateExecutionCount;
private final ConfigurationSource conf;
- private final Set endpoints;
+ private final Set endpoints;
// Only the latest full report of each type is kept
private final AtomicReference containerReports;
private final AtomicReference nodeReport;
private final AtomicReference pipelineReports;
// Incremental reports are queued in the map below
- private final Map>
- incrementalReportsQueue;
- private final Map> containerActions;
- private final Map pipelineActions;
+ private final Map> incrementalReportsQueue;
+ private final Map> containerActions;
+ private final Map pipelineActions;
private DatanodeStateMachine.DatanodeStates state;
private boolean shutdownOnError = false;
private boolean shutdownGracefully = false;
@@ -129,7 +128,7 @@ public class StateContext {
private final AtomicLong lastHeartbeatSent;
// Endpoint -> ReportType -> Boolean of whether the full report should be
// queued in getFullReports call.
- private final Map> isFullReportReadyToBeSent;
// List of supported full report types.
private final List fullReportTypeList;
@@ -310,7 +309,7 @@ public void addIncrementalReport(Message report) {
// as an incremental message.
// see XceiverServerRatis#sendPipelineReport
synchronized (incrementalReportsQueue) {
- for (InetSocketAddress endpoint : endpoints) {
+ for (HostAndPort endpoint : endpoints) {
incrementalReportsQueue.get(endpoint).add(report);
}
}
@@ -349,7 +348,7 @@ public void refreshFullReport(Message report) {
* heartbeat.
*/
public void putBackReports(List reportsToPutBack,
- InetSocketAddress endpoint) {
+ HostAndPort endpoint) {
if (LOG.isDebugEnabled()) {
LOG.debug("endpoint: {}, size of reportsToPutBack: {}",
endpoint, reportsToPutBack.size());
@@ -375,8 +374,7 @@ public void putBackReports(List reportsToPutBack,
* @return List of reports
*/
public List getAllAvailableReports(
- InetSocketAddress endpoint
- ) {
+ HostAndPort endpoint) {
int maxLimit = Integer.MAX_VALUE;
// TODO: It is highly unlikely that we will reach maxLimit for the number
// for the number of reports, specially as it does not apply to the
@@ -400,7 +398,7 @@ public ContainerReportsProto getFullContainerReportDiscardPendingICR()
synchronized (parentDatanodeStateMachine
.getContainer()) {
synchronized (incrementalReportsQueue) {
- for (Map.Entry>
+ for (Map.Entry>
entry : incrementalReportsQueue.entrySet()) {
if (entry.getValue() != null) {
entry.getValue().removeIf(
@@ -419,7 +417,7 @@ public ContainerReportsProto getFullContainerReportDiscardPendingICR()
@VisibleForTesting
List getAllAvailableReportsUpToLimit(
- InetSocketAddress endpoint,
+ HostAndPort endpoint,
int limit) {
List reports = getFullReports(endpoint, limit);
List incrementalReports = getIncrementalReports(endpoint,
@@ -429,7 +427,7 @@ List getAllAvailableReportsUpToLimit(
}
List getIncrementalReports(
- InetSocketAddress endpoint, int maxLimit) {
+ HostAndPort endpoint, int maxLimit) {
List reportsToReturn = new LinkedList<>();
synchronized (incrementalReportsQueue) {
List reportsForEndpoint =
@@ -445,7 +443,7 @@ List getIncrementalReports(
}
List getFullReports(
- InetSocketAddress endpoint, int maxLimit) {
+ HostAndPort endpoint, int maxLimit) {
int count = 0;
Map mp = isFullReportReadyToBeSent.get(endpoint);
List fullReports = new LinkedList<>();
@@ -482,7 +480,7 @@ List getFullReports(
*/
public void addContainerAction(ContainerAction containerAction) {
synchronized (containerActions) {
- for (InetSocketAddress endpoint : endpoints) {
+ for (HostAndPort endpoint : endpoints) {
containerActions.get(endpoint).add(containerAction);
}
}
@@ -495,7 +493,7 @@ public void addContainerAction(ContainerAction containerAction) {
*/
public void addContainerActionIfAbsent(ContainerAction containerAction) {
synchronized (containerActions) {
- for (InetSocketAddress endpoint : endpoints) {
+ for (HostAndPort endpoint : endpoints) {
if (!containerActions.get(endpoint).contains(containerAction)) {
containerActions.get(endpoint).add(containerAction);
}
@@ -510,7 +508,7 @@ public void addContainerActionIfAbsent(ContainerAction containerAction) {
* @return {@literal List}
*/
public List getPendingContainerAction(
- InetSocketAddress endpoint,
+ HostAndPort endpoint,
int maxLimit) {
List containerActionList = new ArrayList<>();
synchronized (containerActions) {
@@ -538,7 +536,7 @@ public boolean addPipelineActionIfAbsent(PipelineAction pipelineAction) {
// Put only if the pipeline id with the same action is absent.
final PipelineKey key = new PipelineKey(pipelineAction);
boolean added = false;
- for (InetSocketAddress endpoint : endpoints) {
+ for (HostAndPort endpoint : endpoints) {
added = pipelineActions.get(endpoint).putIfAbsent(key, pipelineAction) || added;
}
return added;
@@ -551,7 +549,7 @@ public boolean addPipelineActionIfAbsent(PipelineAction pipelineAction) {
* @return {@literal List}
*/
public List getPendingPipelineAction(
- InetSocketAddress endpoint,
+ HostAndPort endpoint,
int maxLimit) {
final PipelineActionMap map = pipelineActions.get(endpoint);
if (map == null) {
@@ -894,7 +892,7 @@ public long getHeartbeatFrequency() {
return heartbeatFrequency.get();
}
- public void addEndpoint(InetSocketAddress endpoint) {
+ public void addEndpoint(HostAndPort endpoint) {
if (!endpoints.contains(endpoint)) {
this.endpoints.add(endpoint);
this.containerActions.put(endpoint, new LinkedList<>());
@@ -911,7 +909,7 @@ public void addEndpoint(InetSocketAddress endpoint) {
}
}
- public void removeEndpoint(InetSocketAddress endpoint) {
+ public void removeEndpoint(HostAndPort endpoint) {
this.endpoints.remove(endpoint);
this.containerActions.remove(endpoint);
this.pipelineActions.remove(endpoint);
@@ -948,17 +946,17 @@ public long getReconHeartbeatFrequency() {
return reconHeartbeatFrequency.get();
}
- public Map getPipelineActionQueueSize() {
+ public Map getPipelineActionQueueSize() {
return pipelineActions.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().size()));
}
- public Map getContainerActionQueueSize() {
+ public Map getContainerActionQueueSize() {
return containerActions.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().size()));
}
- public Map getIncrementalReportQueueSize() {
+ public Map getIncrementalReportQueueSize() {
return incrementalReportsQueue.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().size()));
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
index 45ea5bffdec4..c947e3a30821 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
@@ -73,6 +73,7 @@
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.Time;
+import org.apache.ratis.util.ExitUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -254,6 +255,9 @@ public void run() {
DeleteCmdInfo cmd = deleteCommandQueues.poll();
try {
processCmd(cmd);
+ } catch (Error e) {
+ ExitUtils.terminate(1,
+ "Fatal error while processing delete blocks command", e, LOG);
} catch (Throwable e) {
LOG.error("taskProcess failed.", e);
}
@@ -500,6 +504,9 @@ public void handleTasksResults(
DeleteBlockTransactionExecutionResult result = f.get();
handler.accept(result);
} catch (ExecutionException e) {
+ if (e.getCause() instanceof Error) {
+ throw (Error) e.getCause();
+ }
LOG.error("task failed.", e);
} catch (InterruptedException e) {
LOG.error("task interrupted.", e);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
index 135c6fdb0391..94290e1e34fd 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
@@ -18,9 +18,6 @@
package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
import com.google.common.base.Preconditions;
-import java.util.List;
-import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
@@ -31,32 +28,20 @@
import org.apache.hadoop.ozone.container.replication.ReplicationTask;
import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
- * Command handler to copy containers from sources.
+ * Command handler to push containers to a target datanode.
*/
public class ReplicateContainerCommandHandler implements CommandHandler {
- static final Logger LOG =
- LoggerFactory.getLogger(ReplicateContainerCommandHandler.class);
-
private ReplicationSupervisor supervisor;
- private ContainerReplicator downloadReplicator;
-
private ContainerReplicator pushReplicator;
private static final String METRIC_NAME = ReplicationTask.METRIC_NAME;
- public ReplicateContainerCommandHandler(
- ConfigurationSource conf,
- ReplicationSupervisor supervisor,
- ContainerReplicator downloadReplicator,
- ContainerReplicator pushReplicator) {
+ public ReplicateContainerCommandHandler(ReplicationSupervisor supervisor, ContainerReplicator pushReplicator) {
this.supervisor = supervisor;
- this.downloadReplicator = downloadReplicator;
this.pushReplicator = pushReplicator;
}
@@ -70,20 +55,13 @@ public void handle(SCMCommand command, OzoneContainer container,
final ReplicateContainerCommand replicateCommand =
(ReplicateContainerCommand) command;
- final List sourceDatanodes =
- replicateCommand.getSourceDatanodes();
final long containerID = replicateCommand.getContainerID();
- final DatanodeDetails target = replicateCommand.getTargetDatanode();
-
- Preconditions.checkArgument(!sourceDatanodes.isEmpty() || target != null,
- "Replication command is received for container %s "
- + "without source or target datanodes.", containerID);
- ContainerReplicator replicator =
- replicateCommand.getTargetDatanode() == null ?
- downloadReplicator : pushReplicator;
+ Preconditions.checkArgument(replicateCommand.getTargetDatanode() != null,
+ "Replication command received for container %s without a target datanode.",
+ containerID);
- ReplicationTask task = new ReplicationTask(replicateCommand, replicator);
+ ReplicationTask task = new ReplicationTask(replicateCommand, pushReplicator);
supervisor.addTask(task);
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
index 2787093b1bf4..d0c0a60b03ee 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
@@ -18,12 +18,10 @@
package org.apache.hadoop.ozone.container.common.states.datanode;
import static org.apache.hadoop.hdds.utils.HddsServerUtil.getReconAddressForDatanodes;
-import static org.apache.hadoop.hdds.utils.HddsServerUtil.getSCMAddressForDatanodes;
import com.google.common.base.Strings;
import java.io.File;
import java.io.IOException;
-import java.net.InetSocketAddress;
import java.util.Collection;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@@ -33,6 +31,7 @@
import java.util.concurrent.TimeoutException;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.net.HostAndPort;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
@@ -76,9 +75,9 @@ public InitDatanodeState(ConfigurationSource conf,
*/
@Override
public DatanodeStateMachine.DatanodeStates call() throws Exception {
- Collection addresses = null;
+ final Collection addresses;
try {
- addresses = getSCMAddressForDatanodes(conf);
+ addresses = HddsServerUtil.getSCMAddressForDatanodes(conf);
} catch (IllegalArgumentException e) {
if (!Strings.isNullOrEmpty(e.getMessage())) {
LOG.error("Failed to get SCM addresses: {}", e.getMessage());
@@ -90,8 +89,8 @@ public DatanodeStateMachine.DatanodeStates call() throws Exception {
LOG.error("Null or empty SCM address list found.");
return DatanodeStateMachine.DatanodeStates.SHUTDOWN;
} else {
- for (InetSocketAddress addr : addresses) {
- if (addr.isUnresolved()) {
+ for (HostAndPort addr : addresses) {
+ if (addr.getAddress().isUnresolved()) {
LOG.warn("One SCM address ({}) can't (yet?) be resolved. Postpone "
+ "initialization.", addr);
@@ -100,11 +99,11 @@ public DatanodeStateMachine.DatanodeStates call() throws Exception {
return this.context.getState();
}
}
- for (InetSocketAddress addr : addresses) {
+ for (HostAndPort addr : addresses) {
connectionManager.addSCMServer(addr, context.getThreadNamePrefix());
this.context.addEndpoint(addr);
}
- InetSocketAddress reconAddress = getReconAddressForDatanodes(conf);
+ final HostAndPort reconAddress = getReconAddressForDatanodes(conf);
if (reconAddress != null) {
connectionManager.addReconServer(reconAddress,
context.getThreadNamePrefix());
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index 6ee9c20bfe97..3eedbecf3894 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -143,6 +143,17 @@ public XceiverServerGrpc(DatanodeDetails datanodeDetails,
.channelType(channelType)
.withOption(ChannelOption.SO_BACKLOG, dnConf.getGrpcSoBacklog())
.executor(readExecutors)
+ // If a client does not send an actual functional business RPC for 15 minutes,
+ // the server kicks them off with a GOAWAY frame.
+ .maxConnectionIdle(15, TimeUnit.MINUTES)
+ // If the server receives absolutely zero network traffic from a client for
+ // 5 minutes, the server proactively sends an HTTP/2 PING frame to verify
+ // if the network wire or client machine is still alive.
+ .keepAliveTime(5, TimeUnit.MINUTES)
+ // If the server fires a ping and the client fails to respond with a
+ // PING ACK within 30 seconds, the server assumes the socket is a dead
+ // "zombie connection" and immediately destroys the TCP socket.
+ .keepAliveTimeout(30, TimeUnit.SECONDS)
.addService(ServerInterceptors.intercept(
xceiverService.bindServiceWithZeroCopy(),
new GrpcServerInterceptor()));
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index c99b33f8c682..3de4110a01b3 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -790,29 +790,48 @@ private ExecutorService getChunkExecutor(WriteChunkRequestProto req) {
@Override
public CompletableFuture write(LogEntryProto entry, TransactionContext trx) {
try {
- metrics.incNumWriteStateMachineOps();
- long writeStateMachineStartTime = Time.monotonicNowNanos();
- final Context context = (Context) trx.getStateMachineContext();
- Objects.requireNonNull(context, "context == null");
- final ContainerCommandRequestProto requestProto = context.getRequestProto();
- final Type cmdType = requestProto.getCmdType();
-
- // For only writeChunk, there will be writeStateMachineData call.
- // CreateContainer will happen as a part of writeChunk only.
- switch (cmdType) {
- case WriteChunk:
- return writeStateMachineData(requestProto, entry.getIndex(),
- entry.getTerm(), writeStateMachineStartTime);
- default:
- throw new IllegalStateException("Cmd Type:" + cmdType
- + " should not have state machine data");
- }
- } catch (Exception e) {
- metrics.incNumWriteStateMachineFails();
+ return writeImpl(entry, trx).whenComplete((r, e) -> {
+ if (e != null) {
+ closeServer(e);
+ }
+ });
+ } catch (Throwable e) {
+ closeServer(e);
return completeExceptionally(e);
}
}
+ private CompletableFuture writeImpl(LogEntryProto entry, TransactionContext trx) {
+ metrics.incNumWriteStateMachineOps();
+ long writeStateMachineStartTime = Time.monotonicNowNanos();
+ final Context context = (Context) trx.getStateMachineContext();
+ Objects.requireNonNull(context, "context == null");
+ final ContainerCommandRequestProto requestProto = context.getRequestProto();
+ final Type cmdType = requestProto.getCmdType();
+
+ // For only writeChunk, there will be writeStateMachineData call.
+ // CreateContainer will happen as a part of writeChunk only.
+ switch (cmdType) {
+ case WriteChunk:
+ return writeStateMachineData(requestProto, entry.getIndex(),
+ entry.getTerm(), writeStateMachineStartTime);
+ default:
+ throw new IllegalStateException("Cmd Type:" + cmdType
+ + " should not have state machine data");
+ }
+ }
+
+ private void closeServer(Throwable e) {
+ metrics.incNumWriteStateMachineFails();
+ try {
+ LOG.error("{}: Failed to writeStateMachineData, close server", getId(), e);
+ getServer().get().getDivision(getGroupId()).close();
+ } catch (Throwable t) {
+ e.addSuppressed(t);
+ LOG.error("{}: Failed to close server", getId(), t);
+ }
+ }
+
@Override
public CompletableFuture query(Message request) {
try {
@@ -1228,7 +1247,7 @@ private void removeCacheDataUpTo(long index) {
stateMachineDataCache.removeIf(k -> k <= index);
}
- private static CompletableFuture completeExceptionally(Exception e) {
+ private static CompletableFuture completeExceptionally(Throwable e) {
final CompletableFuture future = new CompletableFuture<>();
future.completeExceptionally(e);
return future;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerLogger.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerLogger.java
index a4eb1765b867..03b2a8a1ac8c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerLogger.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerLogger.java
@@ -181,40 +181,38 @@ public static void logReconciled(ContainerData containerData, long oldDataChecks
/**
* Logged when a container is successfully moved from one data volume to another.
*
- * @param containerId The ID of the moved container.
+ * @param containerData The container after it has been moved to the destination volume.
* @param sourceVolume The source volume path.
* @param destinationVolume The destination volume path.
* @param containerSize The size of data moved from container in bytes.
* @param timeTaken The time taken for the move in milliseconds.
*/
- public static void logMoveSuccess(long containerId, StorageVolume sourceVolume,
+ public static void logMoveSuccess(ContainerData containerData, StorageVolume sourceVolume,
StorageVolume destinationVolume, long containerSize, long timeTaken) {
- LOG.info(getMessage(containerId, sourceVolume, destinationVolume, containerSize, timeTaken));
+ LOG.info(getMessage(containerData,
+ "SrcVolume=" + sourceVolume,
+ "DestVolume=" + destinationVolume,
+ "Size=" + containerSize + " bytes",
+ "TimeTaken=" + timeTaken + " ms",
+ "Container is moved from SrcVolume to DestVolume"));
}
- private static String getMessage(ContainerData containerData,
- String message) {
+ private static String getMessage(ContainerData containerData, String message) {
return String.join(FIELD_SEPARATOR, getMessage(containerData), message);
}
- private static String getMessage(ContainerData containerData) {
+ private static String getMessage(ContainerData containerData, String... fields) {
return String.join(FIELD_SEPARATOR,
"ID=" + containerData.getContainerID(),
"Index=" + containerData.getReplicaIndex(),
"BCSID=" + containerData.getBlockCommitSequenceId(),
"State=" + containerData.getState(),
- "Volume=" + containerData.getVolume(),
- "DataChecksum=" + checksumToString(containerData.getDataChecksum()));
+ String.join(FIELD_SEPARATOR, fields));
}
- private static String getMessage(long containerId, StorageVolume sourceVolume,
- StorageVolume destinationVolume, long containerSize, long timeTaken) {
- return String.join(FIELD_SEPARATOR,
- "ID=" + containerId,
- "SrcVolume=" + sourceVolume,
- "DestVolume=" + destinationVolume,
- "Size=" + containerSize + " bytes",
- "TimeTaken=" + timeTaken + " ms",
- "Container is moved from SrcVolume to DestVolume");
+ private static String getMessage(ContainerData containerData) {
+ return getMessage(containerData,
+ "Volume=" + containerData.getVolume(),
+ "DataChecksum=" + checksumToString(containerData.getDataChecksum()));
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java
index 73e69eddc15b..a88c7a6f8990 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java
@@ -30,9 +30,11 @@
import java.io.SyncFailedException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
import java.util.Arrays;
import java.util.Random;
import java.util.UUID;
+import org.apache.commons.io.IOUtils;
import org.apache.ratis.util.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,6 +44,7 @@
* where the disk is mounted.
*/
public final class DiskCheckUtil {
+ public static final String LINUX_DISK_FULL_MESSAGE = "No space left on device";
// For testing purposes, an alternate check implementation can be provided
// to inject failures.
private static DiskChecks impl = new DiskChecksImpl();
@@ -140,41 +143,53 @@ public boolean checkPermissions(File storageDir) {
public boolean checkReadWrite(File storageDir,
File testFileDir, int numBytesToWrite) {
File testFile = new File(testFileDir, "disk-check-" + UUID.randomUUID());
+ Path testPath = testFile.toPath();
byte[] writtenBytes = new byte[numBytesToWrite];
RANDOM.nextBytes(writtenBytes);
- try (OutputStream fos = FileUtils.newOutputStreamForceAtClose(testFile, CREATE, TRUNCATE_EXISTING, WRITE)) {
+ try (OutputStream fos = FileUtils.newOutputStreamForceAtClose(testPath, CREATE, TRUNCATE_EXISTING, WRITE)) {
fos.write(writtenBytes);
} catch (FileNotFoundException | NoSuchFileException notFoundEx) {
logError(storageDir, String.format("Could not find file %s for " +
"volume check.", testFile.getAbsolutePath()), notFoundEx);
return false;
} catch (SyncFailedException syncEx) {
- logError(storageDir, String.format("Could sync file %s to disk.",
+ logError(storageDir, String.format("Could not sync file %s to disk.",
testFile.getAbsolutePath()), syncEx);
+ FileUtils.deletePathQuietly(testPath);
return false;
} catch (IOException ioEx) {
+ String msg = ioEx.getMessage();
+ if (msg != null && msg.contains(LINUX_DISK_FULL_MESSAGE)) {
+ LOG.warn("Could not write file {} for volume check", testFile.getAbsolutePath(), ioEx);
+ FileUtils.deletePathQuietly(testPath);
+ return true;
+ }
logError(storageDir, String.format("Could not write file %s " +
"for volume check.", testFile.getAbsolutePath()), ioEx);
+ FileUtils.deletePathQuietly(testPath);
return false;
}
// Read data back from the test file.
byte[] readBytes = new byte[numBytesToWrite];
- try (InputStream fis = Files.newInputStream(testFile.toPath())) {
- int numBytesRead = fis.read(readBytes);
+ try (InputStream fis = Files.newInputStream(testPath)) {
+ int numBytesRead = IOUtils.read(fis, readBytes);
if (numBytesRead != numBytesToWrite) {
logError(storageDir, String.format("%d bytes written to file %s " +
"but %d bytes were read back.", numBytesToWrite,
testFile.getAbsolutePath(), numBytesRead));
+ FileUtils.deletePathQuietly(testPath);
return false;
}
} catch (FileNotFoundException | NoSuchFileException notFoundEx) {
logError(storageDir, String.format("Could not find file %s " +
"for volume check.", testFile.getAbsolutePath()), notFoundEx);
+ FileUtils.deletePathQuietly(testPath);
return false;
} catch (IOException ioEx) {
logError(storageDir, String.format("Could not read file %s " +
"for volume check.", testFile.getAbsolutePath()), ioEx);
+ FileUtils.deletePathQuietly(testPath);
return false;
}
@@ -183,6 +198,7 @@ public boolean checkReadWrite(File storageDir,
logError(storageDir, String.format("%d Bytes read from file " +
"%s do not match the %d bytes that were written.",
writtenBytes.length, testFile.getAbsolutePath(), readBytes.length));
+ FileUtils.deletePathQuietly(testPath);
return false;
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/StorageVolumeUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/StorageVolumeUtil.java
index c71fc6cde6d3..eb6747a6bfd0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/StorageVolumeUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/StorageVolumeUtil.java
@@ -28,6 +28,7 @@
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.InconsistentStorageStateException;
+import org.apache.hadoop.ozone.common.Storage;
import org.apache.hadoop.ozone.container.common.HDDSVolumeLayoutVersion;
import org.apache.hadoop.ozone.container.common.volume.DbVolume;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
@@ -274,4 +275,31 @@ public static boolean checkVolume(StorageVolume volume, String scmId,
return success;
}
+
+ public static File resolveContainerCurrentDir(
+ File hddsRoot, String clusterId, File[] storageDirs)
+ throws InconsistentStorageStateException {
+
+ File clusterIdDir = new File(hddsRoot, clusterId);
+ //The subdirectory we should verify containers within.
+ // If this volume was formatted pre SCM HA, this will be the SCM ID.
+ // A cluster ID symlink will exist in this case only if this cluster is
+ // finalized for SCM HA.
+ // If the volume was formatted post SCM HA, this will be the cluster ID.
+ File idDir = clusterIdDir;
+
+ if (storageDirs.length == 1 && !clusterIdDir.exists()) {
+ // If the one directory is not the cluster ID directory, assume it is
+ // the old SCM ID directory used before SCM HA.
+ idDir = storageDirs[0];
+ } else if (!clusterIdDir.exists()) {
+ // There are 1 or more storage directories. We only care about the
+ // cluster ID directory.
+ throw new InconsistentStorageStateException(
+ "Volume " + hddsRoot + " is in an inconsistent state. Expected cluster ID directory "
+ + clusterIdDir + " not found.");
+ }
+
+ return new File(idDir, Storage.STORAGE_DIR_CURRENT);
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/CapacityVolumeChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/CapacityVolumeChoosingPolicy.java
index 08b64327b00f..325b98260bc9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/CapacityVolumeChoosingPolicy.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/CapacityVolumeChoosingPolicy.java
@@ -26,6 +26,7 @@
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
+import org.apache.hadoop.hdds.fs.SpaceUsageSource;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
import org.slf4j.Logger;
@@ -84,9 +85,9 @@ public HddsVolume chooseVolume(List volumes,
HddsVolume selectedVolume = volumesWithEnoughSpace.get(0);
if (count > 1) {
// Even if we don't have too many volumes in volumesWithEnoughSpace, this
- // algorithm will still help us choose the volume with larger
- // available space than other volumes.
- // Say we have vol1 with more available space than vol2, for two choices,
+ // algorithm will still help us choose the volume with lower
+ // utilization than other volumes.
+ // Say we have vol1 with lower utilization than vol2, for two choices,
// the distribution of possibility is as follows:
// 1. vol1 + vol2: 25%, result is vol1
// 2. vol1 + vol1: 25%, result is vol1
@@ -100,11 +101,9 @@ public HddsVolume chooseVolume(List volumes,
HddsVolume firstVolume = volumesWithEnoughSpace.get(firstIndex);
HddsVolume secondVolume = volumesWithEnoughSpace.get(secondIndex);
- long firstAvailable = firstVolume.getCurrentUsage().getAvailable()
- - firstVolume.getCommittedBytes();
- long secondAvailable = secondVolume.getCurrentUsage().getAvailable()
- - secondVolume.getCommittedBytes();
- selectedVolume = firstAvailable < secondAvailable ? secondVolume : firstVolume;
+ double firstRatio = freeSpaceRatio(firstVolume);
+ double secondRatio = freeSpaceRatio(secondVolume);
+ selectedVolume = firstRatio < secondRatio ? secondVolume : firstVolume;
}
selectedVolume.incCommittedBytes(maxContainerSize);
return selectedVolume;
@@ -112,4 +111,20 @@ public HddsVolume chooseVolume(List volumes,
lock.unlock();
}
}
+
+ // Fraction of capacity still free for hdds, excluding space committed to open containers.
+ // Comparing the ratio (not absolute bytes) keeps utilization balanced across volumes of
+ // different capacity.
+ @VisibleForTesting
+ static double freeSpaceRatio(HddsVolume volume) {
+ SpaceUsageSource usage = volume.getCurrentUsage();
+ long capacity = usage.getCapacity();
+ if (capacity <= 0) {
+ return 0;
+ }
+ // Clamp at 0: committed can exceed available, and a negative ratio would skew the
+ // comparison (matches the guard in HddsVolume.checkVolumeUsages).
+ long free = Math.max(0, usage.getAvailable() - volume.getCommittedBytes());
+ return (double) free / capacity;
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
index f1deedc8d330..8827960248de 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
@@ -25,6 +25,7 @@
import jakarta.annotation.Nullable;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Files;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ConcurrentSkipListSet;
@@ -48,6 +49,7 @@
import org.apache.hadoop.ozone.container.common.utils.RawDB;
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
+import org.apache.hadoop.ozone.container.ozoneimpl.ScanTransientIOUtil;
import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures.SchemaV3;
import org.apache.hadoop.util.Time;
@@ -304,24 +306,82 @@ public synchronized VolumeCheckResult check(@Nullable Boolean unused)
return checkDbHealth(dbFile);
}
+ /**
+ * Verifies the per-volume RocksDB's global state files (CURRENT, MANIFEST,
+ * OPTIONS) by opening the DB in secondary mode. A successful open implies
+ * those files are readable and internally consistent and that the
+ * referenced SST file names match what RocksDB expects.
+ *
+ * This check intentionally does not read or checksum SST file
+ * contents or any individual key/value. Per-block / per-key integrity is
+ * verified by the container data scanner, which scans containers (and
+ * their RocksDB rows) on its own schedule.
+ *
+ *
The volume is only marked {@link VolumeCheckResult#FAILED} once the
+ * configured threshold of failures is exceeded, matching the parent class's
+ * intermittent-error tolerance. Open failures whose underlying RocksDB
+ * status is {@code IOError(NoSpace)} are not counted: {@code openAsSecondary}
+ * writes its info LOG into the disk-check directory, so an out-of-space
+ * failure there is unrelated to DB integrity. Any other status — permission
+ * denied, missing path, corruption, generic IO error — is still counted as
+ * a real failure.
+ */
@VisibleForTesting
public VolumeCheckResult checkDbHealth(File dbFile) throws InterruptedException {
if (!(getDiskCheckEnabled() && getDatanodeConfig().isRocksDbDiskCheckEnabled())) {
return VolumeCheckResult.HEALTHY;
}
+ File secondaryDir = new File(getDiskCheckDir(), "rocksdb-secondary-" + Time.now());
+ try {
+ Files.createDirectories(secondaryDir.toPath());
+ } catch (IOException e) {
+ LOG.error("Failed to create secondary instance dir {} for volume {}", secondaryDir, getStorageDir(), e);
+
+ if (!isNoSpaceAvailable(e) && !ScanTransientIOUtil.isTooManyOpenFiles(e)) {
+ getIoTestSlidingWindow().add();
+ }
+
+ return getIoTestSlidingWindow().isExceeded()
+ ? VolumeCheckResult.FAILED
+ : VolumeCheckResult.HEALTHY;
+ }
+
try (ManagedOptions managedOptions = new ManagedOptions();
- ManagedRocksDB ignored = ManagedRocksDB.openReadOnly(managedOptions, dbFile.toString())) {
+ ManagedRocksDB ignored =
+ ManagedRocksDB.openAsSecondary(managedOptions, dbFile.toString(), secondaryDir.getPath())) {
// Do nothing. Only check if rocksdb is accessible.
LOG.debug("Successfully opened the database at \"{}\" for HDDS volume {}.", dbFile, getStorageDir());
} catch (Exception e) {
if (Thread.currentThread().isInterrupted()) {
throw new InterruptedException("Check of database for volume " + this + " interrupted.");
}
- LOG.warn("Could not open Volume DB located at {}", dbFile, e);
- getIoTestSlidingWindow().add();
+
+ // openAsSecondary writes its info LOG into secondaryDir. If that write
+ // fails because the disk is full, RocksDB surfaces the failure as
+ // IOError(NoSpace) (mapped from ENOSPC). That is unrelated to DB
+ // integrity, so don't count it against the sliding window. Any other
+ // status (permission denied, missing path, corruption, generic IO
+ // error) is still treated as a real failure.
+ if (ManagedRocksDB.isNoSpaceFailure(e)) {
+ LOG.warn("Skipping RocksDB health-check failure accounting for volume {}: " +
+ "secondary open returned IOError(NoSpace) for {}.", this, secondaryDir, e);
+ } else if (ScanTransientIOUtil.isTooManyOpenFiles(e)) {
+ LOG.warn("Skipping RocksDB health-check failure accounting for volume {}: " +
+ "secondary open hit file descriptor exhaustion for {}.", this, secondaryDir, e);
+ } else {
+ LOG.error("Could not open Volume DB located at {}", dbFile, e);
+ getIoTestSlidingWindow().add();
+ }
+ } finally {
+ try {
+ FileUtils.deleteDirectory(secondaryDir);
+ } catch (IOException e) {
+ LOG.warn("Failed to delete RocksDB secondary instance dir {}", secondaryDir, e);
+ }
}
+
if (getIoTestSlidingWindow().isExceeded()) {
LOG.error("Failed to open the database at \"{}\" for HDDS volume {}: " +
"encountered more than the {} tolerated failures.",
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
index d9424b76a139..389ef2558a34 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
@@ -534,7 +534,6 @@ public File getTmpDir() {
return this.tmpDir;
}
- @VisibleForTesting
public File getDiskCheckDir() {
return this.diskCheckDir;
}
@@ -851,4 +850,14 @@ private void setStorageDirPermissions() {
ScmConfigKeys.HDDS_DATANODE_DATA_DIR_PERMISSIONS);
}
}
+
+ public static boolean isNoSpaceAvailable(Throwable t) {
+ for (Throwable cause = t; cause != null; cause = cause.getCause()) {
+ String msg = cause.getMessage();
+ if (msg != null && msg.contains("No space left on device")) {
+ return true;
+ }
+ }
+ return false;
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
index 0cb0c9d56a98..27996eb44b7e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
@@ -51,14 +51,25 @@ public class VolumeInfoMetrics implements MetricsSource {
Interns.info("OzoneUsed", "Ozone used space");
private static final MetricsInfo RESERVED =
Interns.info("Reserved", "Reserved Space");
- private static final MetricsInfo TOTAL_CAPACITY =
- Interns.info("TotalCapacity", "Ozone capacity + reserved space");
private static final MetricsInfo FS_CAPACITY =
Interns.info("FilesystemCapacity", "Filesystem capacity as reported by the local filesystem");
private static final MetricsInfo FS_AVAILABLE =
Interns.info("FilesystemAvailable", "Filesystem available space as reported by the local filesystem");
private static final MetricsInfo FS_USED =
Interns.info("FilesystemUsed", "Filesystem used space (FilesystemCapacity - FilesystemAvailable)");
+ private static final MetricsInfo MIN_FREE_SPACE =
+ Interns.info("MinFreeSpace",
+ "Minimum free space threshold (soft limit) reported to SCM, " +
+ "derived from hdds.datanode.volume.min.free.space.percent / hdds.datanode.volume.min.free.space");
+ private static final MetricsInfo HARD_MIN_FREE_SPACE =
+ Interns.info("HardMinFreeSpace",
+ "Minimum free space threshold (hard limit) enforced locally for writes, " +
+ "derived from hdds.datanode.volume.min.free.space.hard.limit.percent " +
+ "/ hdds.datanode.volume.min.free.space");
+ private static final MetricsInfo NON_OZONE_USED =
+ Interns.info("NonOzoneUsed",
+ "Space on the filesystem consumed by non-Ozone workloads " +
+ "(FilesystemUsed - OzoneUsed)");
private final MetricsRegistry registry;
private final String metricsSourceName;
@@ -238,15 +249,18 @@ public void getMetrics(MetricsCollector collector, boolean all) {
SpaceUsageSource.Fixed fsUsage = volumeUsage.realUsage();
SpaceUsageSource usage = volumeUsage.getCurrentUsage(fsUsage);
long reserved = volumeUsage.getReservedInBytes();
+ long ozoneCapacity = usage.getCapacity();
builder
- .addGauge(CAPACITY, usage.getCapacity())
+ .addGauge(CAPACITY, ozoneCapacity)
.addGauge(AVAILABLE, usage.getAvailable())
.addGauge(USED, usage.getUsedSpace())
.addGauge(RESERVED, reserved)
- .addGauge(TOTAL_CAPACITY, usage.getCapacity() + reserved)
.addGauge(FS_CAPACITY, fsUsage.getCapacity())
.addGauge(FS_AVAILABLE, fsUsage.getAvailable())
- .addGauge(FS_USED, fsUsage.getUsedSpace());
+ .addGauge(FS_USED, fsUsage.getCapacity() - fsUsage.getAvailable())
+ .addGauge(MIN_FREE_SPACE, volume.getReportedFreeSpaceToSpare(ozoneCapacity))
+ .addGauge(HARD_MIN_FREE_SPACE, volume.getFreeSpaceToSpare(ozoneCapacity))
+ .addGauge(NON_OZONE_USED, VolumeUsage.getOtherUsed(fsUsage));
}
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
index a962c0e80ff3..fdf90afb61b3 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
@@ -31,16 +31,18 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
+import java.time.Clock;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
@@ -59,6 +61,7 @@
import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
import org.apache.hadoop.hdds.utils.FaultInjector;
+import org.apache.hadoop.hdds.utils.SlidingWindow;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
@@ -94,6 +97,7 @@ public class DiskBalancerService extends BackgroundService {
private OzoneContainer ozoneContainer;
private final ConfigurationSource conf;
+ private final Clock clock;
private double threshold;
private long bandwidthInMB;
@@ -110,7 +114,8 @@ public class DiskBalancerService extends BackgroundService {
private AtomicLong nextAvailableTime = new AtomicLong(Time.monotonicNow());
private Set inProgressContainers;
- private ConcurrentSkipListMap pendingDeletionContainers = new ConcurrentSkipListMap();
+ private final ConcurrentSkipListMap> pendingDeletionContainers =
+ new ConcurrentSkipListMap<>();
private static FaultInjector injector;
/**
@@ -135,10 +140,19 @@ public class DiskBalancerService extends BackgroundService {
public DiskBalancerService(OzoneContainer ozoneContainer,
long serviceCheckInterval, long serviceCheckTimeout, TimeUnit timeUnit,
int workerSize, ConfigurationSource conf) throws IOException {
+ this(ozoneContainer, serviceCheckInterval, serviceCheckTimeout, timeUnit,
+ workerSize, conf, new SlidingWindow.MonotonicClock());
+ }
+
+ DiskBalancerService(OzoneContainer ozoneContainer,
+ long serviceCheckInterval, long serviceCheckTimeout, TimeUnit timeUnit,
+ int workerSize, ConfigurationSource conf, Clock clock)
+ throws IOException {
super("DiskBalancerService", serviceCheckInterval, timeUnit, workerSize,
serviceCheckTimeout);
this.ozoneContainer = ozoneContainer;
this.conf = conf;
+ this.clock = Objects.requireNonNull(clock, "clock");
String diskBalancerInfoPath = getDiskBalancerInfoPath();
Objects.requireNonNull(diskBalancerInfoPath);
@@ -255,13 +269,7 @@ private void applyDiskBalancerInfo(DiskBalancerInfo diskBalancerInfo)
setStopAfterDiskEven(validated.isStopAfterDiskEven());
setVersion(diskBalancerInfo.getVersion());
setContainerStates(validated.getMovableContainerStates());
-
- // Default executorService is ScheduledThreadPoolExecutor, so we can
- // update the poll size by setting corePoolSize.
- if ((getExecutorService() instanceof ScheduledThreadPoolExecutor)) {
- ((ScheduledThreadPoolExecutor) getExecutorService())
- .setCorePoolSize(parallelThread);
- }
+ setPoolSize(parallelThread);
}
/**
@@ -443,8 +451,7 @@ public BackgroundTaskQueue getTasks() {
destVolume);
queue.add(task);
inProgressContainers.add(ContainerID.valueOf(toBalanceContainer.getContainerID()));
- deltaSizes.put(sourceVolume, deltaSizes.getOrDefault(sourceVolume, 0L)
- - toBalanceContainer.getBytesUsed());
+ deltaSizes.merge(sourceVolume, -toBalanceContainer.getBytesUsed(), Long::sum);
}
}
}
@@ -508,7 +515,8 @@ protected class DiskBalancerTask implements BackgroundTask {
@Override
public BackgroundTaskResult call() {
long startTime = Time.monotonicNow();
- boolean moveSucceeded = true;
+ boolean moveSucceeded = false;
+ Container newContainer = null;
long containerId = containerData.getContainerID();
Container container = ozoneContainer.getContainerSet().getContainer(containerId);
boolean readLockReleased = false;
@@ -529,7 +537,6 @@ public BackgroundTaskResult call() {
State containerState = container.getContainerData().getState();
if (!movableContainerStates.contains(containerState)) {
LOG.warn("Container {} is in {} state, skipping move process.", containerId, containerState);
- moveSucceeded = false;
return BackgroundTaskResult.EmptyTaskResult.newResult();
}
@@ -580,7 +587,7 @@ public BackgroundTaskResult call() {
}
// Import the container. importContainer will reset container back to original state
- Container newContainer = ozoneContainer.getController().importContainer(tempContainerData);
+ newContainer = ozoneContainer.getController().importContainer(tempContainerData);
// Step 4: Update container for containerID and mark old container for deletion
// first, update the in-memory set to point to the new replica.
@@ -594,6 +601,7 @@ public BackgroundTaskResult call() {
pauseInjector();
// Mark old container as DELETED and persist state.
// markContainerForDelete require writeLock, so release readLock first
+ moveSucceeded = true;
container.readUnlock();
readLockReleased = true;
try {
@@ -608,9 +616,8 @@ public BackgroundTaskResult call() {
balancedBytesInLastWindow.addAndGet(containerSize);
metrics.incrSuccessBytes(containerSize);
totalBalancedBytes.addAndGet(containerSize);
- } catch (IOException e) {
+ } catch (Throwable e) {
pauseInjector();
- moveSucceeded = false;
LOG.warn("Failed to move container {}", containerId, e);
if (diskBalancerTmpDir != null) {
try {
@@ -633,15 +640,18 @@ public BackgroundTaskResult call() {
if (!readLockReleased) {
container.readUnlock();
}
- if (moveSucceeded) {
+ if (moveSucceeded && newContainer != null) {
// Add current old container to pendingDeletionContainers.
- pendingDeletionContainers.put(System.currentTimeMillis() + replicaDeletionDelay, container);
- ContainerLogger.logMoveSuccess(containerId, sourceVolume,
+ long deadline = clock.millis() + replicaDeletionDelay;
+ pendingDeletionContainers
+ .computeIfAbsent(deadline, ignored -> new ConcurrentLinkedQueue<>())
+ .add(container);
+ ContainerLogger.logMoveSuccess(newContainer.getContainerData(), sourceVolume,
destVolume, containerSize, Time.monotonicNow() - startTime);
}
- postCall(moveSucceeded, startTime);
+ postCall(moveSucceeded && newContainer != null, startTime);
- // pick one expired container from pendingDeletionContainers to delete
+ // Attempt to delete any pending-deletion buckets whose deadline has elapsed.
tryCleanupOnePendingDeletionContainer();
}
return BackgroundTaskResult.EmptyTaskResult.newResult();
@@ -654,8 +664,7 @@ public int getPriority() {
private void postCall(boolean success, long startTime) {
inProgressContainers.remove(ContainerID.valueOf(containerData.getContainerID()));
- deltaSizes.put(sourceVolume, deltaSizes.get(sourceVolume) +
- containerData.getBytesUsed());
+ deltaSizes.merge(sourceVolume, containerData.getBytesUsed(), Long::sum);
destVolume.incCommittedBytes(0 - containerData.getBytesUsed());
long endTime = Time.monotonicNow();
if (success) {
@@ -682,7 +691,8 @@ private void deleteContainer(Container container) {
}
}
- private void cleanupPendingDeletionContainers() {
+ @VisibleForTesting
+ public void cleanupPendingDeletionContainers() {
// delete all pending deletion containers before stop the service
boolean ret;
do {
@@ -691,18 +701,18 @@ private void cleanupPendingDeletionContainers() {
}
private boolean tryCleanupOnePendingDeletionContainer() {
- Map.Entry entry = pendingDeletionContainers.pollFirstEntry();
- if (entry != null) {
- if (entry.getKey() <= System.currentTimeMillis()) {
- // entry container is expired
- deleteContainer(entry.getValue());
- return true;
- } else {
- // put back the container
- pendingDeletionContainers.put(entry.getKey(), entry.getValue());
- }
+ // peek first, only remove when expired
+ Map.Entry> entry = pendingDeletionContainers.firstEntry();
+ if (entry == null || entry.getKey() > clock.millis()) {
+ return false;
}
- return false;
+ if (!pendingDeletionContainers.remove(entry.getKey(), entry.getValue())) {
+ return false;
+ }
+ for (Container pending : entry.getValue()) {
+ deleteContainer(pending);
+ }
+ return true;
}
public DiskBalancerInfo getDiskBalancerInfo() {
@@ -746,6 +756,7 @@ public static List buildVolumeReportProto(List 0 ? computeUtilization(usage, volume.getCommittedBytes(), delta) : null;
+ this.utilization = usage.getCapacity() > 0
+ ? computeUtilization(usage, volume.getCommittedBytes(), delta)
+ : 0.0;
}
public HddsVolume getVolume() {
@@ -189,7 +190,7 @@ public long getEffectiveUsed() {
}
public double getUtilization() {
- return Objects.requireNonNull(utilization, "utilization == null");
+ return utilization;
}
public long computeUsableSpace() {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java
index 1b8ecef32f27..2c539173d4f4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java
@@ -76,10 +76,9 @@ public static DiskBalancerInfo readDiskBalancerInfoFile(File path)
throw new IOException("Unable to parse yaml file.", e);
}
- // getContainerStates() may be null if the key is absent; isNotBlank(null) is false.
- String cs = diskBalancerInfoYaml.getContainerStates();
- String containerStates = StringUtils.isNotBlank(cs)
- ? cs.trim() : DiskBalancerConfiguration.DEFAULT_CONTAINER_STATES;
+ validateRequiredFields(diskBalancerInfoYaml);
+ DiskBalancerVersion version = getValidatedVersion(diskBalancerInfoYaml);
+ String containerStates = getValidatedContainerStates(diskBalancerInfoYaml);
diskBalancerInfo = new DiskBalancerInfo(
diskBalancerInfoYaml.operationalState,
diskBalancerInfoYaml.getThreshold(),
@@ -87,13 +86,53 @@ public static DiskBalancerInfo readDiskBalancerInfoFile(File path)
diskBalancerInfoYaml.getParallelThread(),
diskBalancerInfoYaml.isStopAfterDiskEven(),
containerStates,
- DiskBalancerVersion.getDiskBalancerVersion(
- diskBalancerInfoYaml.version));
+ version);
+ validatePersistedConfiguration(diskBalancerInfo);
}
return diskBalancerInfo;
}
+ private static void validateRequiredFields(
+ DiskBalancerInfoYaml diskBalancerInfoYaml) throws IOException {
+ if (diskBalancerInfoYaml.getOperationalState() == null) {
+ throw new IOException("DiskBalancer operationalState is missing from persisted info.");
+ }
+ if (diskBalancerInfoYaml.getVersion() == null) {
+ throw new IOException("DiskBalancer info version is missing from persisted info.");
+ }
+ }
+
+ private static DiskBalancerVersion getValidatedVersion(
+ DiskBalancerInfoYaml diskBalancerInfoYaml) throws IOException {
+ int rawVersion = diskBalancerInfoYaml.getVersion();
+ DiskBalancerVersion version =
+ DiskBalancerVersion.getDiskBalancerVersion(rawVersion);
+ if (version == null) {
+ throw new IOException("Unsupported DiskBalancer info version: " + rawVersion);
+ }
+ return version;
+ }
+
+ private static String getValidatedContainerStates(
+ DiskBalancerInfoYaml diskBalancerInfoYaml) {
+ // getContainerStates() may be null if the key is absent; isNotBlank(null) is false.
+ String containerStates = diskBalancerInfoYaml.getContainerStates();
+ return StringUtils.isNotBlank(containerStates)
+ ? containerStates.trim() : DiskBalancerConfiguration.DEFAULT_CONTAINER_STATES;
+ }
+
+ private static void validatePersistedConfiguration(
+ DiskBalancerInfo diskBalancerInfo) throws IOException {
+ try {
+ diskBalancerInfo.toConfiguration();
+ } catch (IllegalArgumentException ex) {
+ throw new IOException(
+ "Invalid DiskBalancer configuration in persisted info: "
+ + ex.getMessage(), ex);
+ }
+ }
+
/**
* Datanode DiskBalancer Info to be written to the yaml file.
*/
@@ -105,7 +144,7 @@ public static class DiskBalancerInfoYaml {
private boolean stopAfterDiskEven;
private String containerStates;
- private int version;
+ private Integer version;
public DiskBalancerInfoYaml() {
// Needed for snake-yaml introspection.
@@ -163,11 +202,11 @@ public void setStopAfterDiskEven(boolean stopAfterDiskEven) {
this.stopAfterDiskEven = stopAfterDiskEven;
}
- public void setVersion(int version) {
+ public void setVersion(Integer version) {
this.version = version;
}
- public int getVersion() {
+ public Integer getVersion() {
return this.version;
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index 5be6d20a336a..06a73b333739 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@ -391,7 +391,17 @@ public void markContainerUnhealthy() throws StorageContainerException {
writeLock();
final State prevState = containerData.getState();
try {
- updateContainerState(UNHEALTHY);
+ if (!getContainerFile().getParentFile().exists()) {
+ // Metadata directory is absent (e.g. MISSING_METADATA_DIR detected by scanner).
+ // Attempting to write the .container file would fail
+ // The in-memory UNHEALTHY state is sufficient: SCM will receive it via ICR
+ // and schedule deletion without requiring a persisted .container file.
+ containerData.setState(UNHEALTHY);
+ LOG.debug("Skipping .container file update for container {} with missing metadata directory",
+ containerData.getContainerID());
+ } else {
+ updateContainerState(UNHEALTHY);
+ }
clearPendingPutBlockCache();
} finally {
writeUnlock();
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 170b31590fbc..c01a5d80fe9a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -2112,6 +2112,11 @@ public void deleteUnreferenced(Container container, long localID)
// Since the putBlock request may fail, we don't know if the chunk exists,
// thus we need to check it when receiving the request to delete such blocks
String[] chunkNames = getFilesWithPrefix(prefix, chunkDir);
+ if (chunkNames == null) {
+ throw new IOException("Failed to list chunks under " + chunkDir
+ + " for unreferenced block " + localID + " in container "
+ + containerID);
+ }
if (chunkNames.length == 0) {
LOG.warn("Missing delete block(Container = {}, Block = {}",
containerID, localID);
@@ -2122,12 +2127,20 @@ public void deleteUnreferenced(Container container, long localID)
if (!file.isFile()) {
continue;
}
- FileUtil.fullyDelete(file);
+ if (!deleteUnreferencedFile(file)) {
+ throw new IOException("Failed to delete unreferenced chunk/block "
+ + file + " in container " + containerID);
+ }
LOG.info("Deleted unreferenced chunk/block {} in container {}", name,
containerID);
}
}
+ @VisibleForTesting
+ boolean deleteUnreferencedFile(File file) {
+ return FileUtil.fullyDelete(file);
+ }
+
@Override
public ContainerCommandResponseProto readBlock(
ContainerCommandRequestProto request, Container kvContainer,
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/MappedBufferManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/MappedBufferManager.java
index 8186bdb029f2..7d2f822e5a5e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/MappedBufferManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/MappedBufferManager.java
@@ -20,6 +20,7 @@
import com.google.common.util.concurrent.Striped;
import java.lang.ref.WeakReference;
import java.nio.ByteBuffer;
+import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
@@ -57,10 +58,14 @@ public boolean getQuota(int permits) {
CompletableFuture.runAsync(() -> {
int p = 0;
try {
- for (String key : mappedBuffers.keySet()) {
- ByteBuffer buf = mappedBuffers.get(key).get();
- if (buf == null) {
- mappedBuffers.remove(key);
+ // remove(key, value) only counts entries we observed cleared,
+ // so a concurrent put() that replaced the WeakReference is not
+ // miscounted as freed.
+ for (Map.Entry> entry
+ : mappedBuffers.entrySet()) {
+ final WeakReference ref = entry.getValue();
+ if (ref.get() == null
+ && mappedBuffers.remove(entry.getKey(), ref)) {
p++;
}
}
@@ -93,14 +98,16 @@ public ByteBuffer computeIfAbsent(String file, long position, long size,
Lock fileLock = lock.get(key);
fileLock.lock();
try {
- WeakReference refer = mappedBuffers.get(key);
- if (refer != null && refer.get() != null) {
- // reuse the mapped buffer
+ // Hold a strong reference for the rest of this method so GC cannot
+ // clear the WeakReference between the null check and the return.
+ final WeakReference refer = mappedBuffers.get(key);
+ final ByteBuffer cached = refer != null ? refer.get() : null;
+ if (cached != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("find buffer for key {}", key);
}
releaseQuota(1);
- return refer.get();
+ return cached;
}
ByteBuffer buffer = supplier.get();
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java
index 5535c5128ccc..9c535e5f6e94 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java
@@ -18,13 +18,13 @@
package org.apache.hadoop.ozone.container.keyvalue.statemachine.background;
import java.util.Iterator;
-import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hdds.utils.BackgroundService;
import org.apache.hadoop.hdds.utils.BackgroundTask;
import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet.RecoveringContainer;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,13 +55,13 @@ public BackgroundTaskQueue getTasks() {
BackgroundTaskQueue backgroundTaskQueue =
new BackgroundTaskQueue();
long currentTime = containerSet.getCurrentTime();
- Iterator> it =
+ Iterator it =
containerSet.getRecoveringContainerIterator();
while (it.hasNext()) {
- Map.Entry entry = it.next();
- if (currentTime >= entry.getKey()) {
+ RecoveringContainer entry = it.next();
+ if (currentTime >= entry.getTimeout()) {
backgroundTaskQueue.add(new RecoveringContainerScrubbingTask(
- containerSet, entry.getValue()));
+ containerSet, entry.getContainerId()));
it.remove();
} else {
break;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
index 43aa05c850c5..88a9cc50a40b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
@@ -28,11 +28,12 @@
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.common.Storage;
+import org.apache.hadoop.ozone.common.InconsistentStorageStateException;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
@@ -119,32 +120,19 @@ public void readVolume(File hddsVolumeRootDir) {
// by HddsUtil#checkVolume once we have a cluster ID from SCM. No
// operations to perform here in that case.
if (storageDirs.length > 0) {
- File clusterIDDir = new File(hddsVolumeRootDir,
- hddsVolume.getClusterID());
- // The subdirectory we should verify containers within.
- // If this volume was formatted pre SCM HA, this will be the SCM ID.
- // A cluster ID symlink will exist in this case only if this cluster is
- // finalized for SCM HA.
- // If the volume was formatted post SCM HA, this will be the cluster ID.
- File idDir = clusterIDDir;
- if (storageDirs.length == 1 && !clusterIDDir.exists()) {
- // If the one directory is not the cluster ID directory, assume it is
- // the old SCM ID directory used before SCM HA.
- idDir = storageDirs[0];
- } else {
- // There are 1 or more storage directories. We only care about the
- // cluster ID directory.
- if (!clusterIDDir.exists()) {
- LOG.error("Volume {} is in an inconsistent state. Expected " +
- "clusterID directory {} not found.", hddsVolumeRootDir,
- clusterIDDir);
- volumeSet.failVolume(hddsVolumeRootDir.getPath());
- return;
- }
+ File currentDir;
+ try {
+ currentDir = StorageVolumeUtil.resolveContainerCurrentDir(hddsVolumeRootDir,
+ hddsVolume.getClusterID(), storageDirs);
+ } catch (InconsistentStorageStateException e) {
+ LOG.error("Volume {} is in an inconsistent state. Expected " +
+ "clusterID directory {} not found.", hddsVolumeRootDir,
+ new File(hddsVolumeRootDir, hddsVolume.getClusterID()));
+ volumeSet.failVolume(hddsVolumeRootDir.getPath());
+ return;
}
LOG.info("Start to verify containers on volume {}", hddsVolumeRootDir);
- File currentDir = new File(idDir, Storage.STORAGE_DIR_CURRENT);
File[] containerTopDirs = currentDir.listFiles();
if (containerTopDirs != null && containerTopDirs.length > 0) {
for (File containerTopDir : containerTopDirs) {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java
index 4c4a45c55d4a..65a7a1371d3e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java
@@ -66,27 +66,35 @@ public void scanData(Container container, DataTransferThrottler throttler, Ca
long containerId = containerData.getContainerID();
logScanStart(containerData, "data");
DataScanResult result = container.scanData(throttler, canceler);
-
+ Instant now = Instant.now();
+
if (result.isDeleted()) {
log.debug("Container [{}] has been deleted during the data scan.", containerId);
- } else {
+ logScanCompleted(containerData, now);
+ return;
+ }
+
+ boolean isTransientFailure = ScanTransientIOUtil.scanErrorsAreOnlyTooManyOpenFiles(result);
+
+ if (!isTransientFailure) {
try {
controller.updateContainerChecksum(containerId, result.getDataTree());
} catch (IOException ex) {
log.warn("Failed to update container checksum after scan of container {}", containerId, ex);
}
- if (result.hasErrors()) {
- handleUnhealthyScanResult(containerData, result);
- }
- metrics.incNumContainersScanned();
}
- Instant now = Instant.now();
- if (!result.isDeleted()) {
+ if (result.hasErrors()) {
+ handleUnhealthyScanResult(containerData, result, isTransientFailure);
+ }
+
+ if (!isTransientFailure) {
+ metrics.incNumContainersScanned();
controller.updateDataScanTimestamp(containerId, now);
+ logScanCompleted(containerData, now);
+ } else {
+ logScanIncomplete(containerData, now, "data");
}
- // Even if the container was deleted, mark the scan as completed since we already logged it as starting.
- logScanCompleted(containerData, now);
}
public void scanMetadata(Container container)
@@ -103,20 +111,37 @@ public void scanMetadata(Container container)
log.debug("Container [{}] has been deleted during metadata scan.", containerId);
return;
}
+
+ boolean isTransientFailure = ScanTransientIOUtil.scanErrorsAreOnlyTooManyOpenFiles(result);
+
if (result.hasErrors()) {
- handleUnhealthyScanResult(containerData, result);
+ handleUnhealthyScanResult(containerData, result, isTransientFailure);
}
Instant now = Instant.now();
// Do not update the scan timestamp after the scan since this was just a
// metadata scan, not a full data scan.
- metrics.incNumContainersScanned();
- // Even if the container was deleted, mark the scan as completed since we already logged it as starting.
- logScanCompleted(containerData, now);
+ if (!isTransientFailure) {
+ metrics.incNumContainersScanned();
+ // Even if the container was deleted, mark the scan as completed since we already logged it as starting.
+ logScanCompleted(containerData, now);
+ } else {
+ logScanIncomplete(containerData, now, "metadata");
+ }
}
- public void handleUnhealthyScanResult(ContainerData containerData, ScanResult result) throws IOException {
+ /**
+ * Marks container UNHEALTHY when the scan reports real errors.
+ * If every scan error is related to file-descriptor exhaustion, return without marking container unhealthy.
+ */
+ public void handleUnhealthyScanResult(ContainerData containerData, ScanResult result,
+ boolean isTransientFailure) throws IOException {
long containerID = containerData.getContainerID();
+ if (isTransientFailure) {
+ log.warn("Skipped marking container UNHEALTHY [{}]: scan failed due to transient " +
+ "file descriptor exhaustion ('Too many open files'). {}", containerID, result);
+ return;
+ }
log.error("Corruption detected in container [{}]. Marking it UNHEALTHY. {}", containerID, result);
if (log.isDebugEnabled()) {
StringBuilder allErrorString = new StringBuilder();
@@ -205,4 +230,11 @@ private void logScanCompleted(
log.debug("Completed scan of container {} at {}",
containerData.getContainerID(), timestamp);
}
+
+ private void logScanIncomplete(ContainerData containerData, Instant timestamp, String scanType) {
+ if (log.isDebugEnabled()) {
+ log.debug("Incomplete {} scan of container {} at {} due to transient file descriptor exhaustion",
+ scanType, containerData.getContainerID(), timestamp);
+ }
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 96983cf59a87..37fee50e6a85 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -52,6 +52,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name;
@@ -245,7 +246,6 @@ public OzoneContainer(HddsDatanodeService hddsDatanodeService,
datanodeDetails, config, hddsDispatcher, controller, certClient, context);
replicationServer = new ReplicationServer(
- controller,
conf.getObject(ReplicationConfig.class),
secConf,
certClient,
@@ -304,7 +304,8 @@ public OzoneContainer(HddsDatanodeService hddsDatanodeService,
config);
} else {
diskBalancerService = null;
- LOG.info("Disk Balancer is disabled.");
+ LOG.info("Disk Balancer is not enabled. Please enable the " +
+ HddsConfigKeys.HDDS_DATANODE_DISK_BALANCER_ENABLED_KEY + " configuration key.");
}
Duration recoveringContainerScrubbingSvcInterval =
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ScanTransientIOUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ScanTransientIOUtil.java
new file mode 100644
index 000000000000..1be9acf3816d
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ScanTransientIOUtil.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.ozoneimpl;
+
+import java.nio.file.FileSystemException;
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.Locale;
+import java.util.Set;
+import org.apache.hadoop.ozone.container.common.interfaces.ScanResult;
+
+/**
+ * Utility to catch transient scan failures (typically related to file-descriptor exhaustion)
+ * that should not be treated as container data corruption.
+ */
+public final class ScanTransientIOUtil {
+
+ private static final int MAX_CAUSE_CHAIN_DEPTH = 64;
+
+ private static final String TOO_MANY_OPEN_FILES = "too many open files";
+
+ private ScanTransientIOUtil() {
+ }
+
+ /**
+ * Returns true when every scan error is related to file-descriptor exhaustion.
+ * Each error's exception chain is checked via {@link #isTooManyOpenFiles(Throwable)}.
+ */
+ public static boolean scanErrorsAreOnlyTooManyOpenFiles(ScanResult scanResult) {
+ if (!scanResult.hasErrors()) {
+ return false;
+ }
+ return scanResult.getErrors().stream()
+ .allMatch(scanError -> isTooManyOpenFiles(scanError.getException()));
+ }
+
+ public static boolean isTooManyOpenFiles(Throwable throwable) {
+ if (throwable == null) {
+ return false;
+ }
+ Set visited = Collections.newSetFromMap(new IdentityHashMap<>());
+ int depth = 0;
+ for (Throwable cause = throwable;
+ cause != null && depth < MAX_CAUSE_CHAIN_DEPTH;
+ cause = cause.getCause(), depth++) {
+ if (!visited.add(cause)) {
+ break;
+ }
+ if (matchesTooManyOpenFiles(cause)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean matchesTooManyOpenFiles(Throwable throwable) {
+ if (throwable instanceof FileSystemException) {
+ String reason = ((FileSystemException) throwable).getReason();
+ if (reason != null && containsTooManyOpenFiles(reason)) {
+ return true;
+ }
+ }
+ String message = throwable.getMessage();
+ return message != null && containsTooManyOpenFiles(message);
+ }
+
+ private static boolean containsTooManyOpenFiles(String text) {
+ return text.toLowerCase(Locale.ROOT).contains(TOO_MANY_OPEN_FILES);
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/AbstractReplicationTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/AbstractReplicationTask.java
index 05932e6edf79..aa9b985c5a58 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/AbstractReplicationTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/AbstractReplicationTask.java
@@ -42,8 +42,6 @@ public abstract class AbstractReplicationTask {
private ReplicationCommandPriority priority = NORMAL;
- private boolean shouldOnlyRunOnInServiceDatanodes = true;
-
protected AbstractReplicationTask(long containerID,
long deadlineMsSinceEpoch, long term) {
this(containerID, deadlineMsSinceEpoch, term,
@@ -115,24 +113,6 @@ public ReplicationCommandPriority getPriority() {
return priority;
}
- /**
- * Returns true if the task should only run on in service datanodes. False
- * otherwise.
- */
- public boolean shouldOnlyRunOnInServiceDatanodes() {
- return shouldOnlyRunOnInServiceDatanodes;
- }
-
- /**
- * Set whether the task should only run on in service datanodes. Passing false
- * allows the task to run on out of service datanodes as well.
- * @param runOnInServiceOnly
- */
- protected void setShouldOnlyRunOnInServiceDatanodes(
- boolean runOnInServiceOnly) {
- this.shouldOnlyRunOnInServiceDatanodes = runOnInServiceOnly;
- }
-
/**
* Hook for subclasses to provide info about the command.
* @return string representation of the command
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/CopyContainerResponseStream.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/CopyContainerResponseStream.java
deleted file mode 100644
index 61cecf1255b1..000000000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/CopyContainerResponseStream.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.replication;
-
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.CopyContainerResponseProto;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-import org.apache.ratis.thirdparty.io.grpc.stub.CallStreamObserver;
-
-/**
- * Output stream adapter for CopyContainerResponse.
- */
-class CopyContainerResponseStream
- extends GrpcOutputStream {
-
- CopyContainerResponseStream(
- CallStreamObserver streamObserver,
- long containerId, int bufferSize) {
- super(streamObserver, containerId, bufferSize);
- }
-
- @Override
- protected void sendPart(boolean eof, int length, ByteString data) {
- CopyContainerResponseProto response =
- CopyContainerResponseProto.newBuilder()
- .setContainerID(getContainerId())
- .setData(data)
- .setEof(eof)
- .setReadOffset(getWrittenBytes())
- .setLen(length)
- .build();
- getStreamObserver().onNext(response);
- }
-}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
deleted file mode 100644
index 2457b592b141..000000000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.replication;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.List;
-import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
-import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.ozone.container.replication.AbstractReplicationTask.Status;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Default replication implementation.
- *
- * This class does the real job. Executes the download and import the container
- * to the container set.
- */
-public class DownloadAndImportReplicator implements ContainerReplicator {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(DownloadAndImportReplicator.class);
-
- private final ConfigurationSource conf;
- private final ContainerDownloader downloader;
- private final ContainerImporter containerImporter;
- private final ContainerSet containerSet;
-
- public DownloadAndImportReplicator(
- ConfigurationSource conf, ContainerSet containerSet,
- ContainerImporter containerImporter,
- ContainerDownloader downloader) {
- this.conf = conf;
- this.containerSet = containerSet;
- this.downloader = downloader;
- this.containerImporter = containerImporter;
- }
-
- @Override
- public void replicate(ReplicationTask task) {
- long containerID = task.getContainerId();
- if (containerSet.getContainer(containerID) != null) {
- LOG.debug("Container {} has already been downloaded.", containerID);
- task.setStatus(Status.SKIPPED);
- return;
- }
-
- List sourceDatanodes = task.getSources();
- CopyContainerCompression compression =
- CopyContainerCompression.getConf(conf);
-
- LOG.info("Starting replication of container {} from {} using {}",
- containerID, sourceDatanodes, compression);
- HddsVolume targetVolume = null;
-
- try {
- targetVolume = containerImporter.chooseNextVolume(
- containerImporter.getDefaultReplicationSpace());
-
- // Wait for the download. This thread pool is limiting the parallel
- // downloads, so it's ok to block here and wait for the full download.
- Path tarFilePath =
- downloader.getContainerDataFromReplicas(containerID, sourceDatanodes,
- ContainerImporter.getUntarDirectory(targetVolume), compression);
- if (tarFilePath == null) {
- task.setStatus(Status.FAILED);
- return;
- }
- long bytes = Files.size(tarFilePath);
- LOG.info("Container {} is downloaded with size {}, starting to import.",
- containerID, bytes);
- task.setTransferredBytes(bytes);
-
- containerImporter.importContainer(containerID, tarFilePath, targetVolume,
- compression);
-
- LOG.info("Container {} is replicated successfully", containerID);
- task.setStatus(Status.DONE);
- } catch (IOException e) {
- LOG.error("Container {} replication was unsuccessful.", containerID, e);
- task.setStatus(Status.FAILED);
- } finally {
- if (targetVolume != null) {
- targetVolume.incCommittedBytes(-containerImporter.getDefaultReplicationSpace());
- }
- }
- }
-
-}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcContainerUploader.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcContainerUploader.java
index 64adcb6c6168..3e726671cca5 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcContainerUploader.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcContainerUploader.java
@@ -104,7 +104,7 @@ protected GrpcReplicationClient createReplicationClient(
throws IOException {
return new GrpcReplicationClient(target.getIpAddress(),
target.getPort(Port.Name.REPLICATION).getValue(),
- securityConfig, certClient, compression);
+ securityConfig, certClient);
}
/**
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
index 3df3fb361efb..3ced8ce98d0d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
@@ -18,16 +18,8 @@
package org.apache.hadoop.ozone.container.replication;
import java.io.IOException;
-import java.io.OutputStream;
-import java.io.UncheckedIOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Objects;
-import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.CopyContainerRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.CopyContainerResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.SendContainerRequest;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.SendContainerResponse;
import org.apache.hadoop.hdds.protocol.datanode.proto.IntraDatanodeProtocolServiceGrpc;
@@ -35,7 +27,6 @@
import org.apache.hadoop.hdds.security.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
import org.apache.ratis.thirdparty.io.grpc.netty.GrpcSslContexts;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
@@ -46,7 +37,7 @@
import org.slf4j.LoggerFactory;
/**
- * Client to read container data from gRPC.
+ * Client to push container data to another datanode via gRPC.
*/
public class GrpcReplicationClient implements AutoCloseable {
@@ -57,15 +48,12 @@ public class GrpcReplicationClient implements AutoCloseable {
private final IntraDatanodeProtocolServiceStub client;
- private final CopyContainerCompression compression;
-
private final AtomicBoolean closed = new AtomicBoolean();
private final String debugString;
public GrpcReplicationClient(
String host, int port,
- SecurityConfig secConfig, CertificateClient certClient,
- CopyContainerCompression compression)
+ SecurityConfig secConfig, CertificateClient certClient)
throws IOException {
NettyChannelBuilder channelBuilder =
NettyChannelBuilder.forAddress(host, port)
@@ -90,33 +78,12 @@ public GrpcReplicationClient(
}
channel = channelBuilder.build();
client = IntraDatanodeProtocolServiceGrpc.newStub(channel);
- this.compression = compression;
debugString = getClass().getSimpleName()
+ "{" + host + ":" + port + "}"
+ "@" + Integer.toHexString(hashCode());
LOG.debug("{}: created", this);
}
- public CompletableFuture download(long containerId, Path dir) {
- CopyContainerRequestProto request =
- CopyContainerRequestProto.newBuilder()
- .setContainerID(containerId)
- .setLen(-1)
- .setReadOffset(0)
- .setCompression(compression.toProto())
- .build();
-
- CompletableFuture response = new CompletableFuture<>();
-
- Path destinationPath = dir
- .resolve(ContainerUtils.getContainerTarName(containerId));
-
- client.download(request,
- new StreamDownloader(containerId, response, destinationPath));
-
- return response;
- }
-
public StreamObserver upload(
StreamObserver responseObserver) {
return client.upload(responseObserver);
@@ -144,94 +111,4 @@ public void close() throws Exception {
public String toString() {
return debugString;
}
-
- /**
- * gRPC stream observer to CompletableFuture adapter.
- */
- public static class StreamDownloader
- implements StreamObserver {
-
- private final CompletableFuture response;
- private final long containerId;
- private final OutputStream stream;
- private final Path outputPath;
-
- public StreamDownloader(long containerId, CompletableFuture response,
- Path outputPath) {
- this.response = response;
- this.containerId = containerId;
- this.outputPath = Objects.requireNonNull(outputPath, "outputPath == null");
-
- final Path parentPath = this.outputPath.getParent();
- if (parentPath == null) {
- throw new NullPointerException("Output path has no parent: " + this.outputPath);
- }
-
- try {
- Files.createDirectories(parentPath);
- stream = Files.newOutputStream(this.outputPath);
- } catch (IOException e) {
- throw new UncheckedIOException(
- "Output path can't be used: " + this.outputPath, e);
- }
- }
-
- @Override
- public void onNext(CopyContainerResponseProto chunk) {
- try {
- chunk.getData().writeTo(stream);
- } catch (IOException e) {
- LOG.error("Failed to write the stream buffer to {} for container {}",
- outputPath, containerId, e);
- try {
- stream.close();
- } catch (IOException ex) {
- LOG.error("Failed to close OutputStream {}", outputPath, e);
- } finally {
- deleteOutputOnFailure();
- response.completeExceptionally(e);
- }
- }
- }
-
- @Override
- public void onError(Throwable throwable) {
- try {
- LOG.error("Download of container {} was unsuccessful",
- containerId, throwable);
- stream.close();
- deleteOutputOnFailure();
- response.completeExceptionally(throwable);
- } catch (IOException e) {
- LOG.error("Failed to close {} for container {}",
- outputPath, containerId, e);
- deleteOutputOnFailure();
- response.completeExceptionally(e);
- }
- }
-
- @Override
- public void onCompleted() {
- try {
- stream.close();
- LOG.info("Container {} is downloaded to {}", containerId, outputPath);
- response.complete(outputPath);
- } catch (IOException e) {
- LOG.error("Downloaded container {} OK, but failed to close {}",
- containerId, outputPath, e);
- deleteOutputOnFailure();
- response.completeExceptionally(e);
- }
- }
-
- private void deleteOutputOnFailure() {
- try {
- Files.delete(outputPath);
- } catch (IOException ex) {
- LOG.error("Failed to delete temporary destination {} for " +
- "unsuccessful download of container {}",
- outputPath, containerId, ex);
- }
- }
- }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationService.java
index 10cba29845f3..b8bf68fe7003 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationService.java
@@ -17,58 +17,37 @@
package org.apache.hadoop.ozone.container.replication;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.IntraDatanodeProtocolServiceGrpc.getDownloadMethod;
import static org.apache.hadoop.hdds.protocol.datanode.proto.IntraDatanodeProtocolServiceGrpc.getUploadMethod;
-import static org.apache.hadoop.ozone.container.replication.CopyContainerCompression.fromProto;
-import java.io.IOException;
-import java.io.OutputStream;
import java.util.HashSet;
import java.util.Set;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.CopyContainerRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.CopyContainerResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.SendContainerRequest;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.SendContainerResponse;
import org.apache.hadoop.hdds.protocol.datanode.proto.IntraDatanodeProtocolServiceGrpc;
-import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.ratis.grpc.util.ZeroCopyMessageMarshaller;
import org.apache.ratis.thirdparty.com.google.protobuf.MessageLite;
import org.apache.ratis.thirdparty.io.grpc.MethodDescriptor;
import org.apache.ratis.thirdparty.io.grpc.ServerCallHandler;
import org.apache.ratis.thirdparty.io.grpc.ServerServiceDefinition;
-import org.apache.ratis.thirdparty.io.grpc.stub.CallStreamObserver;
import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Service to make containers available for replication.
*/
-public class GrpcReplicationService extends
- IntraDatanodeProtocolServiceGrpc.IntraDatanodeProtocolServiceImplBase {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(GrpcReplicationService.class);
+public class GrpcReplicationService extends IntraDatanodeProtocolServiceGrpc.IntraDatanodeProtocolServiceImplBase {
static final int BUFFER_SIZE = 1024 * 1024;
- private final ContainerReplicationSource source;
private final ContainerImporter importer;
private final ZeroCopyMessageMarshaller
sendContainerZeroCopyMessageMarshaller;
- private final ZeroCopyMessageMarshaller
- copyContainerZeroCopyMessageMarshaller;
-
- public GrpcReplicationService(ContainerReplicationSource source, ContainerImporter importer) {
- this.source = source;
+ public GrpcReplicationService(ContainerImporter importer) {
this.importer = importer;
sendContainerZeroCopyMessageMarshaller = new ZeroCopyMessageMarshaller<>(
SendContainerRequest.getDefaultInstance());
- copyContainerZeroCopyMessageMarshaller = new ZeroCopyMessageMarshaller<>(
- CopyContainerRequestProto.getDefaultInstance());
}
public ServerServiceDefinition bindServiceWithZeroCopy() {
@@ -85,13 +64,6 @@ public ServerServiceDefinition bindServiceWithZeroCopy() {
sendContainerZeroCopyMessageMarshaller);
methodNames.add(uploadMethod.getFullMethodName());
- // Add `download` method with zerocopy marshaller.
- MethodDescriptor
- downloadMethod = getDownloadMethod();
- addZeroCopyMethod(orig, builder, downloadMethod,
- copyContainerZeroCopyMessageMarshaller);
- methodNames.add(downloadMethod.getFullMethodName());
-
// Add other methods as is.
orig.getMethods().stream().filter(
x -> !methodNames.contains(x.getMethodDescriptor().getFullMethodName())
@@ -117,31 +89,6 @@ private static void addZeroCopyMethod(
newServiceBuilder.addMethod(newMethod, serverCallHandler);
}
- @Override
- public void download(CopyContainerRequestProto request,
- StreamObserver responseObserver) {
- long containerID = request.getContainerID();
- CopyContainerCompression compression = fromProto(request.getCompression());
- LOG.info("Streaming container data ({}) to other datanode " +
- "with compression {}", containerID, compression);
- OutputStream outputStream = null;
- try {
- outputStream = new CopyContainerResponseStream(
- // gRPC runtime always provides implementation of CallStreamObserver
- // that allows flow control.
- (CallStreamObserver) responseObserver,
- containerID, BUFFER_SIZE);
- source.copyData(containerID, outputStream, compression);
- } catch (IOException e) {
- LOG.warn("Error streaming container {}", containerID, e);
- responseObserver.onError(e);
- } finally {
- // output may have already been closed, ignore such errors
- IOUtils.cleanupWithLogger(LOG, outputStream);
- copyContainerZeroCopyMessageMarshaller.release(request);
- }
- }
-
@Override
public StreamObserver upload(
StreamObserver responseObserver) {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java
index f2d06c2f6b1c..8375b1100bdc 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java
@@ -37,7 +37,6 @@
import org.apache.hadoop.hdds.tracing.GrpcServerInterceptor;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.ratis.thirdparty.io.grpc.Server;
import org.apache.ratis.thirdparty.io.grpc.ServerInterceptors;
import org.apache.ratis.thirdparty.io.grpc.netty.GrpcSslContexts;
@@ -62,20 +61,16 @@ public class ReplicationServer {
private CertificateClient caClient;
- private ContainerController controller;
-
private int port;
private final ContainerImporter importer;
private ThreadPoolExecutor executor;
- public ReplicationServer(ContainerController controller,
- ReplicationConfig replicationConfig, SecurityConfig secConf,
- CertificateClient caClient, ContainerImporter importer,
- String threadNamePrefix) {
+ public ReplicationServer(ReplicationConfig replicationConfig,
+ SecurityConfig secConf, CertificateClient caClient,
+ ContainerImporter importer, String threadNamePrefix) {
this.secConf = secConf;
this.caClient = caClient;
- this.controller = controller;
this.importer = importer;
this.port = replicationConfig.getPort();
@@ -103,8 +98,7 @@ public ReplicationServer(ContainerController controller,
}
public void init() {
- GrpcReplicationService grpcReplicationService = new GrpcReplicationService(
- new OnDemandContainerReplicationSource(controller), importer);
+ GrpcReplicationService grpcReplicationService = new GrpcReplicationService(importer);
NettyServerBuilder nettyServerBuilder = NettyServerBuilder.forPort(port)
.maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.addService(ServerInterceptors.intercept(
@@ -182,10 +176,10 @@ public static final class ReplicationConfig {
public static final int REPLICATION_MAX_STREAMS_DEFAULT = 10;
private static final String OUTOFSERVICE_FACTOR_KEY =
"outofservice.limit.factor";
- private static final double OUTOFSERVICE_FACTOR_MIN = 1;
+ static final double OUTOFSERVICE_FACTOR_MIN = 1;
static final double OUTOFSERVICE_FACTOR_DEFAULT = 2;
private static final String OUTOFSERVICE_FACTOR_DEFAULT_VALUE = "2.0";
- private static final double OUTOFSERVICE_FACTOR_MAX = 10;
+ static final double OUTOFSERVICE_FACTOR_MAX = 10;
static final String REPLICATION_OUTOFSERVICE_FACTOR_KEY =
PREFIX + "." + OUTOFSERVICE_FACTOR_KEY;
@@ -274,14 +268,16 @@ public void validate() {
if (outOfServiceFactor < OUTOFSERVICE_FACTOR_MIN ||
outOfServiceFactor > OUTOFSERVICE_FACTOR_MAX) {
+ double clamped = Math.min(OUTOFSERVICE_FACTOR_MAX,
+ Math.max(OUTOFSERVICE_FACTOR_MIN, outOfServiceFactor));
LOG.warn(
- "{} must be between {} and {} but was set to {}. Defaulting to {}",
+ "{} must be between {} and {} but was set to {}. Clamping to {}",
REPLICATION_OUTOFSERVICE_FACTOR_KEY,
OUTOFSERVICE_FACTOR_MIN,
OUTOFSERVICE_FACTOR_MAX,
outOfServiceFactor,
- OUTOFSERVICE_FACTOR_DEFAULT);
- outOfServiceFactor = OUTOFSERVICE_FACTOR_DEFAULT;
+ clamped);
+ outOfServiceFactor = clamped;
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisor.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisor.java
index 8dee840db226..8b6daf6aae5a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisor.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisor.java
@@ -58,7 +58,7 @@
import org.slf4j.LoggerFactory;
/**
- * Single point to schedule the downloading tasks based on priorities.
+ * Single point to schedule container replication tasks based on priorities.
*/
public final class ReplicationSupervisor {
@@ -90,9 +90,9 @@ public final class ReplicationSupervisor {
}
/**
- * A set of container IDs that are currently being downloaded
- * or queued for download. Tracked so we don't schedule > 1
- * concurrent download for the same container. Note that the uniqueness of a
+ * A set of container IDs that are currently being sent
+ * or queued. Tracked so we don't schedule > 1
+ * concurrent replications for the same container. Note that the uniqueness of a
* task is defined by the tasks equals and hashCode methods.
*/
private final Set inFlight;
@@ -227,7 +227,7 @@ private ReplicationSupervisor(StateContext context, ExecutorService executor,
}
/**
- * Queue an asynchronous download of the given container.
+ * Queue an asynchronous replication of the given container.
*/
public void addTask(AbstractReplicationTask task) {
if (queueHasRoomFor(task)) {
@@ -346,21 +346,31 @@ public int getMaxQueueSize() {
public void nodeStateUpdated(HddsProtos.NodeOperationalState newState) {
if (state.getAndSet(newState) != newState) {
- int threadCount = replicationConfig.getReplicationMaxStreams();
- int newMaxQueueSize = datanodeConfig.getCommandQueueLimit();
+ resize(newState);
+ }
+ }
- if (isMaintenance(newState) || isDecommission(newState)) {
- threadCount = replicationConfig.scaleOutOfServiceLimit(threadCount);
- newMaxQueueSize =
- replicationConfig.scaleOutOfServiceLimit(newMaxQueueSize);
- }
+ public void setReplicationMaxStreams(int replicationMaxStreams) {
+ replicationConfig.setReplicationMaxStreams(replicationMaxStreams);
+ resize(state.get());
+ }
- LOG.info("Node state updated to {}, scaling executor pool size to {}",
- newState, threadCount);
+ private void resize(HddsProtos.NodeOperationalState nodeState) {
+ int threadCount = replicationConfig.getReplicationMaxStreams();
+ int newMaxQueueSize = datanodeConfig.getCommandQueueLimit();
- maxQueueSize = newMaxQueueSize;
- executorThreadUpdater.accept(threadCount);
+ if (isMaintenance(nodeState) || isDecommission(nodeState)) {
+ threadCount = replicationConfig.scaleOutOfServiceLimit(threadCount);
+ newMaxQueueSize =
+ replicationConfig.scaleOutOfServiceLimit(newMaxQueueSize);
}
+
+ LOG.info("Scaling replication supervisor for node state {} to executor " +
+ "pool size {} and queue size {}", nodeState, threadCount,
+ newMaxQueueSize);
+
+ maxQueueSize = newMaxQueueSize;
+ executorThreadUpdater.accept(threadCount);
}
/**
@@ -389,15 +399,6 @@ public void run() {
}
if (context != null) {
- DatanodeDetails dn = context.getParent().getDatanodeDetails();
- if (dn != null && dn.getPersistedOpState() !=
- HddsProtos.NodeOperationalState.IN_SERVICE
- && task.shouldOnlyRunOnInServiceDatanodes()) {
- LOG.info("Ignoring {} since datanode is not in service ({})",
- this, dn.getPersistedOpState());
- return;
- }
-
final OptionalLong currentTerm = context.getTermOfLeaderSCM();
final long taskTerm = task.getTerm();
if (currentTerm.isPresent() && taskTerm < currentTerm.getAsLong()) {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationTask.java
index a32e9b41ab1b..ce8f535e0c4a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationTask.java
@@ -17,13 +17,12 @@
package org.apache.hadoop.ozone.container.replication;
-import java.util.List;
import java.util.Objects;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
/**
- * The task to download a container from the sources.
+ * Task to push a container to a target datanode.
*/
public class ReplicationTask extends AbstractReplicationTask {
@@ -44,28 +43,9 @@ public ReplicationTask(ReplicateContainerCommand cmd,
setPriority(cmd.getPriority());
this.cmd = cmd;
this.replicator = replicator;
- if (cmd.getTargetDatanode() != null) {
- // Only push replication will have a target datanode set, and it must be
- // sent to the source datanode to be executed. It is possible the source
- // is out of service, so we need to set the flag to allow the command to
- // run.
- setShouldOnlyRunOnInServiceDatanodes(false);
- }
debugString = cmd.toString();
}
- /**
- * Intended to only be used in tests.
- */
- protected ReplicationTask(
- long containerId,
- List sources,
- ContainerReplicator replicator
- ) {
- this(ReplicateContainerCommand.fromSources(containerId, sources),
- replicator);
- }
-
@Override
public String getMetricName() {
return METRIC_NAME;
@@ -99,10 +79,6 @@ public long getContainerId() {
return cmd.getContainerID();
}
- public List getSources() {
- return cmd.getSourceDatanodes();
- }
-
@Override
protected Object getCommandForDebug() {
return debugString;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SimpleContainerDownloader.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SimpleContainerDownloader.java
deleted file mode 100644
index 145d63680c23..000000000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SimpleContainerDownloader.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.replication;
-
-import com.google.common.annotations.VisibleForTesting;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name;
-import org.apache.hadoop.hdds.security.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
-import org.apache.hadoop.hdds.utils.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Simple ContainerDownloaderImplementation to download the missing container
- * from the first available datanode.
- *
- * This is not the most effective implementation as it uses only one source
- * for he container download.
- */
-public class SimpleContainerDownloader implements ContainerDownloader {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(SimpleContainerDownloader.class);
-
- private final SecurityConfig securityConfig;
- private final CertificateClient certClient;
-
- public SimpleContainerDownloader(
- ConfigurationSource conf, CertificateClient certClient) {
- securityConfig = new SecurityConfig(conf);
- this.certClient = certClient;
- }
-
- @Override
- public Path getContainerDataFromReplicas(
- long containerId, List sourceDatanodes,
- Path downloadDir, CopyContainerCompression compression) {
-
- if (downloadDir == null) {
- downloadDir = Paths.get(System.getProperty("java.io.tmpdir"))
- .resolve(ContainerImporter.CONTAINER_COPY_DIR);
- }
-
- final List shuffledDatanodes =
- shuffleDatanodes(sourceDatanodes);
-
- for (int i = 0; i < shuffledDatanodes.size(); i++) {
- DatanodeDetails datanode = shuffledDatanodes.get(i);
- GrpcReplicationClient client = null;
- try {
- client = createReplicationClient(datanode, compression);
- CompletableFuture result =
- downloadContainer(client, containerId, downloadDir);
- return result.get();
- } catch (InterruptedException e) {
- logError(e, containerId, datanode, i, shuffledDatanodes.size());
- Thread.currentThread().interrupt();
- } catch (Exception e) {
- logError(e, containerId, datanode, i, shuffledDatanodes.size());
- } finally {
- IOUtils.close(LOG, client);
- }
- }
- LOG.error("Container {} could not be downloaded from any datanode",
- containerId);
- return null;
- }
-
- private static void logError(Exception e,
- long containerId, DatanodeDetails datanode, int datanodeIndex,
- int shuffledDatanodesSize) {
- StringBuilder sb =
- new StringBuilder("Error on replicating container: {} from {}. ");
- if (datanodeIndex < shuffledDatanodesSize - 1) {
- sb.append("Will try next datanode.");
- }
- LOG.error(sb.toString(), containerId,
- datanode, e);
- }
-
- //There is a chance for the download is successful but import is failed,
- //due to data corruption. We need a random selected datanode to have a
- //chance to succeed next time.
- @VisibleForTesting
- protected List shuffleDatanodes(
- List sourceDatanodes) {
-
- final ArrayList shuffledDatanodes =
- new ArrayList<>(sourceDatanodes);
-
- Collections.shuffle(shuffledDatanodes);
-
- return shuffledDatanodes;
- }
-
- @VisibleForTesting
- protected GrpcReplicationClient createReplicationClient(
- DatanodeDetails datanode, CopyContainerCompression compression
- ) throws IOException {
- return new GrpcReplicationClient(datanode.getIpAddress(),
- datanode.getPort(Name.REPLICATION).getValue(),
- securityConfig, certClient, compression);
- }
-
- @VisibleForTesting
- protected CompletableFuture downloadContainer(
- GrpcReplicationClient client, long containerId, Path downloadDir) {
- return client.download(containerId, downloadDir);
- }
-
- @Override
- public void close() {
- // noop
- }
-}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java
index bc8040b24bfc..c35439096532 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java
@@ -17,13 +17,8 @@
package org.apache.hadoop.ozone.protocol.commands;
-import static java.util.Collections.emptyList;
-
-import java.util.List;
import java.util.Objects;
-import java.util.stream.Collectors;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ReplicateContainerCommandProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ReplicateContainerCommandProto.Builder;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ReplicationCommandPriority;
@@ -31,47 +26,33 @@
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
/**
- * SCM command to request replication of a container.
+ * SCM command to request push-replication of a container to a target datanode.
*/
public final class ReplicateContainerCommand
extends SCMCommand {
private final long containerID;
- private final List sourceDatanodes;
private final DatanodeDetails targetDatanode;
private int replicaIndex = 0;
private ReplicationCommandPriority priority =
ReplicationCommandPriority.NORMAL;
- public static ReplicateContainerCommand fromSources(long containerID,
- List sourceDatanodes) {
- return new ReplicateContainerCommand(containerID, sourceDatanodes, null);
- }
-
public static ReplicateContainerCommand toTarget(long containerID,
DatanodeDetails target) {
- return new ReplicateContainerCommand(containerID, emptyList(), target);
- }
-
- public static ReplicateContainerCommand forTest(long containerID) {
- return new ReplicateContainerCommand(containerID, emptyList(), null);
+ return new ReplicateContainerCommand(containerID, target);
}
- private ReplicateContainerCommand(long containerID,
- List sourceDatanodes, DatanodeDetails target) {
+ private ReplicateContainerCommand(long containerID, DatanodeDetails target) {
this.containerID = containerID;
- this.sourceDatanodes = sourceDatanodes;
- this.targetDatanode = target;
+ this.targetDatanode = Objects.requireNonNull(target, "target == null");
}
// Should be called only for protobuf conversion
- private ReplicateContainerCommand(long containerID,
- List sourceDatanodes, long id,
+ private ReplicateContainerCommand(long containerID, long id,
DatanodeDetails targetDatanode) {
super(id);
this.containerID = containerID;
- this.sourceDatanodes = sourceDatanodes;
- this.targetDatanode = targetDatanode;
+ this.targetDatanode = Objects.requireNonNull(targetDatanode, "target == null");
}
public void setReplicaIndex(int index) {
@@ -96,15 +77,10 @@ public boolean contributesToQueueSize() {
public ReplicateContainerCommandProto getProto() {
Builder builder = ReplicateContainerCommandProto.newBuilder()
.setCmdId(getId())
- .setContainerID(containerID);
- for (DatanodeDetails dd : sourceDatanodes) {
- builder.addSources(dd.getProtoBufMessage());
- }
- builder.setReplicaIndex(replicaIndex);
- if (targetDatanode != null) {
- builder.setTarget(targetDatanode.getProtoBufMessage());
- }
- builder.setPriority(priority);
+ .setContainerID(containerID)
+ .setReplicaIndex(replicaIndex)
+ .setTarget(targetDatanode.getProtoBufMessage())
+ .setPriority(priority);
return builder.build();
}
@@ -112,19 +88,12 @@ public static ReplicateContainerCommand getFromProtobuf(
ReplicateContainerCommandProto protoMessage) {
Objects.requireNonNull(protoMessage, "protoMessage == null");
- List sources = protoMessage.getSourcesList();
- List sourceNodes = !sources.isEmpty()
- ? sources.stream()
- .map(DatanodeDetails::getFromProtoBuf)
- .collect(Collectors.toList())
- : emptyList();
- DatanodeDetails targetNode = protoMessage.hasTarget()
- ? DatanodeDetails.getFromProtoBuf(protoMessage.getTarget())
- : null;
+ DatanodeDetails targetNode =
+ DatanodeDetails.getFromProtoBuf(protoMessage.getTarget());
ReplicateContainerCommand cmd =
new ReplicateContainerCommand(protoMessage.getContainerID(),
- sourceNodes, protoMessage.getCmdId(), targetNode);
+ protoMessage.getCmdId(), targetNode);
if (protoMessage.hasReplicaIndex()) {
cmd.setReplicaIndex(protoMessage.getReplicaIndex());
}
@@ -138,10 +107,6 @@ public long getContainerID() {
return containerID;
}
- public List getSourceDatanodes() {
- return sourceDatanodes;
- }
-
public DatanodeDetails getTargetDatanode() {
return targetDatanode;
}
@@ -156,20 +121,14 @@ public ReplicationCommandPriority getPriority() {
@Override
public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(getType())
- .append(": cmdID: ").append(getId())
- .append(", encodedToken: \"").append(getEncodedToken()).append('"')
- .append(", term: ").append(getTerm())
- .append(", deadlineMsSinceEpoch: ").append(getDeadline())
- .append(", containerId=").append(getContainerID())
- .append(", replicaIndex=").append(getReplicaIndex());
- if (targetDatanode != null) {
- sb.append(", targetNode=").append(targetDatanode);
- } else {
- sb.append(", sourceNodes=").append(sourceDatanodes);
- }
- sb.append(", priority=").append(priority);
- return sb.toString();
+ return getType()
+ + ": cmdID: " + getId()
+ + ", encodedToken: \"" + getEncodedToken() + '"'
+ + ", term: " + getTerm()
+ + ", deadlineMsSinceEpoch: " + getDeadline()
+ + ", containerId=" + getContainerID()
+ + ", replicaIndex=" + getReplicaIndex()
+ + ", targetNode=" + targetDatanode
+ + ", priority=" + priority;
}
}
diff --git a/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn-overview.html b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn-overview.html
index f1a89b779ee2..e25ce4a9c822 100644
--- a/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn-overview.html
+++ b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn-overview.html
@@ -57,10 +57,12 @@ Volume Information
Ozone Used |
Ozone Available |
Reserved |
- Total Capacity (Ozone Capacity + Reserved) |
Filesystem Capacity |
Filesystem Available |
Filesystem Used |
+ Min Free Space |
+ Hard Min Free Space |
+ Non-Ozone Used |
Containers |
State |
@@ -74,10 +76,12 @@ Volume Information
{{volumeInfo.OzoneUsed}} |
{{volumeInfo.OzoneAvailable}} |
{{volumeInfo.Reserved}} |
- {{volumeInfo.TotalCapacity}} |
{{volumeInfo.FilesystemCapacity}} |
{{volumeInfo.FilesystemAvailable}} |
{{volumeInfo.FilesystemUsed}} |
+ {{volumeInfo.MinFreeSpace}} |
+ {{volumeInfo.HardMinFreeSpace}} |
+ {{volumeInfo.NonOzoneUsed}} |
{{volumeInfo.Containers}} |
{{volumeInfo["tag.VolumeState"]}} |
diff --git a/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn.js b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn.js
index cd3a238a883f..1f5ae99f15e0 100644
--- a/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn.js
+++ b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn.js
@@ -34,10 +34,12 @@
volume.OzoneUsed = transform(volume.OzoneUsed);
volume.OzoneAvailable = transform(volume.OzoneAvailable);
volume.Reserved = transform(volume.Reserved);
- volume.TotalCapacity = transform(volume.TotalCapacity);
volume.FilesystemCapacity = transform(volume.FilesystemCapacity);
volume.FilesystemAvailable = transform(volume.FilesystemAvailable);
volume.FilesystemUsed = transform(volume.FilesystemUsed);
+ volume.MinFreeSpace = transform(volume.MinFreeSpace);
+ volume.HardMinFreeSpace = transform(volume.HardMinFreeSpace);
+ volume.NonOzoneUsed = transform(volume.NonOzoneUsed);
})
});
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
index 588d8572f035..ca11cf2f7105 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
@@ -31,10 +31,13 @@
import java.io.File;
import java.io.IOException;
+import java.lang.management.ManagementFactory;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -53,6 +56,7 @@
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
import org.apache.hadoop.util.ServicePlugin;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
@@ -158,6 +162,27 @@ public void testDeletedContainersClearedOnShutdown(String schemaVersion)
assertEquals(0, deletedContainersAfterShutdown.length);
}
+ @Test
+ public void testDatanodeUuidInMXBean() throws Exception {
+ try {
+ service.start(conf);
+
+ ObjectName bean = new ObjectName(
+ "Hadoop:service=HddsDatanodeService,"
+ + "name=HddsDatanodeServiceInfo,"
+ + "component=ServerRuntime");
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ String datanodeUuid = (String) mbs.getAttribute(bean, "DatanodeUuid");
+
+ assertEquals(service.getDatanodeDetails().getUuidString(), datanodeUuid);
+ } finally {
+ service.stop();
+ service.join();
+ service.close();
+ DefaultMetricsSystem.shutdown();
+ }
+ }
+
@ParameterizedTest
@EnumSource
void testHttpPorts(HttpConfig.Policy policy) {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java
index fa3c209fa873..8231b2b3c7f9 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java
@@ -46,6 +46,7 @@
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
+import org.apache.hadoop.hdds.scm.net.HostAndPort;
import org.apache.hadoop.hdds.security.token.TokenVerifier;
import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.hdfs.util.Canceler;
@@ -150,7 +151,7 @@ public static EndpointStateMachine createEndpoint(Configuration conf,
StorageContainerDatanodeProtocolClientSideTranslatorPB rpcClient =
new StorageContainerDatanodeProtocolClientSideTranslatorPB(rpcProxy);
- return new EndpointStateMachine(address, rpcClient,
+ return new EndpointStateMachine(new HostAndPort(address.getHostName(), address.getPort()), rpcClient,
new LegacyHadoopConfigurationSource(conf), "");
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java
index a21813956f59..fe4204b159f9 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java
@@ -58,7 +58,7 @@
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.StaleRecoveringContainerScrubbingService;
-import org.apache.ozone.test.TestClock;
+import org.apache.ozone.test.MockClock;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.io.TempDir;
@@ -77,8 +77,8 @@ public class TestStaleRecoveringContainerScrubbingService {
private int containerIdNum = 0;
private MutableVolumeSet volumeSet;
private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy;
- private final TestClock testClock =
- new TestClock(Instant.now(), ZoneOffset.UTC);
+ private final MockClock testClock =
+ new MockClock(Instant.now(), ZoneOffset.UTC);
private void initVersionInfo(ContainerTestVersionInfo versionInfo)
throws IOException {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestSCMConnectionManager.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestSCMConnectionManager.java
new file mode 100644
index 000000000000..0225b6aa873f
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestSCMConnectionManager.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.statemachine;
+
+import static org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine.EndPointStates.HEARTBEAT;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.net.HostAndPort;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for SCMConnectionManager.
+ */
+public class TestSCMConnectionManager {
+
+ @Test
+ public void testRemoveSCMServerDoesNotMarkEndpointShutdown()
+ throws Exception {
+ try (SCMConnectionManager connectionManager =
+ new SCMConnectionManager(new OzoneConfiguration())) {
+ final HostAndPort address = new HostAndPort("127.0.0.1", 9861);
+ connectionManager.addSCMServer(address, "");
+ EndpointStateMachine endpoint =
+ connectionManager.getValues().iterator().next();
+ endpoint.setState(HEARTBEAT);
+
+ connectionManager.removeSCMServer(address);
+
+ Assertions.assertTrue(connectionManager.getValues().isEmpty());
+ Assertions.assertEquals(HEARTBEAT, endpoint.getState());
+ }
+ }
+}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestStateContext.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestStateContext.java
index 8d79335591b9..42220c6b99fe 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestStateContext.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestStateContext.java
@@ -33,7 +33,6 @@
import com.google.protobuf.Descriptors.Descriptor;
import com.google.protobuf.Message;
import java.io.IOException;
-import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -50,6 +49,7 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerAction;
@@ -58,6 +58,7 @@
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.scm.net.HostAndPort;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
@@ -88,9 +89,9 @@ public void testPutBackReports() {
StateContext ctx = new StateContext(conf, DatanodeStates.getInitState(),
datanodeStateMachineMock, "");
- InetSocketAddress scm1 = new InetSocketAddress("scm1", 9001);
+ HostAndPort scm1 = new HostAndPort("scm1", 9001);
ctx.addEndpoint(scm1);
- InetSocketAddress scm2 = new InetSocketAddress("scm2", 9001);
+ HostAndPort scm2 = new HostAndPort("scm2", 9001);
ctx.addEndpoint(scm2);
Map expectedReportCount = new HashMap<>();
@@ -142,9 +143,9 @@ public void testPutBackReports() {
@Test
public void testReportQueueWithAddReports() throws IOException {
StateContext ctx = createSubject();
- InetSocketAddress scm1 = new InetSocketAddress("scm1", 9001);
+ HostAndPort scm1 = new HostAndPort("scm1", 9001);
ctx.addEndpoint(scm1);
- InetSocketAddress scm2 = new InetSocketAddress("scm2", 9001);
+ HostAndPort scm2 = new HostAndPort("scm2", 9001);
ctx.addEndpoint(scm2);
// Check initial state
assertEquals(0, ctx.getAllAvailableReports(scm1).size());
@@ -303,9 +304,9 @@ private StateContext newStateContext(OzoneConfiguration conf,
DatanodeStateMachine datanodeStateMachineMock) {
StateContext stateContext = new StateContext(conf,
DatanodeStates.getInitState(), datanodeStateMachineMock, "");
- InetSocketAddress scm1 = new InetSocketAddress("scm1", 9001);
+ HostAndPort scm1 = new HostAndPort("scm1", 9001);
stateContext.addEndpoint(scm1);
- InetSocketAddress scm2 = new InetSocketAddress("scm2", 9001);
+ HostAndPort scm2 = new HostAndPort("scm2", 9001);
stateContext.addEndpoint(scm2);
return stateContext;
}
@@ -332,8 +333,8 @@ public void testReportAPIs() {
StateContext stateContext = new StateContext(conf,
DatanodeStates.getInitState(), datanodeStateMachineMock, "");
- InetSocketAddress scm1 = new InetSocketAddress("scm1", 9001);
- InetSocketAddress scm2 = new InetSocketAddress("scm2", 9001);
+ HostAndPort scm1 = new HostAndPort("scm1", 9001);
+ HostAndPort scm2 = new HostAndPort("scm2", 9001);
Message generatedMessage =
newMockReport(StateContext.COMMAND_STATUS_REPORTS_PROTO_NAME);
@@ -394,7 +395,7 @@ public void testClosePipelineActions() {
StateContext stateContext = new StateContext(conf,
DatanodeStates.getInitState(), datanodeStateMachineMock, "");
- InetSocketAddress scm1 = new InetSocketAddress("scm1", 9001);
+ HostAndPort scm1 = new HostAndPort("scm1", 9001);
// Add SCM endpoint.
stateContext.addEndpoint(scm1);
@@ -452,8 +453,8 @@ public void testActionAPIs() {
StateContext stateContext = new StateContext(conf,
DatanodeStates.getInitState(), datanodeStateMachineMock, "");
- InetSocketAddress scm1 = new InetSocketAddress("scm1", 9001);
- InetSocketAddress scm2 = new InetSocketAddress("scm2", 9001);
+ HostAndPort scm1 = new HostAndPort("scm1", 9001);
+ HostAndPort scm2 = new HostAndPort("scm2", 9001);
// Try to get containerActions for endpoint which is not yet added.
List containerActions =
@@ -655,9 +656,9 @@ public void testGetReports() {
StateContext ctx = new StateContext(conf, DatanodeStates.getInitState(),
datanodeStateMachineMock, "");
- InetSocketAddress scm1 = new InetSocketAddress("scm1", 9001);
+ HostAndPort scm1 = new HostAndPort("scm1", 9001);
ctx.addEndpoint(scm1);
- InetSocketAddress scm2 = new InetSocketAddress("scm2", 9001);
+ HostAndPort scm2 = new HostAndPort("scm2", 9001);
ctx.addEndpoint(scm2);
// Check initial state
assertEquals(0, ctx.getAllAvailableReports(scm1).size());
@@ -702,10 +703,10 @@ public void testGetReports() {
@Test
public void testCommandQueueSummary() throws IOException {
StateContext ctx = createSubject();
- ctx.addCommand(ReplicateContainerCommand.forTest(1));
+ ctx.addCommand(ReplicateContainerCommand.toTarget(1, MockDatanodeDetails.randomDatanodeDetails()));
ctx.addCommand(new ClosePipelineCommand(PipelineID.randomId()));
- ctx.addCommand(ReplicateContainerCommand.forTest(2));
- ctx.addCommand(ReplicateContainerCommand.forTest(3));
+ ctx.addCommand(ReplicateContainerCommand.toTarget(2, MockDatanodeDetails.randomDatanodeDetails()));
+ ctx.addCommand(ReplicateContainerCommand.toTarget(3, MockDatanodeDetails.randomDatanodeDetails()));
ctx.addCommand(new ClosePipelineCommand(PipelineID.randomId()));
ctx.addCommand(new CloseContainerCommand(1, PipelineID.randomId()));
ctx.addCommand(new ReconcileContainerCommand(4, Collections.emptySet()));
@@ -772,7 +773,7 @@ private static StateContext createSubject() throws IOException {
}
private static SCMCommand someCommand() {
- return ReplicateContainerCommand.forTest(1);
+ return ReplicateContainerCommand.toTarget(1, MockDatanodeDetails.randomDatanodeDetails());
}
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteBlocksCommandHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteBlocksCommandHandler.java
index 6ac39d9ba4f3..a85f80ca373d 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteBlocksCommandHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteBlocksCommandHandler.java
@@ -28,6 +28,8 @@
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
@@ -52,6 +54,7 @@
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
@@ -82,6 +85,8 @@
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.protocol.commands.CommandStatus;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
+import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ratis.util.ExitUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -167,6 +172,7 @@ private void setup() throws Exception {
public void tearDown() {
handler.stop();
BlockDeletingServiceMetrics.unRegister();
+ ExitUtils.clear();
}
@ContainerTestVersionInfo.ContainerTest
@@ -315,6 +321,108 @@ public void testDeleteBlocksCommandHandlerExceptionShouldNotInterrupt() throws E
assertEquals(1, deleteBlockTransactionResults.size());
}
+ @Test
+ public void testDeleteBlocksCommandHandlerErrorShouldInterrupt() throws Exception {
+ setup();
+ Error error = new AssertionError("Simulated Error");
+ CompletableFuture failed =
+ new CompletableFuture<>();
+ failed.completeExceptionally(error);
+ CompletableFuture unprocessed =
+ CompletableFuture.completedFuture(
+ new DeleteBlockTransactionExecutionResult(null, false));
+ AtomicInteger processed = new AtomicInteger();
+
+ Error thrown = assertThrows(Error.class, () -> handler.handleTasksResults(
+ Arrays.asList(failed, unprocessed), result -> processed.incrementAndGet()));
+
+ assertSame(error, thrown);
+ assertEquals(0, processed.get());
+ }
+
+ @Test
+ public void testDeleteBlocksCommandHandlerErrorOnRetryShouldInterrupt()
+ throws Exception {
+ setup();
+ DeletedBlocksTransaction transaction =
+ createDeletedBlocksTransaction(1, 1);
+ DeleteBlockTransactionResult retryResult = DeleteBlockTransactionResult
+ .newBuilder()
+ .setTxID(transaction.getTxID())
+ .setContainerID(transaction.getContainerID())
+ .setSuccess(false)
+ .build();
+ CompletableFuture retry =
+ CompletableFuture.completedFuture(
+ new DeleteBlockTransactionExecutionResult(retryResult, true));
+ Error error = new AssertionError("Simulated retry Error");
+ CompletableFuture failed =
+ new CompletableFuture<>();
+ failed.completeExceptionally(error);
+ AtomicInteger invocation = new AtomicInteger();
+ doAnswer(ignored -> invocation.getAndIncrement() == 0
+ ? Collections.singletonList(retry)
+ : Collections.singletonList(failed))
+ .when(handler).submitTasks(any());
+
+ Error thrown = assertThrows(Error.class,
+ () -> handler.executeCmdWithRetry(
+ Collections.singletonList(transaction)));
+
+ assertSame(error, thrown);
+ assertEquals(2, invocation.get());
+ }
+
+ @Test
+ public void testDeleteCmdWorkerTerminatesOnError() throws Exception {
+ setup();
+ ExitUtils.disableSystemExit();
+ Container container = containerSet.getContainer(1);
+ String schemaVersionOrDefault = ((KeyValueContainerData)
+ container.getContainerData()).getSupportedSchemaVersionOrDefault();
+ Error error = new AssertionError("Simulated worker Error");
+ SchemaHandler schemaHandler =
+ handler.getSchemaHandlers().get(schemaVersionOrDefault);
+ CountDownLatch processingStarted = new CountDownLatch(1);
+ CountDownLatch failProcessing = new CountDownLatch(1);
+ doAnswer(ignored -> {
+ processingStarted.countDown();
+ assertTrue(failProcessing.await(5, TimeUnit.SECONDS));
+ throw error;
+ }).when(schemaHandler).handle(any(), any());
+
+ OzoneConfiguration conf = new OzoneConfiguration();
+ DatanodeStateMachine stateMachine = mock(DatanodeStateMachine.class);
+ DatanodeDetails datanodeDetails =
+ MockDatanodeDetails.randomDatanodeDetails();
+ when(stateMachine.getDatanodeDetails()).thenReturn(datanodeDetails);
+ StateContext context = new StateContext(conf,
+ DatanodeStateMachine.DatanodeStates.RUNNING, stateMachine, "");
+ DeleteBlocksCommand fatalCommand = new DeleteBlocksCommand(
+ Collections.singletonList(createDeletedBlocksTransaction(1, 1)));
+ DeleteBlocksCommand queuedCommand = new DeleteBlocksCommand(emptyList());
+ context.addCommand(fatalCommand);
+ context.addCommand(queuedCommand);
+
+ handler.handle(fatalCommand, mock(OzoneContainer.class), context,
+ mock(SCMConnectionManager.class));
+ assertTrue(processingStarted.await(5, TimeUnit.SECONDS));
+ try {
+ handler.handle(queuedCommand, mock(OzoneContainer.class), context,
+ mock(SCMConnectionManager.class));
+ } finally {
+ failProcessing.countDown();
+ }
+
+ GenericTestUtils.waitFor(ExitUtils::isTerminated, 10, 5000);
+
+ assertSame(error, ExitUtils.getFirstExitException().getCause());
+ CommandStatus fatalStatus = context.getCmdStatus(fatalCommand.getId());
+ assertEquals(Status.FAILED, fatalStatus.getStatus());
+ assertFalse(fatalStatus.getProtoBufMessage().hasBlockDeletionAck());
+ assertEquals(Status.PENDING, context.getCmdStatus(queuedCommand.getId()).getStatus());
+ }
+
@ContainerTestVersionInfo.ContainerTest
public void testDeleteCmdWorkerInterval(
ContainerTestVersionInfo versionInfo) throws Exception {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteContainerCommandHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteContainerCommandHandler.java
index e4f35691544f..ca970fb882b6 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteContainerCommandHandler.java
@@ -42,7 +42,7 @@
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.protocol.commands.DeleteContainerCommand;
-import org.apache.ozone.test.TestClock;
+import org.apache.ozone.test.MockClock;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -51,14 +51,14 @@
*/
public class TestDeleteContainerCommandHandler {
- private TestClock clock;
+ private MockClock clock;
private OzoneContainer ozoneContainer;
private ContainerController controller;
private StateContext context;
@BeforeEach
public void setup() {
- clock = new TestClock(Instant.now(), ZoneId.systemDefault());
+ clock = new MockClock(Instant.now(), ZoneId.systemDefault());
ozoneContainer = mock(OzoneContainer.class);
controller = mock(ContainerController.class);
when(ozoneContainer.getController()).thenReturn(controller);
@@ -114,7 +114,7 @@ public void testCommandForCurrentTermIsExecuted()
when(context.getTermOfLeaderSCM())
.thenReturn(OptionalLong.of(command.getTerm()));
- TestClock testClock = new TestClock(Instant.now(), ZoneId.systemDefault());
+ MockClock testClock = new MockClock(Instant.now(), ZoneId.systemDefault());
CountDownLatch latch = new CountDownLatch(1);
ThreadFactory threadFactory = new ThreadFactoryBuilder().build();
ThreadPoolWithLockExecutor executor = new ThreadPoolWithLockExecutor(
@@ -181,12 +181,12 @@ public void testQueueSize() throws IOException {
}
private static DeleteContainerCommandHandler createSubject() {
- TestClock clock = new TestClock(Instant.now(), ZoneId.systemDefault());
+ MockClock clock = new MockClock(Instant.now(), ZoneId.systemDefault());
return createSubject(clock, 1000);
}
private static DeleteContainerCommandHandler createSubject(
- TestClock clock, int queueSize) {
+ MockClock clock, int queueSize) {
ThreadFactory threadFactory = new ThreadFactoryBuilder().build();
ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.
newFixedThreadPool(1, threadFactory);
@@ -194,7 +194,7 @@ private static DeleteContainerCommandHandler createSubject(
}
private static DeleteContainerCommandHandler createSubjectWithPoolSize(
- TestClock clock, int queueSize) {
+ MockClock clock, int queueSize) {
return new DeleteContainerCommandHandler(1, clock, queueSize, "");
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReconcileContainerCommandHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReconcileContainerCommandHandler.java
index 72969f976e58..3ce546214aba 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReconcileContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReconcileContainerCommandHandler.java
@@ -177,7 +177,7 @@ private void verifyAllContainerReports(Map r
for (Map.Entry entry: reportsSent.entrySet()) {
ContainerID id = entry.getKey();
- assertNotNull(containerSet.getContainer(id.getId()));
+ assertNotNull(containerSet.getContainer(id.getIdForTesting()));
long sentDataChecksum = entry.getValue().getDataChecksum();
// Current implementation is incomplete, and uses a mocked checksum.
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerCommandHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerCommandHandler.java
index b88b6da7ea7d..74edeae7aff9 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerCommandHandler.java
@@ -23,11 +23,8 @@
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
@@ -47,9 +44,7 @@
* Test cases to verify {@link ReplicateContainerCommandHandler}.
*/
public class TestReplicateContainerCommandHandler {
- private OzoneConfiguration conf;
private ReplicationSupervisor supervisor;
- private ContainerReplicator downloadReplicator;
private ContainerReplicator pushReplicator;
private OzoneContainer ozoneContainer;
private StateContext stateContext;
@@ -57,9 +52,7 @@ public class TestReplicateContainerCommandHandler {
@BeforeEach
public void setUp() {
- conf = new OzoneConfiguration();
supervisor = mock(ReplicationSupervisor.class);
- downloadReplicator = mock(ContainerReplicator.class);
pushReplicator = mock(ContainerReplicator.class);
ozoneContainer = mock(OzoneContainer.class);
connectionManager = mock(SCMConnectionManager.class);
@@ -69,36 +62,30 @@ public void setUp() {
@Test
public void testMetrics() {
ReplicateContainerCommandHandler commandHandler =
- new ReplicateContainerCommandHandler(conf, supervisor,
- downloadReplicator, pushReplicator);
+ new ReplicateContainerCommandHandler(supervisor, pushReplicator);
Map handlerMap = new HashMap<>();
handlerMap.put(commandHandler.getCommandType(), commandHandler);
CommandHandlerMetrics metrics = CommandHandlerMetrics.create(handlerMap);
try {
doNothing().when(supervisor).addTask(any());
- DatanodeDetails source = MockDatanodeDetails.randomDatanodeDetails();
DatanodeDetails target = MockDatanodeDetails.randomDatanodeDetails();
- List sourceList = new ArrayList<>();
- sourceList.add(source);
- ReplicateContainerCommand command = ReplicateContainerCommand.fromSources(
- 1, sourceList);
+ ReplicateContainerCommand command =
+ ReplicateContainerCommand.toTarget(1, target);
commandHandler.handle(command, ozoneContainer, stateContext, connectionManager);
String metricsName = ReplicationTask.METRIC_NAME;
assertEquals(commandHandler.getMetricsName(), metricsName);
when(supervisor.getReplicationRequestCount(metricsName)).thenReturn(1L);
assertEquals(commandHandler.getInvocationCount(), 1);
- commandHandler.handle(ReplicateContainerCommand.fromSources(2, sourceList),
+ commandHandler.handle(ReplicateContainerCommand.toTarget(2, target),
ozoneContainer, stateContext, connectionManager);
- commandHandler.handle(ReplicateContainerCommand.fromSources(3, sourceList),
+ commandHandler.handle(ReplicateContainerCommand.toTarget(3, target),
ozoneContainer, stateContext, connectionManager);
commandHandler.handle(ReplicateContainerCommand.toTarget(4, target),
ozoneContainer, stateContext, connectionManager);
commandHandler.handle(ReplicateContainerCommand.toTarget(5, target),
ozoneContainer, stateContext, connectionManager);
- commandHandler.handle(ReplicateContainerCommand.fromSources(6, sourceList),
- ozoneContainer, stateContext, connectionManager);
when(supervisor.getReplicationRequestCount(metricsName)).thenReturn(5L);
when(supervisor.getReplicationRequestTotalTime(metricsName)).thenReturn(10L);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java
index e2b4fa167ea8..12a38d3dcfb3 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java
@@ -30,7 +30,6 @@
import static org.mockito.Mockito.when;
import com.google.protobuf.UnsafeByteOperations;
-import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -50,6 +49,7 @@
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
+import org.apache.hadoop.hdds.scm.net.HostAndPort;
import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
@@ -67,8 +67,7 @@
*/
public class TestHeartbeatEndpointTask {
- private static final InetSocketAddress TEST_SCM_ENDPOINT =
- new InetSocketAddress("test-scm-1", 9861);
+ private static final HostAndPort TEST_SCM_ENDPOINT = new HostAndPort("test-scm-1", 9861);
@Test
public void handlesReconstructContainerCommand() throws Exception {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestContainerStateMachine.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachineTests.java
similarity index 95%
rename from hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestContainerStateMachine.java
rename to hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachineTests.java
index ced776034484..8ad92ec358e9 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachineTests.java
@@ -70,7 +70,7 @@
* Test class to ContainerStateMachine class.
*/
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
-abstract class TestContainerStateMachine {
+abstract class ContainerStateMachineTests {
private ContainerDispatcher dispatcher;
private final OzoneConfiguration conf = new OzoneConfiguration();
private ContainerStateMachine stateMachine;
@@ -82,7 +82,7 @@ abstract class TestContainerStateMachine {
private final boolean isLeader;
private static final String CONTAINER_DATA = "Test Data";
- TestContainerStateMachine(boolean isLeader) {
+ ContainerStateMachineTests(boolean isLeader) {
this.isLeader = isLeader;
}
@@ -107,6 +107,11 @@ public void setup() throws IOException {
when(ratisServer.getServerDivision(any())).thenReturn(division);
stateMachine = new ContainerStateMachine(null,
RaftGroupId.randomId(), dispatcher, controller, executor, ratisServer, conf, "containerOp");
+ try {
+ stateMachine.initialize(raftServer, stateMachine.getGroupId(), null);
+ } catch (Exception e) {
+ // Ingore exception, as need init server to be closed
+ }
}
@AfterEach
@@ -149,8 +154,8 @@ public void testWriteFailure(boolean failWithException) throws ExecutionExceptio
stateMachine.write(entryNext, trx).exceptionally(catcher.asSetter()).get();
verify(dispatcher, times(0)).dispatch(any(ContainerProtos.ContainerCommandRequestProto.class),
any(DispatcherContext.class));
- assertInstanceOf(StorageContainerException.class, catcher.getReceived());
- StorageContainerException sce = (StorageContainerException) catcher.getReceived();
+ assertInstanceOf(StorageContainerException.class, catcher.getReceived().getCause());
+ StorageContainerException sce = (StorageContainerException) catcher.getReceived().getCause();
assertEquals(ContainerProtos.Result.CONTAINER_UNHEALTHY, sce.getResult());
}
@@ -233,12 +238,12 @@ public void testWriteTimout() throws Exception {
CompletableFuture secondWrite = stateMachine.write(entryNext, trx);
firstWrite.exceptionally(catcher.asSetter()).get();
assertNotNull(catcher.getCaught());
- assertInstanceOf(InterruptedException.class, catcher.getReceived());
+ assertInstanceOf(InterruptedException.class, catcher.getReceived().getCause());
secondWrite.exceptionally(catcher.asSetter()).get();
- assertNotNull(catcher.getReceived());
- assertInstanceOf(StorageContainerException.class, catcher.getReceived());
- StorageContainerException sce = (StorageContainerException) catcher.getReceived();
+ assertNotNull(catcher.getReceived().getCause());
+ assertInstanceOf(StorageContainerException.class, catcher.getReceived().getCause());
+ StorageContainerException sce = (StorageContainerException) catcher.getReceived().getCause();
assertEquals(ContainerProtos.Result.CONTAINER_INTERNAL_ERROR, sce.getResult());
}
@@ -269,8 +274,12 @@ private void assertResults(boolean failWithException, AtomicReference
if (failWithException) {
assertInstanceOf(RuntimeException.class, throwable.get());
} else {
- assertInstanceOf(StorageContainerException.class, throwable.get());
- StorageContainerException sce = (StorageContainerException) throwable.get();
+ Throwable th = throwable.get();
+ if (null != th.getCause()) {
+ th = th.getCause();
+ }
+ assertInstanceOf(StorageContainerException.class, th);
+ StorageContainerException sce = (StorageContainerException) th;
assertEquals(ContainerProtos.Result.CONTAINER_INTERNAL_ERROR, sce.getResult());
}
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestContainerStateMachineFollower.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestContainerStateMachineFollower.java
index e63dfac3ffed..3c068f0d93ff 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestContainerStateMachineFollower.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestContainerStateMachineFollower.java
@@ -20,7 +20,7 @@
/**
* Test class to ContainerStateMachine class for follower.
*/
-public class TestContainerStateMachineFollower extends TestContainerStateMachine {
+public class TestContainerStateMachineFollower extends ContainerStateMachineTests {
public TestContainerStateMachineFollower() {
super(false);
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestContainerStateMachineLeader.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestContainerStateMachineLeader.java
index 29ded1465b14..0a7e184c1a6c 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestContainerStateMachineLeader.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestContainerStateMachineLeader.java
@@ -20,7 +20,7 @@
/**
* Test class to ContainerStateMachine class for leader.
*/
-public class TestContainerStateMachineLeader extends TestContainerStateMachine {
+public class TestContainerStateMachineLeader extends ContainerStateMachineTests {
public TestContainerStateMachineLeader() {
super(true);
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java
index d66f39455949..6df9ac4dd47b 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java
@@ -20,11 +20,21 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mockStatic;
import java.io.File;
+import java.nio.file.FileSystemException;
+import java.nio.file.OpenOption;
+import java.nio.file.Path;
+import org.apache.ratis.util.FileUtils;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
+import org.mockito.MockedStatic;
/**
* Tests {@link DiskCheckUtil} does not incorrectly identify an unhealthy
@@ -32,6 +42,7 @@
* Tests that it identifies an improperly configured directory mount point.
*
*/
+@Execution(ExecutionMode.SAME_THREAD)
public class TestDiskCheckUtil {
@TempDir
@@ -74,10 +85,30 @@ public void testExistence() {
@Test
public void testReadWrite() {
assertTrue(DiskCheckUtil.checkReadWrite(testDir, testDir, 10));
+ assertTestFileDeleted();
+ }
+ private void assertTestFileDeleted() {
// Test file should have been deleted.
File[] children = testDir.listFiles();
assertNotNull(children);
assertEquals(0, children.length);
}
+
+ @Test
+ public void testCheckReadWriteDiskFull() {
+ try (MockedStatic mockService = mockStatic(FileUtils.class)) {
+ // fos.write(writtenBytes) also through FileSystemException with the message
+ mockService.when(() -> FileUtils.newOutputStreamForceAtClose(any(Path.class), any(OpenOption[].class)))
+ .thenThrow(new FileSystemException("No space left on device"));
+
+ assertThrows(FileSystemException.class,
+ () -> FileUtils.newOutputStreamForceAtClose(testDir.toPath(), new OpenOption[2]));
+
+ // Test that checkReadWrite returns true for the disk full case
+ boolean result = DiskCheckUtil.checkReadWrite(testDir, testDir, 1024);
+ assertTrue(result, "checkReadWrite should return true when disk is full");
+ assertTestFileDeleted();
+ }
+ }
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestCapacityVolumeChoosingPolicy.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestCapacityVolumeChoosingPolicy.java
index 7917ebf80bd9..e9d6f8739cf9 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestCapacityVolumeChoosingPolicy.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestCapacityVolumeChoosingPolicy.java
@@ -121,6 +121,79 @@ public void testCapacityVolumeChoosingPolicy() throws Exception {
assertThat(chooseCount.get(hddsVolume3)).isGreaterThan(chooseCount.get(hddsVolume2));
}
+ @Test
+ public void testChoosesLowerUtilizationAcrossDifferentCapacities() throws Exception {
+ // big has more free bytes but is 90% full; small is only 60% full.
+ // The policy must prefer the less-utilized volume, not the one with more free bytes.
+ SpaceUsageSource bigSource = MockSpaceUsageSource.fixed(1000, 100);
+ HddsVolume bigVolume = new HddsVolume.Builder(baseDir + "big")
+ .conf(CONF)
+ .usageCheckFactory(MockSpaceUsageCheckFactory.of(
+ bigSource, Duration.ZERO, SpaceUsagePersistence.None.INSTANCE))
+ .build();
+ SpaceUsageSource smallSource = MockSpaceUsageSource.fixed(200, 80);
+ HddsVolume smallVolume = new HddsVolume.Builder(baseDir + "small")
+ .conf(CONF)
+ .usageCheckFactory(MockSpaceUsageCheckFactory.of(
+ smallSource, Duration.ZERO, SpaceUsagePersistence.None.INSTANCE))
+ .build();
+
+ List mixedVolumes = new ArrayList<>();
+ mixedVolumes.add(bigVolume);
+ mixedVolumes.add(smallVolume);
+
+ Map chooseCount = new HashMap<>();
+ chooseCount.put(bigVolume, 0);
+ chooseCount.put(smallVolume, 0);
+
+ try {
+ for (int i = 0; i < 1000; i++) {
+ HddsVolume volume = policy.chooseVolume(mixedVolumes, 0);
+ chooseCount.put(volume, chooseCount.get(volume) + 1);
+ }
+ assertThat(chooseCount.get(smallVolume))
+ .isGreaterThan(chooseCount.get(bigVolume));
+ } finally {
+ bigVolume.shutdown();
+ smallVolume.shutdown();
+ }
+ }
+
+ @Test
+ public void testFreeSpaceRatioIsZeroWhenCapacityUnknown() throws Exception {
+ // A volume with unknown capacity (<= 0) yields ratio 0 instead of dividing by zero,
+ // so it is never preferred over a volume with known capacity.
+ SpaceUsageSource unknown = MockSpaceUsageSource.fixed(0, 0);
+ HddsVolume volume = new HddsVolume.Builder(baseDir + "unknown")
+ .conf(CONF)
+ .usageCheckFactory(MockSpaceUsageCheckFactory.of(
+ unknown, Duration.ZERO, SpaceUsagePersistence.None.INSTANCE))
+ .build();
+ try {
+ assertEquals(0.0, CapacityVolumeChoosingPolicy.freeSpaceRatio(volume));
+ } finally {
+ volume.shutdown();
+ }
+ }
+
+ @Test
+ public void testFreeSpaceRatioIsClampedToZeroWhenOverCommitted() throws Exception {
+ // committed exceeds available, so the raw free space is negative.
+ // The ratio must clamp to 0 rather than return a negative value.
+ SpaceUsageSource source = MockSpaceUsageSource.fixed(1000, 50);
+ HddsVolume volume = new HddsVolume.Builder(baseDir + "overcommitted")
+ .conf(CONF)
+ .usageCheckFactory(MockSpaceUsageCheckFactory.of(
+ source, Duration.ZERO, SpaceUsagePersistence.None.INSTANCE))
+ .build();
+ try {
+ volume.incCommittedBytes(100);
+ assertEquals(0.0, CapacityVolumeChoosingPolicy.freeSpaceRatio(volume));
+ } finally {
+ volume.shutdown();
+ }
+ }
+
@Test
public void throwsDiskOutOfSpaceIfRequestMoreThanAvailable() {
Exception e = assertThrows(DiskOutOfSpaceException.class,
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java
index f3e3a7fc2a53..da8aa0ce95bb 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java
@@ -274,8 +274,22 @@ public void testNumScansSkipped() throws Exception {
final List volumes = makeVolumes(3, expectedVolumeHealth);
FakeTimer timer = new FakeTimer();
+ // Configure diskCheckTimeout=0 so checkAllVolumes uses latch.await(0, ...) which
+ // returns immediately once the synchronous checks below have already fired the latch.
+ OzoneConfiguration testConf = new OzoneConfiguration();
+ DatanodeConfiguration dnConf = testConf.getObject(DatanodeConfiguration.class);
+ dnConf.setDiskCheckTimeout(Duration.ZERO);
+ testConf.setFromObject(dnConf);
final StorageVolumeChecker checker =
- new StorageVolumeChecker(new OzoneConfiguration(), timer, "");
+ new StorageVolumeChecker(testConf, timer, "");
+ // Use a synchronous (direct-executor) ThrottledAsyncChecker so that
+ // completedChecks is always fully updated before checkAllVolumes returns,
+ // eliminating the race between async callback completion and timer.advance().
+ checker.setDelegateChecker(new ThrottledAsyncChecker<>(
+ timer,
+ dnConf.getDiskCheckMinGap().toMillis(),
+ 0L,
+ MoreExecutors.newDirectExecutorService()));
VolumeInfoMetrics metrics1 = new VolumeInfoMetrics("test-volume-1", volumes.get(0));
VolumeInfoMetrics metrics2 = new VolumeInfoMetrics("test-volume-2", volumes.get(1));
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeHealthChecks.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeHealthChecks.java
index 675abed1696c..467c528a6db1 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeHealthChecks.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeHealthChecks.java
@@ -34,7 +34,7 @@
import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
import org.apache.hadoop.ozone.container.common.utils.DiskCheckUtil;
-import org.apache.ozone.test.TestClock;
+import org.apache.ozone.test.MockClock;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Named;
import org.junit.jupiter.api.Test;
@@ -52,7 +52,7 @@ public class TestStorageVolumeHealthChecks {
private static final String DATANODE_UUID = UUID.randomUUID().toString();
private static final String CLUSTER_ID = UUID.randomUUID().toString();
private static final OzoneConfiguration CONF = new OzoneConfiguration();
- private static final TestClock TEST_CLOCK = TestClock.newInstance();
+ private static final MockClock TEST_CLOCK = MockClock.newInstance();
@TempDir
private static Path volumePath;
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeInfoMetrics.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeInfoMetrics.java
index 7dc96458fdb7..c428be693db8 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeInfoMetrics.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeInfoMetrics.java
@@ -19,6 +19,7 @@
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -43,20 +44,24 @@ void testVolumeInfoMetricsExposeOzoneAndFilesystemGauges() {
when(volume.getType()).thenReturn(HddsVolume.VolumeType.DATA_VOLUME);
when(volume.getCommittedBytes()).thenReturn(10L);
when(volume.getContainers()).thenReturn(3L);
+ when(volume.getReportedFreeSpaceToSpare(anyLong())).thenReturn(20L);
+ when(volume.getFreeSpaceToSpare(anyLong())).thenReturn(15L);
VolumeUsage volumeUsage = mock(VolumeUsage.class);
when(volume.getVolumeUsage()).thenReturn(volumeUsage);
- // Ozone-usable usage and reserved
- when(volumeUsage.getCurrentUsage(any())).thenReturn(new SpaceUsageSource.Fixed(
- 1000L,
- 900L,
- 100L
- ));
when(volumeUsage.getReservedInBytes()).thenReturn(50L);
- // Raw filesystem stats
- when(volumeUsage.realUsage()).thenReturn(new SpaceUsageSource.Fixed(2000L, 1500L, 500L));
+ // Raw filesystem stats (used = Ozone DU usage on disk)
+ SpaceUsageSource.Fixed fsUsage = new SpaceUsageSource.Fixed(2000L, 1100L, 500L);
+ when(volumeUsage.realUsage()).thenReturn(fsUsage);
+
+ // getCurrentUsage(real) preserves real.getUsedSpace(); capacity/available are adjusted for reserved
+ when(volumeUsage.getCurrentUsage(any())).thenReturn(new SpaceUsageSource.Fixed(
+ 1950L, // fsCapacity - reserved
+ 1100L,
+ 500L // same as fsUsage.getUsedSpace()
+ ));
VolumeInfoMetrics metrics = new VolumeInfoMetrics("test-vol-1", volume);
try {
@@ -67,13 +72,18 @@ void testVolumeInfoMetricsExposeOzoneAndFilesystemGauges() {
MetricsRecordImpl rec = collector.getRecords().get(0);
Iterable all = rec.metrics();
- assertThat(findMetric(all, "OzoneCapacity")).isEqualTo(1000L);
- assertThat(findMetric(all, "OzoneAvailable")).isEqualTo(900L);
- assertThat(findMetric(all, "OzoneUsed")).isEqualTo(100L);
+ assertThat(findMetric(all, "OzoneCapacity")).isEqualTo(1950L);
+ assertThat(findMetric(all, "OzoneAvailable")).isEqualTo(1100L);
+ assertThat(findMetric(all, "OzoneUsed")).isEqualTo(500L);
assertThat(findMetric(all, "FilesystemCapacity")).isEqualTo(2000L);
- assertThat(findMetric(all, "FilesystemAvailable")).isEqualTo(1500L);
- assertThat(findMetric(all, "FilesystemUsed")).isEqualTo(500L);
+ assertThat(findMetric(all, "FilesystemAvailable")).isEqualTo(1100L);
+ assertThat(findMetric(all, "FilesystemUsed")).isEqualTo(900L); // FilesystemCapacity - FilesystemAvailable
+
+ assertThat(findMetric(all, "MinFreeSpace")).isEqualTo(20L);
+ assertThat(findMetric(all, "HardMinFreeSpace")).isEqualTo(15L);
+ // NonOzoneUsed = FilesystemUsed - OzoneUsed = 900 - 500
+ assertThat(findMetric(all, "NonOzoneUsed")).isEqualTo(400L);
} finally {
metrics.unregister();
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSetDiskChecks.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSetDiskChecks.java
index e310eaf3dea7..c2c6ec822ce3 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSetDiskChecks.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSetDiskChecks.java
@@ -32,7 +32,6 @@
import com.google.protobuf.Message;
import java.io.File;
import java.io.IOException;
-import java.net.InetSocketAddress;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
@@ -47,6 +46,7 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.net.HostAndPort;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
@@ -316,7 +316,7 @@ public void testVolumeFailure() throws IOException {
new OzoneConfiguration(), DatanodeStateMachine
.DatanodeStates.getInitState(),
datanodeStateMachineMock, "");
- InetSocketAddress scm1 = new InetSocketAddress("scm1", 9001);
+ HostAndPort scm1 = new HostAndPort("scm1", 9001);
stateContext.addEndpoint(scm1);
when(datanodeStateMachineMock.getContainer()).thenReturn(ozoneContainer);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceTestImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceTestImpl.java
index ef1503219070..3f9bb3840022 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceTestImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceTestImpl.java
@@ -19,6 +19,7 @@
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.io.IOException;
+import java.time.Clock;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
@@ -46,6 +47,13 @@ public DiskBalancerServiceTestImpl(OzoneContainer container,
TimeUnit.MILLISECONDS, threadCount, conf);
}
+ public DiskBalancerServiceTestImpl(OzoneContainer container,
+ int serviceInterval, ConfigurationSource conf, int threadCount,
+ Clock clock) throws IOException {
+ super(container, serviceInterval, SERVICE_TIMEOUT_IN_MILLISECONDS,
+ TimeUnit.MILLISECONDS, threadCount, conf, clock);
+ }
+
public void runBalanceTasks() {
if (latch.getCount() > 0) {
this.latch.countDown();
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerProtocolServer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerProtocolServer.java
index f07543300d56..03494eb78f9d 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerProtocolServer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerProtocolServer.java
@@ -108,6 +108,7 @@ void setup() throws IOException {
.setUtilization(TEST_UTILIZATION_1)
.setCommittedBytes(TEST_COMMITTED_BYTES_1)
.setTotalCapacity(TEST_TOTAL_CAPACITY)
+ .setOzoneAvailable(TEST_TOTAL_CAPACITY - TEST_USED_SPACE_1)
.setUsedSpace(TEST_USED_SPACE_1)
.setEffectiveUsedSpace(TEST_EFFECTIVE_USED_SPACE_1)
.build(),
@@ -117,6 +118,7 @@ void setup() throws IOException {
.setUtilization(TEST_UTILIZATION_2)
.setCommittedBytes(TEST_COMMITTED_BYTES_2)
.setTotalCapacity(TEST_TOTAL_CAPACITY)
+ .setOzoneAvailable(TEST_TOTAL_CAPACITY - TEST_USED_SPACE_2)
.setUsedSpace(TEST_USED_SPACE_2)
.setEffectiveUsedSpace(TEST_EFFECTIVE_USED_SPACE_2)
.build()));
@@ -157,6 +159,7 @@ void testGetDiskBalancerInfoReport() throws IOException {
assertEquals(TEST_UTILIZATION_1, volReport0.getUtilization());
assertEquals(TEST_COMMITTED_BYTES_1, volReport0.getCommittedBytes());
assertEquals(TEST_TOTAL_CAPACITY, volReport0.getTotalCapacity());
+ assertEquals(TEST_TOTAL_CAPACITY - TEST_USED_SPACE_1, volReport0.getOzoneAvailable());
assertEquals(TEST_USED_SPACE_1, volReport0.getUsedSpace());
assertEquals(TEST_EFFECTIVE_USED_SPACE_1, volReport0.getEffectiveUsedSpace());
assertEquals(TEST_STORAGE_ID_2, volReport1.getStorageId());
@@ -164,6 +167,7 @@ void testGetDiskBalancerInfoReport() throws IOException {
assertEquals(TEST_UTILIZATION_2, volReport1.getUtilization());
assertEquals(TEST_COMMITTED_BYTES_2, volReport1.getCommittedBytes());
assertEquals(TEST_TOTAL_CAPACITY, volReport1.getTotalCapacity());
+ assertEquals(TEST_TOTAL_CAPACITY - TEST_USED_SPACE_2, volReport1.getOzoneAvailable());
assertEquals(TEST_USED_SPACE_2, volReport1.getUsedSpace());
assertEquals(TEST_EFFECTIVE_USED_SPACE_2, volReport1.getEffectiveUsedSpace());
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerTask.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerTask.java
index fc3fdb7b1401..c858ba186189 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerTask.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerTask.java
@@ -20,6 +20,7 @@
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_INTERNAL_ERROR;
import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.createDbInstancesForTestIfNeeded;
import static org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerService.DISK_BALANCER_DIR;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
@@ -39,6 +40,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -47,6 +49,9 @@
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.commons.io.FileUtils;
@@ -59,6 +64,8 @@
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
import org.apache.hadoop.hdds.utils.FaultInjector;
import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
@@ -83,6 +90,7 @@
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ozone.test.GenericTestUtils.LogCapturer;
+import org.apache.ozone.test.MockClock;
import org.assertj.core.api.Fail;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
@@ -113,6 +121,7 @@ public class TestDiskBalancerTask {
private HddsVolume sourceVolume;
private HddsVolume destVolume;
private DiskBalancerServiceTestImpl diskBalancerService;
+ private MockClock clock;
private static final long CONTAINER_ID = 1L;
private static final long CONTAINER_SIZE = 1024L * 1024L; // 1 MB
@@ -243,8 +252,9 @@ public void setup() throws Exception {
DiskBalancerConfiguration diskBalancerConfiguration = conf.getObject(DiskBalancerConfiguration.class);
diskBalancerConfiguration.setDiskBalancerShouldRun(true);
conf.setFromObject(diskBalancerConfiguration);
+ clock = MockClock.newInstance();
diskBalancerService = new DiskBalancerServiceTestImpl(ozoneContainer,
- 100, conf, 1);
+ 100, conf, 1, clock);
diskBalancerService.setReplicaDeletionDelay(0);
KeyValueContainer.setInjector(kvFaultInjector);
}
@@ -495,6 +505,115 @@ public void moveFailsDuringInMemoryUpdate(ContainerTestVersionInfo versionInfo)
assertEquals(initialSourceDelta, diskBalancerService.getDeltaSizes().get(sourceVolume));
}
+ /**
+ * When markContainerForDelete fails after import and ContainerSet update,
+ * the move is still reported as success, the destination replica is active, the source
+ * replica is queued for lazy deletion, and cleanup removes it after the delay.
+ */
+ @ContainerTestVersionInfo.ContainerTest
+ public void moveSucceedsWhenMarkContainerForDeleteFails(
+ ContainerTestVersionInfo versionInfo)
+ throws IOException, InterruptedException, TimeoutException {
+ setLayoutAndSchemaForTest(versionInfo);
+ long delay = 2_000L;
+ diskBalancerService.setReplicaDeletionDelay(delay);
+
+ KeyValueContainer container = createContainer(CONTAINER_ID, sourceVolume, State.CLOSED);
+ File oldContainerDir = new File(container.getContainerData().getContainerPath());
+ Path destDirPath = Paths.get(
+ KeyValueContainerLocationUtil.getBaseContainerLocation(
+ destVolume.getHddsRootDir().toString(), scmId, CONTAINER_ID));
+ assertThat(destDirPath.toFile())
+ .as("Destination container should not exist before task execution")
+ .doesNotExist();
+
+ KeyValueContainer spyContainer = spy(container);
+ containerSet.removeContainer(CONTAINER_ID);
+ containerSet.addContainer(spyContainer);
+ doThrow(new RuntimeException("simulated markContainerForDelete failure"))
+ .when(spyContainer).markContainerForDelete();
+
+ LogCapturer serviceLog = GenericTestUtils.LogCapturer.captureLogs(DiskBalancerService.class);
+ DiskBalancerService.DiskBalancerTask task = getTask();
+ task.call();
+
+ assertThat(serviceLog.getOutput())
+ .contains("Failed to mark the old container " + CONTAINER_ID + " for delete");
+ assertThat(serviceLog.getOutput())
+ .as("move should not roll back when markContainerForDelete fails")
+ .doesNotContain("Rolling back move");
+
+ assertEquals(1, diskBalancerService.getMetrics().getSuccessCount());
+ assertEquals(0, diskBalancerService.getMetrics().getFailureCount());
+ assertEquals(CONTAINER_SIZE, diskBalancerService.getMetrics().getSuccessBytes());
+
+ Container activeReplica = containerSet.getContainer(CONTAINER_ID);
+ assertThat(activeReplica).isNotSameAs(spyContainer);
+ assertEquals(destVolume, activeReplica.getContainerData().getVolume());
+ assertThat(new File(activeReplica.getContainerData().getContainerPath())).exists();
+ assertThat(oldContainerDir)
+ .as("Source replica should remain on disk until lazy deletion runs")
+ .exists();
+ assertEquals(1, diskBalancerService.getPendingDeletionQueueSize(),
+ "Source replica should be queued for lazy deletion after mark failure");
+
+ clock.fastForward(delay);
+ diskBalancerService.cleanupPendingDeletionContainers();
+
+ assertThat(oldContainerDir)
+ .as("Source replica should be removed after lazy deletion delay")
+ .doesNotExist();
+ assertEquals(0, diskBalancerService.getPendingDeletionQueueSize());
+ }
+
+ /**
+ * When lazy deletion fails, the pending queue entry is dropped and
+ * the source replica is not retried for deletion.
+ */
+ @ContainerTestVersionInfo.ContainerTest
+ public void lazyDeletionFailureDoesNotRetry(
+ ContainerTestVersionInfo versionInfo) throws Exception {
+ setLayoutAndSchemaForTest(versionInfo);
+ long delay = 2_000L;
+ diskBalancerService.setReplicaDeletionDelay(delay);
+
+ Container container = createContainer(CONTAINER_ID, sourceVolume, State.CLOSED);
+ File oldContainerDir = new File(container.getContainerData().getContainerPath());
+
+ DiskBalancerService.DiskBalancerTask task = getTask();
+ task.call();
+
+ assertEquals(1, diskBalancerService.getMetrics().getSuccessCount());
+ assertEquals(1, diskBalancerService.getPendingDeletionQueueSize());
+ assertThat(oldContainerDir).exists();
+
+ clock.fastForward(delay);
+
+ LogCapturer serviceLog = GenericTestUtils.LogCapturer.captureLogs(DiskBalancerService.class);
+ try (MockedStatic mockedUtil =
+ mockStatic(KeyValueContainerUtil.class, Mockito.CALLS_REAL_METHODS)) {
+ mockedUtil.when(() -> KeyValueContainerUtil.removeContainer(
+ any(KeyValueContainerData.class), any(OzoneConfiguration.class)))
+ .thenThrow(new IOException("simulated lazy deletion failure"));
+
+ diskBalancerService.cleanupPendingDeletionContainers();
+
+ assertThat(oldContainerDir)
+ .as("Source replica should remain when lazy deletion fails")
+ .exists();
+ assertEquals(0, diskBalancerService.getPendingDeletionQueueSize(),
+ "Failed deletion should be removed from the pending queue");
+ assertThat(serviceLog.getOutput())
+ .contains("Failed to delete old container " + CONTAINER_ID);
+ assertThat(serviceLog.getOutput()).contains("background scanners");
+ }
+
+ diskBalancerService.cleanupPendingDeletionContainers();
+ assertThat(oldContainerDir)
+ .as("Source replica should not be retried after lazy deletion failure")
+ .exists();
+ }
+
@ContainerTestVersionInfo.ContainerTest
public void moveFailsDuringOldContainerRemove(ContainerTestVersionInfo versionInfo) throws IOException {
setLayoutAndSchemaForTest(versionInfo);
@@ -599,10 +718,8 @@ public void testOldReplicaDelayedDeletion(ContainerTestVersionInfo versionInfo)
// create another container to trigger the deletion of old replicas
createContainer(CONTAINER_ID + 1, sourceVolume, State.CLOSED);
task = getTask();
- // Wait until the delayed deletion is eligible, then trigger cleanup.
- long deletionEligibleAt = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(delay);
- GenericTestUtils.waitFor(
- () -> System.nanoTime() >= deletionEligibleAt, 50, 12_000);
+ // Advance the injected clock until the delayed deletion is eligible.
+ clock.fastForward(delay);
task.call();
// Verify that the old container is deleted
assertFalse(oldContainerDir.exists());
@@ -679,6 +796,81 @@ public void testMoveSkippedWhenContainerStateChanged(State invalidState)
assertEquals(initialSourceDelta, diskBalancerService.getDeltaSizes().get(sourceVolume));
}
+ @ContainerTestVersionInfo.ContainerTest
+ public void testPendingDeletionDoesNotDropReplicasOnSameMillisecondKey(
+ ContainerTestVersionInfo versionInfo)
+ throws Exception {
+ setLayoutAndSchemaForTest(versionInfo);
+
+ long delayMs = 2_000L;
+ diskBalancerService.setReplicaDeletionDelay(delayMs);
+
+ long id1 = CONTAINER_ID;
+ long id2 = CONTAINER_ID + 1;
+ long initialSourceUsed = sourceVolume.getCurrentUsage().getUsedSpace();
+
+ Container c1 = createContainer(id1, sourceVolume, State.CLOSED);
+ Container c2 = createContainer(id2, sourceVolume, State.CLOSED);
+
+ File oldDir1 = new File(c1.getContainerData().getContainerPath());
+ File oldDir2 = new File(c2.getContainerData().getContainerPath());
+ assertTrue(oldDir1.exists());
+ assertTrue(oldDir2.exists());
+
+ // Reserve dest space like the choosing policy would.
+ destVolume.incCommittedBytes(c1.getContainerData().getBytesUsed());
+ destVolume.incCommittedBytes(c2.getContainerData().getBytesUsed());
+
+ // Schedule two moves (parallelThread default is 5 in config).
+ BackgroundTaskQueue queue = diskBalancerService.getTasks();
+ assertEquals(2, queue.size());
+ DiskBalancerService.DiskBalancerTask task1 =
+ (DiskBalancerService.DiskBalancerTask) queue.poll();
+ DiskBalancerService.DiskBalancerTask task2 =
+ (DiskBalancerService.DiskBalancerTask) queue.poll();
+ assertNotNull(task1);
+ assertNotNull(task2);
+
+ // Run both moves concurrently; fixed MockClock => same deadline key.
+ ExecutorService pool = Executors.newFixedThreadPool(2);
+ try {
+ List> futures = pool.invokeAll(Arrays.asList(
+ task1::call,
+ task2::call));
+ for (Future future : futures) {
+ future.get(30, TimeUnit.SECONDS);
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+
+ assertEquals(2, diskBalancerService.getMetrics().getSuccessCount());
+
+ assertEquals(1, diskBalancerService.getPendingDeletionDeadlineCount(),
+ "both moves should share one deadline key");
+ assertEquals(2, diskBalancerService.getPendingDeletionQueueSize(),
+ "both container replicas should be queued for deletion");
+
+ // Not deleted yet — delay has not elapsed.
+ assertTrue(oldDir1.exists());
+ assertTrue(oldDir2.exists());
+
+ clock.fastForward(delayMs);
+ diskBalancerService.cleanupPendingDeletionContainers();
+
+ assertEquals(0, diskBalancerService.getPendingDeletionQueueSize());
+ assertFalse(oldDir1.exists());
+ assertFalse(oldDir2.exists());
+ assertFalse(sourceVolume.getContainerIterator().hasNext(),
+ "source volume should have no containers after delayed deletion");
+ assertEquals(initialSourceUsed, sourceVolume.getCurrentUsage().getUsedSpace(),
+ "source volume used space should return to pre-move level after old replicas are deleted");
+
+ // New replicas live on dest volume.
+ assertTrue(new File(containerSet.getContainer(id1).getContainerData().getContainerPath()).exists());
+ assertTrue(new File(containerSet.getContainer(id2).getContainerData().getContainerPath()).exists());
+ }
+
private KeyValueContainer createContainer(long containerId, HddsVolume vol, State state)
throws IOException {
KeyValueContainerData containerData = new KeyValueContainerData(
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerVolumeCalculation.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerVolumeCalculation.java
index a32da8a3de9d..4289af7afb7f 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerVolumeCalculation.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerVolumeCalculation.java
@@ -83,6 +83,26 @@ void calculateVolumeDataDensityIgnoresZeroCapacityVolumes()
Arrays.asList(zeroCapacity, lowUsage, highUsage)), 0.0);
}
+ @Test
+ void getUtilizationReturnsZeroForZeroCapacityVolume()
+ throws IOException {
+ HddsVolume volume = createVolume("zero-capacity-utilization", 0, 0);
+
+ assertEquals(0.0, DiskBalancerVolumeCalculation.newVolumeFixedUsage(
+ volume, null).getUtilization());
+ }
+
+ @Test
+ void buildVolumeReportProtoReportsZeroUtilizationForZeroCapacityVolume()
+ throws IOException {
+ HddsVolume volume = createVolume("zero-capacity-report", 0, 0);
+
+ assertEquals(0.0, DiskBalancerService.buildVolumeReportProto(
+ Collections.singletonList(
+ DiskBalancerVolumeCalculation.newVolumeFixedUsage(volume, null)))
+ .get(0).getUtilization());
+ }
+
@Test
void getIdealUsageRejectsNegativeCapacity() throws IOException {
HddsVolume negativeCapacityVolume = createVolume(
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerYaml.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerYaml.java
index 9ec501a9a9df..fa4f05b00742 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerYaml.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerYaml.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.container.diskbalancer;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_SCM_DATANODE_DISK_BALANCER_INFO_FILE_DEFAULT;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import java.io.File;
import java.io.IOException;
@@ -101,4 +102,56 @@ public void testReadYamlNullContainerStatesUsesDefault() throws IOException {
DiskBalancerInfo info = DiskBalancerYaml.readDiskBalancerInfoFile(file);
Assertions.assertEquals(DiskBalancerConfiguration.DEFAULT_CONTAINER_STATES, info.getContainerStates());
}
+
+ @ParameterizedTest
+ @MethodSource("invalidDiskBalancerYamlCases")
+ public void testReadYamlRejectsInvalidPersistedInfo(String yaml,
+ String expectedMessage) throws IOException {
+ File file = new File(tmpDir.toString(),
+ OZONE_SCM_DATANODE_DISK_BALANCER_INFO_FILE_DEFAULT);
+ Files.write(file.toPath(), yaml.getBytes(StandardCharsets.UTF_8));
+
+ IOException ex = assertThrows(IOException.class,
+ () -> DiskBalancerYaml.readDiskBalancerInfoFile(file));
+
+ Assertions.assertTrue(ex.getMessage().contains(expectedMessage),
+ () -> "Expected message to contain '" + expectedMessage + "': "
+ + ex.getMessage());
+ }
+
+ public static Stream invalidDiskBalancerYamlCases() {
+ return Stream.of(
+ Arguments.of(validYaml()
+ .replace("version: 1\n", "version: 99\n"),
+ "Unsupported DiskBalancer info version: 99"),
+ Arguments.of(validYaml()
+ .replace("version: 1\n", ""),
+ "DiskBalancer info version is missing"),
+ Arguments.of(validYaml()
+ .replace("operationalState: RUNNING\n", ""),
+ "DiskBalancer operationalState is missing"),
+ Arguments.of(validYaml()
+ .replace("threshold: 10.0\n", "threshold: 0.0\n"),
+ "Invalid DiskBalancer configuration in persisted info"),
+ Arguments.of(validYaml()
+ .replace("bandwidthInMB: 100\n", "bandwidthInMB: 0\n"),
+ "Invalid DiskBalancer configuration in persisted info"),
+ Arguments.of(validYaml()
+ .replace("parallelThread: 5\n", "parallelThread: 0\n"),
+ "Invalid DiskBalancer configuration in persisted info"),
+ Arguments.of(validYaml()
+ .replace("containerStates: CLOSED,QUASI_CLOSED\n",
+ "containerStates: OPEN\n"),
+ "Invalid DiskBalancer configuration in persisted info"));
+ }
+
+ private static String validYaml() {
+ return "operationalState: RUNNING\n"
+ + "threshold: 10.0\n"
+ + "bandwidthInMB: 100\n"
+ + "parallelThread: 5\n"
+ + "stopAfterDiskEven: true\n"
+ + "containerStates: CLOSED,QUASI_CLOSED\n"
+ + "version: 1\n";
+ }
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
index 37728cde4e0d..134770befc0a 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
@@ -23,6 +23,7 @@
import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.buildTestTree;
import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.verifyAllDataChecksumsMatch;
import static org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.CONTAINER_SCHEMA_V3_ENABLED;
+import static org.apache.hadoop.ozone.container.keyvalue.TestContainerCorruptions.MISSING_METADATA_DIR;
import static org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil.isSameSchemaVersion;
import static org.apache.hadoop.ozone.container.replication.CopyContainerCompression.NO_COMPRESSION;
import static org.assertj.core.api.Assertions.assertThat;
@@ -759,6 +760,35 @@ public void testReportOfUnhealthyContainer(
assertNotNull(keyValueContainer.getContainerReport());
}
+ /**
+ * When a container's metadata directory is missing (MISSING_METADATA_DIR detected by the scanner),
+ * markContainerUnhealthy must succeed without throwing. Writing a partial .container file with only
+ * the state field would lose other metadata and is more harmful than writing nothing. The in-memory
+ * UNHEALTHY state is sufficient for SCM to receive it via ICR and schedule deletion.
+ */
+ @ContainerTestVersionInfo.ContainerTest
+ public void testMarkUnhealthyWithMissingMetadataDir(ContainerTestVersionInfo versionInfo) throws Exception {
+ init(versionInfo);
+ keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId);
+
+ // Simulate MISSING_METADATA_DIR using the same corruption helper used in scanner tests.
+ File metadataDir = new File(keyValueContainerData.getMetadataPath());
+ assertTrue(metadataDir.exists(), "Metadata dir should exist before corruption");
+ MISSING_METADATA_DIR.applyTo(keyValueContainer);
+
+ // markContainerUnhealthy must not throw even though the metadata dir is absent.
+ keyValueContainer.markContainerUnhealthy();
+
+ // In-memory state must be UNHEALTHY.
+ assertEquals(ContainerProtos.ContainerDataProto.State.UNHEALTHY,
+ keyValueContainer.getContainerState());
+
+ // Regression guards: if a future change adds mkdirs/persist logic, these catch it early.
+ assertFalse(metadataDir.exists(), "Metadata dir should not be recreated by markContainerUnhealthy");
+ assertFalse(keyValueContainer.getContainerFile().exists(),
+ "Container file should not be written when metadata dir is missing");
+ }
+
@ContainerTestVersionInfo.ContainerTest
public void testUpdateContainer(ContainerTestVersionInfo versionInfo)
throws Exception {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index 4c73d24e1581..2670f1585a25 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -702,6 +702,46 @@ public void testContainerChecksumInvocation(ContainerLayoutVersion layoutVersion
Assertions.assertEquals(1, icrCount.get());
}
+ @ContainerLayoutTestInfo.ContainerTest
+ public void testDeleteUnreferencedFailsWhenChunkDirCannotBeListed(
+ ContainerLayoutVersion layoutVersion) throws Exception {
+ KeyValueHandler keyValueHandler = new KeyValueHandler(conf,
+ DATANODE_UUID, newContainerSet(), mock(MutableVolumeSet.class),
+ mock(ContainerMetrics.class), c -> { },
+ new ContainerChecksumTreeManager(conf));
+ KeyValueContainer container = createContainerWithChunksPath(layoutVersion,
+ Files.createFile(tempDir.resolve("chunks-file")));
+
+ IOException exception = Assertions.assertThrows(IOException.class,
+ () -> keyValueHandler.deleteUnreferenced(container, 1L));
+
+ assertThat(exception)
+ .hasMessageContaining("Failed to list chunks under")
+ .hasMessageContaining("for unreferenced block 1")
+ .hasMessageContaining("in container " + DUMMY_CONTAINER_ID);
+ }
+
+ @ContainerLayoutTestInfo.ContainerTest
+ public void testDeleteUnreferencedFailsWhenFileDeletionFails(
+ ContainerLayoutVersion layoutVersion) throws Exception {
+ FailingUnreferencedDeleteKeyValueHandler keyValueHandler =
+ new FailingUnreferencedDeleteKeyValueHandler(conf);
+ Path chunkDir = Files.createDirectory(tempDir.resolve("chunks"));
+ Path chunkFile = Files.createFile(chunkDir.resolve(
+ getUnreferencedChunkName(layoutVersion, 1L)));
+ KeyValueContainer container =
+ createContainerWithChunksPath(layoutVersion, chunkDir);
+
+ IOException exception = Assertions.assertThrows(IOException.class,
+ () -> keyValueHandler.deleteUnreferenced(container, 1L));
+
+ assertThat(exception)
+ .hasMessageContaining("Failed to delete unreferenced chunk/block")
+ .hasMessageContaining(chunkFile.toString())
+ .hasMessageContaining("in container " + DUMMY_CONTAINER_ID);
+ assertTrue(Files.exists(chunkFile));
+ }
+
@ContainerLayoutTestInfo.ContainerTest
public void testUpdateContainerChecksum(ContainerLayoutVersion layoutVersion) throws Exception {
conf = new OzoneConfiguration();
@@ -1087,4 +1127,39 @@ public void onCompleted() {
ContainerMetrics.remove();
}
}
+
+ private KeyValueContainer createContainerWithChunksPath(
+ ContainerLayoutVersion layoutVersion, Path chunksPath) {
+ KeyValueContainerData data = new KeyValueContainerData(DUMMY_CONTAINER_ID,
+ layoutVersion, GB, PipelineID.randomId().toString(), DATANODE_UUID);
+ data.setChunksPath(chunksPath.toString());
+ return new KeyValueContainer(data, conf);
+ }
+
+ private static String getUnreferencedChunkName(
+ ContainerLayoutVersion layoutVersion, long localID) {
+ switch (layoutVersion) {
+ case FILE_PER_BLOCK:
+ return localID + ".block";
+ case FILE_PER_CHUNK:
+ return localID + "_chunk_0";
+ default:
+ throw new IllegalArgumentException(
+ "Unsupported container layout version " + layoutVersion);
+ }
+ }
+
+ private static final class FailingUnreferencedDeleteKeyValueHandler
+ extends KeyValueHandler {
+ private FailingUnreferencedDeleteKeyValueHandler(OzoneConfiguration conf) {
+ super(conf, DATANODE_UUID, newContainerSet(), mock(MutableVolumeSet.class),
+ mock(ContainerMetrics.class), c -> { },
+ new ContainerChecksumTreeManager(conf));
+ }
+
+ @Override
+ boolean deleteUnreferencedFile(File file) {
+ return false;
+ }
+ }
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScannerTests.java
similarity index 99%
rename from hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java
rename to hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScannerTests.java
index 7bd45c3b503a..88fbd29fccd1 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScannerTests.java
@@ -58,7 +58,7 @@
*/
@MockitoSettings(strictness = Strictness.LENIENT)
@SuppressWarnings("checkstyle:VisibilityModifier")
-public abstract class TestContainerScannersAbstract {
+public abstract class ContainerScannerTests {
private static final AtomicLong CONTAINER_SEQ_ID = new AtomicLong(100);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java
index 535982422545..50968c49d496 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java
@@ -44,6 +44,7 @@
import java.io.IOException;
import java.time.Duration;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Optional;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
@@ -56,12 +57,14 @@
import org.apache.hadoop.hdfs.util.Canceler;
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeWriter;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.ScanResult;
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.hadoop.ozone.container.metadata.DatanodeSchemaThreeDBDefinition;
import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerScanError.FailureType;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -75,7 +78,7 @@
*/
@MockitoSettings(strictness = Strictness.LENIENT)
public class TestBackgroundContainerDataScanner extends
- TestContainerScannersAbstract {
+ ContainerScannerTests {
private BackgroundContainerDataScanner scanner;
@@ -402,4 +405,28 @@ public void testMerkleTreeWritten() throws Exception {
.updateContainerChecksum(eq(container.getContainerData().getContainerID()), any());
}
}
+
+ /**
+ * When data scan reports only "too many open files" errors due to file-descriptor exhaustion,
+ * the container must not be marked UNHEALTHY.
+ */
+ @Test
+ public void testDataScanOnlyTooManyOpenFilesDoesNotMarkUnhealthy() throws Exception {
+ Container container = mockKeyValueContainer();
+ IOException ex = new IOException("Too many open files");
+ DataScanResult scanResult = DataScanResult.fromErrors(Collections.singletonList(
+ new ContainerScanError(FailureType.CORRUPT_CHUNK, new File("."), ex)),
+ new ContainerMerkleTreeWriter());
+ when(container.scanData(any(DataTransferThrottler.class), any(Canceler.class))).thenReturn(scanResult);
+
+ setContainers(container);
+ scanner.runIteration();
+
+ verify(controller, never()).markContainerUnhealthy(anyLong(), any(ScanResult.class));
+ verify(controller, never()).updateContainerChecksum(eq(container.getContainerData().getContainerID()), any());
+ verify(controller, never()).updateDataScanTimestamp(eq(container.getContainerData().getContainerID()), any());
+ assertEquals(1, scanner.getMetrics().getNumScanIterations());
+ assertEquals(0, scanner.getMetrics().getNumContainersScanned());
+ assertEquals(0, scanner.getMetrics().getNumUnHealthyContainers());
+ }
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java
index 9b6c6aed3f05..9d87da355949 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java
@@ -38,8 +38,10 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.io.File;
import java.io.IOException;
import java.time.Duration;
+import java.util.Collections;
import java.util.Optional;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@@ -49,6 +51,7 @@
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.ScanResult;
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerScanError.FailureType;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -61,7 +64,7 @@
*/
@MockitoSettings(strictness = Strictness.LENIENT)
public class TestBackgroundContainerMetadataScanner extends
- TestContainerScannersAbstract {
+ ContainerScannerTests {
private BackgroundContainerMetadataScanner scanner;
@@ -256,4 +259,25 @@ public void testShutdownDuringScan() throws Exception {
// The container should remain healthy.
verifyContainerMarkedUnhealthy(healthy, never());
}
+
+ /**
+ * When metadata scan reports only "too many open files" errors due to file-descriptor exhaustion,
+ * the container must not be marked UNHEALTHY.
+ */
+ @Test
+ public void testMetadataScanOnlyTooManyOpenFilesDoesNotMarkUnhealthy() throws Exception {
+ Container container = mockKeyValueContainer();
+ IOException emf = new IOException("Too many open files");
+ MetadataScanResult scanResult = MetadataScanResult.fromErrors(Collections.singletonList(
+ new ContainerScanError(FailureType.CORRUPT_CONTAINER_FILE, new File("."), emf)));
+ when(container.scanMetaData()).thenReturn(scanResult);
+
+ setContainers(container);
+ scanner.runIteration();
+
+ verify(controller, never()).markContainerUnhealthy(anyLong(), any(ScanResult.class));
+ assertEquals(1, scanner.getMetrics().getNumScanIterations());
+ assertEquals(0, scanner.getMetrics().getNumContainersScanned());
+ assertEquals(0, scanner.getMetrics().getNumUnHealthyContainers());
+ }
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java
index 69b117db1235..5f72471e9d0d 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java
@@ -65,7 +65,7 @@
*/
@MockitoSettings(strictness = Strictness.LENIENT)
public class TestOnDemandContainerScanner extends
- TestContainerScannersAbstract {
+ ContainerScannerTests {
private OnDemandContainerScanner onDemandScanner;
private static final String TEST_SCAN = "Test Scan";
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestScanTransientIOUtil.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestScanTransientIOUtil.java
new file mode 100644
index 000000000000..a92603eb21a1
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestScanTransientIOUtil.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.ozoneimpl;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.FileSystemException;
+import java.util.Arrays;
+import java.util.Collections;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerScanError.FailureType;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Unit tests for {@link ScanTransientIOUtil}.
+ */
+public class TestScanTransientIOUtil {
+
+ @Test
+ public void detectsTooManyOpenFilesInFileSystemException() {
+ assertTrue(ScanTransientIOUtil.isTooManyOpenFiles(new FileSystemException(null, null, "Too many open files")));
+ }
+
+ @Test
+ public void detectsTooManyOpenFilesInFileNotFoundExceptionMessage() {
+ String msg = "/data/container/metadata/16341719.container (Too many open files)";
+ assertTrue(ScanTransientIOUtil.isTooManyOpenFiles(new FileNotFoundException(msg)));
+ }
+
+ @Test
+ public void detectsTooManyOpenFilesInMessageCauseChain() {
+ IOException throwable = new IOException("Too many open files");
+ assertTrue(ScanTransientIOUtil.isTooManyOpenFiles(new IOException(throwable)));
+ }
+
+ @Test
+ public void rejectsUnrelatedIOException() {
+ assertFalse(ScanTransientIOUtil.isTooManyOpenFiles(new IOException("disk full")));
+ }
+
+ @Test
+ public void scanErrorsOnlyTooManyOpenFilesReturnsTrue() {
+ IOException ex = new IOException("Too many open files");
+ MetadataScanResult scanResult = MetadataScanResult.fromErrors(Collections.singletonList(
+ new ContainerScanError(FailureType.CORRUPT_CONTAINER_FILE, new File("."), ex)));
+ assertTrue(ScanTransientIOUtil.scanErrorsAreOnlyTooManyOpenFiles(scanResult));
+ }
+
+ @Test
+ public void scanErrorsMixedReturnsFalse() {
+ IOException ioException = new IOException("Too many open files");
+ FileNotFoundException fileNotFoundException = new FileNotFoundException("missing");
+ MetadataScanResult scanResult = MetadataScanResult.fromErrors(Arrays.asList(
+ new ContainerScanError(FailureType.CORRUPT_CHUNK, new File("."), ioException),
+ new ContainerScanError(FailureType.MISSING_CONTAINER_FILE, new File("."), fileNotFoundException)));
+ assertFalse(ScanTransientIOUtil.scanErrorsAreOnlyTooManyOpenFiles(scanResult));
+ }
+
+ @Test
+ public void emptyScanResult() {
+ assertFalse(ScanTransientIOUtil.scanErrorsAreOnlyTooManyOpenFiles(
+ MetadataScanResult.fromErrors(Collections.emptyList())));
+ }
+}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisorSchedulingBenchmark.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisorSchedulingBenchmark.java
index 1b8c041bacb8..e3d0c4126da1 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisorSchedulingBenchmark.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisorSchedulingBenchmark.java
@@ -17,12 +17,10 @@
package org.apache.hadoop.ozone.container.replication;
-import static org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand.fromSources;
+import static org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand.toTarget;
import static org.assertj.core.api.Assertions.assertThat;
-import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -43,63 +41,34 @@ public class ReplicationSupervisorSchedulingBenchmark {
@Test
public void test() throws InterruptedException {
- List datanodes = new ArrayList<>();
- datanodes.add(MockDatanodeDetails.randomDatanodeDetails());
- datanodes.add(MockDatanodeDetails.randomDatanodeDetails());
+ DatanodeDetails source1 = MockDatanodeDetails.randomDatanodeDetails();
+ DatanodeDetails source2 = MockDatanodeDetails.randomDatanodeDetails();
+ DatanodeDetails target = MockDatanodeDetails.randomDatanodeDetails();
- //locks representing the limited resource of remote and local disks
-
- //datanode -> disk -> lock object (remote resources)
+ //locks representing the limited resource of local disks on the source
final Map> volumeLocks = new HashMap<>();
- //disk -> lock (local resources)
- Map destinationLocks = new HashMap<>();
-
- //init the locks
- for (DatanodeDetails datanode : datanodes) {
- volumeLocks.put(datanode.getID(), new HashMap<>());
+ for (DatanodeDetails dn : new DatanodeDetails[]{source1, source2}) {
+ volumeLocks.put(dn.getID(), new HashMap<>());
for (int i = 0; i < 10; i++) {
- volumeLocks.get(datanode.getID()).put(i, new Object());
+ volumeLocks.get(dn.getID()).put(i, new Object());
}
}
- for (int i = 0; i < 10; i++) {
- destinationLocks.put(i, new Object());
- }
-
- //simplified executor emulating the current sequential download +
- //import.
+ //simplified executor emulating push upload
ContainerReplicator replicator = task -> {
- //download, limited by the number of source datanodes
- final DatanodeDetails sourceDatanode =
- task.getSources().get(random.nextInt(task.getSources().size()));
-
+ //upload, limited by the source datanode's volume
final Map volumes =
- volumeLocks.get(sourceDatanode.getID());
+ volumeLocks.get(source1.getID());
Object volumeLock = volumes.get(random.nextInt(volumes.size()));
synchronized (volumeLock) {
- System.out.println("Downloading " + task.getContainerId() + " from " + sourceDatanode);
+ System.out.println("Uploading " + task.getContainerId() + " to " + task.getTarget());
try {
volumeLock.wait(1000);
} catch (InterruptedException ex) {
throw new IllegalStateException(ex);
}
}
-
- //import, limited by the destination datanode
- final int volumeIndex = random.nextInt(destinationLocks.size());
- Object destinationLock = destinationLocks.get(volumeIndex);
- synchronized (destinationLock) {
- System.out.println(
- "Importing " + task.getContainerId() + " to disk "
- + volumeIndex);
-
- try {
- destinationLock.wait(1000);
- } catch (InterruptedException ex) {
- throw new IllegalStateException(ex);
- }
- }
};
ReplicationSupervisor rs = ReplicationSupervisor.newBuilder().build();
@@ -108,10 +77,7 @@ public void test() throws InterruptedException {
//schedule 100 container replication
for (int i = 0; i < 100; i++) {
- List sources = new ArrayList<>();
- sources.add(datanodes.get(random.nextInt(datanodes.size())));
-
- rs.addTask(new ReplicationTask(fromSources(i, sources), replicator));
+ rs.addTask(new ReplicationTask(toTarget(i, target), replicator));
}
rs.shutdownAfterFinish();
final long executionTime = Time.monotonicNow() - start;
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestCopyContainerResponseStream.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestCopyContainerResponseStream.java
deleted file mode 100644
index 183a00daf3a1..000000000000
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestCopyContainerResponseStream.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.replication;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.io.OutputStream;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.CopyContainerResponseProto;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-
-/**
- * Test for {@link CopyContainerResponseStream}.
- */
-class TestCopyContainerResponseStream
- extends GrpcOutputStreamTest {
-
- TestCopyContainerResponseStream() {
- super(CopyContainerResponseProto.class);
- }
-
- @Override
- protected OutputStream createSubject() {
- return new CopyContainerResponseStream(getObserver(),
- getContainerId(), getBufferSize());
- }
-
- @Override
- protected ByteString verifyPart(CopyContainerResponseProto response,
- int expectedOffset, int size) {
- assertEquals(getContainerId(), response.getContainerID());
- assertEquals(expectedOffset, response.getReadOffset());
- assertEquals(size, response.getLen());
- return response.getData();
- }
-}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestDownloadAndImportReplicator.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestDownloadAndImportReplicator.java
deleted file mode 100644
index c690b50d6425..000000000000
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestDownloadAndImportReplicator.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.replication;
-
-import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyLong;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Semaphore;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.conf.StorageUnit;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
-import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
-import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
-import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
-import org.apache.hadoop.ozone.container.common.volume.VolumeChoosingPolicyFactory;
-import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
-import org.apache.ozone.test.GenericTestUtils;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.Timeout;
-import org.junit.jupiter.api.io.TempDir;
-
-/**
- * Test for DownloadAndImportReplicator.
- */
-@Timeout(300)
-public class TestDownloadAndImportReplicator {
-
- @TempDir
- private File tempDir;
-
- private MutableVolumeSet volumeSet;
- private SimpleContainerDownloader downloader;
- private DownloadAndImportReplicator replicator;
- private long containerMaxSize;
-
- @BeforeEach
- void setup() throws IOException {
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, tempDir.getAbsolutePath());
- VolumeChoosingPolicy volumeChoosingPolicy = VolumeChoosingPolicyFactory.getPolicy(conf);
- ContainerSet containerSet = newContainerSet(0);
- volumeSet = new MutableVolumeSet("test", conf, null,
- StorageVolume.VolumeType.DATA_VOLUME, null);
- ContainerImporter importer = new ContainerImporter(conf, containerSet,
- mock(ContainerController.class), volumeSet, volumeChoosingPolicy);
- downloader = mock(SimpleContainerDownloader.class);
- replicator = new DownloadAndImportReplicator(conf, containerSet, importer,
- downloader);
- containerMaxSize = (long) conf.getStorageSize(
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
- }
-
- @Test
- public void testCommitSpaceReleasedOnReplicationFailure() throws Exception {
- long containerId = 1;
- HddsVolume volume = (HddsVolume) volumeSet.getVolumesList().get(0);
- long initialCommittedBytes = volume.getCommittedBytes();
-
- // Mock downloader to throw exception
- Semaphore semaphore = new Semaphore(1);
- when(downloader.getContainerDataFromReplicas(anyLong(), any(), any(), any()))
- .thenAnswer(invocation -> {
- semaphore.acquire();
- throw new IOException("Download failed");
- });
-
- ReplicationTask task = new ReplicationTask(containerId,
- Collections.singletonList(mock(DatanodeDetails.class)), replicator);
-
- // Acquire semaphore so that container import will pause before downloading.
- semaphore.acquire();
- CompletableFuture.runAsync(() -> {
- assertThrows(IOException.class, () -> replicator.replicate(task));
- });
-
- // Wait such that first container import reserve space
- GenericTestUtils.waitFor(() ->
- volume.getCommittedBytes() > initialCommittedBytes,
- 1000, 50000);
- assertEquals(volume.getCommittedBytes(), initialCommittedBytes + 2 * containerMaxSize);
- semaphore.release();
-
- GenericTestUtils.waitFor(() ->
- volume.getCommittedBytes() == initialCommittedBytes,
- 1000, 50000);
-
- // Verify commit space is released
- assertEquals(initialCommittedBytes, volume.getCommittedBytes());
- }
-}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestGrpcReplicationService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestGrpcReplicationService.java
index 8b831fa06466..079c41fcbecc 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestGrpcReplicationService.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestGrpcReplicationService.java
@@ -21,8 +21,6 @@
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
import static org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand.toTarget;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
@@ -32,7 +30,6 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
@@ -43,8 +40,6 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.CopyContainerRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.CopyContainerResponseProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.security.SecurityConfig;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -59,7 +54,7 @@
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
-import org.apache.ratis.thirdparty.io.grpc.stub.CallStreamObserver;
+import org.apache.hadoop.ozone.container.replication.AbstractReplicationTask.Status;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -155,7 +150,7 @@ public void init() throws Exception {
conf).build());
replicationServer =
- new ReplicationServer(containerController, replicationConfig, secConf,
+ new ReplicationServer(replicationConfig, secConf,
null, importer, datanode.threadNamePrefix());
replicationServer.start();
}
@@ -165,29 +160,6 @@ public void cleanup() {
replicationServer.stop();
}
- @Test
- public void testDownload() throws IOException {
- SimpleContainerDownloader downloader =
- new SimpleContainerDownloader(conf, null);
- Path downloadDir = Files.createDirectory(tempDir.resolve("DownloadDir"));
- Path result = downloader.getContainerDataFromReplicas(
- CONTAINER_ID,
- Collections.singletonList(datanode), downloadDir,
- CopyContainerCompression.NO_COMPRESSION);
-
- assertTrue(result.toString().startsWith(downloadDir.toString()));
-
- File[] files = downloadDir.toFile().listFiles();
-
- assertNotNull(files);
- assertEquals(files.length, 1);
-
- assertTrue(files[0].getName().startsWith("container-" +
- CONTAINER_ID + "-"));
-
- downloader.close();
- }
-
@Test
public void testUpload() {
ContainerReplicationSource source =
@@ -206,8 +178,8 @@ public void testUpload() {
}
@Test
- void closesStreamOnError() {
- // GIVEN
+ void closesStreamOnError() throws Exception {
+ // GIVEN: a source whose copyData always fails
ContainerReplicationSource source = new ContainerReplicationSource() {
@Override
public void prepare(long containerId) {
@@ -220,26 +192,22 @@ public void copyData(long containerId, OutputStream destination,
throw new IOException("testing");
}
};
- ContainerImporter importer = mock(ContainerImporter.class);
- GrpcReplicationService subject =
- new GrpcReplicationService(source, importer);
-
- CopyContainerRequestProto request = CopyContainerRequestProto.newBuilder()
- .setContainerID(1)
- .setReadOffset(0)
- .setLen(123)
- .build();
- CallStreamObserver observer =
- mock(CallStreamObserver.class);
- when(observer.isReady()).thenReturn(true);
+
+ OutputStream uploadStream = mock(OutputStream.class);
+ ContainerUploader uploader = mock(ContainerUploader.class);
+ when(uploader.startUpload(anyLong(), any(), any(), any()))
+ .thenReturn(uploadStream);
+
+ PushReplicator pushReplicator = new PushReplicator(conf, source, uploader);
+ ReplicationTask task = new ReplicationTask(
+ toTarget(CONTAINER_ID, datanode), pushReplicator);
// WHEN
- subject.download(request, observer);
+ pushReplicator.replicate(task);
- // THEN
- // onCompleted is called by GrpcOutputStream#close
- // so we indirectly verify that the stream is closed
- verify(observer).onCompleted();
+ // THEN: task is failed and the upload stream was closed despite the error
+ assertEquals(Status.FAILED, task.getStatus());
+ verify(uploadStream).close();
}
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestMeasuredReplicator.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestMeasuredReplicator.java
index 25a12be03b98..f0423bcc0fc4 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestMeasuredReplicator.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestMeasuredReplicator.java
@@ -17,12 +17,14 @@
package org.apache.hadoop.ozone.container.replication;
-import static org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand.forTest;
+import static org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand.toTarget;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.ozone.container.replication.AbstractReplicationTask.Status;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -33,6 +35,9 @@
*/
public class TestMeasuredReplicator {
+ private static final DatanodeDetails TARGET =
+ MockDatanodeDetails.randomDatanodeDetails();
+
private MeasuredReplicator measuredReplicator;
private ContainerReplicator replicator;
@@ -64,9 +69,9 @@ public void closeReplicator() throws Exception {
@Test
public void measureFailureSuccessAndBytes() {
//WHEN
- measuredReplicator.replicate(new ReplicationTask(forTest(1), replicator));
- measuredReplicator.replicate(new ReplicationTask(forTest(2), replicator));
- measuredReplicator.replicate(new ReplicationTask(forTest(3), replicator));
+ measuredReplicator.replicate(new ReplicationTask(toTarget(1, TARGET), replicator));
+ measuredReplicator.replicate(new ReplicationTask(toTarget(2, TARGET), replicator));
+ measuredReplicator.replicate(new ReplicationTask(toTarget(3, TARGET), replicator));
//THEN
//even containers should be failed
@@ -84,9 +89,9 @@ public void measureFailureSuccessAndBytes() {
public void testReplicationTime() throws Exception {
//WHEN
//will wait at least the 300ms
- measuredReplicator.replicate(new ReplicationTask(forTest(101), replicator));
- measuredReplicator.replicate(new ReplicationTask(forTest(201), replicator));
- measuredReplicator.replicate(new ReplicationTask(forTest(300), replicator));
+ measuredReplicator.replicate(new ReplicationTask(toTarget(101, TARGET), replicator));
+ measuredReplicator.replicate(new ReplicationTask(toTarget(201, TARGET), replicator));
+ measuredReplicator.replicate(new ReplicationTask(toTarget(300, TARGET), replicator));
//THEN
//even containers should be failed
@@ -104,7 +109,7 @@ public void testReplicationTime() throws Exception {
public void testFailureTimeSuccessExcluded() {
//WHEN
//will wait at least the 15ms
- measuredReplicator.replicate(new ReplicationTask(forTest(15), replicator));
+ measuredReplicator.replicate(new ReplicationTask(toTarget(15, TARGET), replicator));
//THEN
@@ -116,7 +121,7 @@ public void testFailureTimeSuccessExcluded() {
public void testSuccessTimeFailureExcluded() {
//WHEN
//will wait at least the 10ms
- measuredReplicator.replicate(new ReplicationTask(forTest(10), replicator));
+ measuredReplicator.replicate(new ReplicationTask(toTarget(10, TARGET), replicator));
//THEN
@@ -127,7 +132,7 @@ public void testSuccessTimeFailureExcluded() {
@Test
public void testReplicationQueueTimeMetrics() {
final Instant queued = Instant.now().minus(1, ChronoUnit.SECONDS);
- ReplicationTask task = new ReplicationTask(forTest(100), replicator) {
+ ReplicationTask task = new ReplicationTask(toTarget(100, TARGET), replicator) {
@Override
public Instant getQueued() {
return queued;
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationConfig.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationConfig.java
index 67c6eda84aa0..f1f182f40a3e 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationConfig.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationConfig.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.ozone.container.replication;
import static org.apache.hadoop.ozone.container.replication.ReplicationServer.ReplicationConfig.OUTOFSERVICE_FACTOR_DEFAULT;
+import static org.apache.hadoop.ozone.container.replication.ReplicationServer.ReplicationConfig.OUTOFSERVICE_FACTOR_MAX;
+import static org.apache.hadoop.ozone.container.replication.ReplicationServer.ReplicationConfig.OUTOFSERVICE_FACTOR_MIN;
import static org.apache.hadoop.ozone.container.replication.ReplicationServer.ReplicationConfig.REPLICATION_MAX_STREAMS_DEFAULT;
import static org.apache.hadoop.ozone.container.replication.ReplicationServer.ReplicationConfig.REPLICATION_OUTOFSERVICE_FACTOR_KEY;
import static org.apache.hadoop.ozone.container.replication.ReplicationServer.ReplicationConfig.REPLICATION_STREAMS_LIMIT_KEY;
@@ -52,14 +54,11 @@ public void acceptsValidValues() {
}
@Test
- public void overridesInvalidValues() {
+ public void overridesInvalidReplicationLimit() {
// GIVEN
int invalidReplicationLimit = -5;
- double invalidOutOfServiceFactor = 0.5;
OzoneConfiguration conf = new OzoneConfiguration();
conf.setInt(REPLICATION_STREAMS_LIMIT_KEY, invalidReplicationLimit);
- conf.setDouble(REPLICATION_OUTOFSERVICE_FACTOR_KEY,
- invalidOutOfServiceFactor);
// WHEN
ReplicationConfig subject = conf.getObject(ReplicationConfig.class);
@@ -67,7 +66,62 @@ public void overridesInvalidValues() {
// THEN
assertEquals(REPLICATION_MAX_STREAMS_DEFAULT,
subject.getReplicationMaxStreams());
- assertEquals(OUTOFSERVICE_FACTOR_DEFAULT,
+ }
+
+ @Test
+ public void clampsOutOfServiceFactorBelowMinToMin() {
+ // GIVEN
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setDouble(REPLICATION_OUTOFSERVICE_FACTOR_KEY,
+ OUTOFSERVICE_FACTOR_MIN - 0.5);
+
+ // WHEN
+ ReplicationConfig subject = conf.getObject(ReplicationConfig.class);
+
+ // THEN
+ assertEquals(OUTOFSERVICE_FACTOR_MIN,
+ subject.getOutOfServiceFactor(), 0.001);
+ }
+
+ @Test
+ public void clampsOutOfServiceFactorAboveMaxToMax() {
+ // GIVEN
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setDouble(REPLICATION_OUTOFSERVICE_FACTOR_KEY,
+ OUTOFSERVICE_FACTOR_MAX + 10);
+
+ // WHEN
+ ReplicationConfig subject = conf.getObject(ReplicationConfig.class);
+
+ // THEN
+ assertEquals(OUTOFSERVICE_FACTOR_MAX,
+ subject.getOutOfServiceFactor(), 0.001);
+ }
+
+ @Test
+ public void acceptsOutOfServiceFactorBoundaryValues() {
+ // GIVEN
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setDouble(REPLICATION_OUTOFSERVICE_FACTOR_KEY,
+ OUTOFSERVICE_FACTOR_MIN);
+
+ // WHEN
+ ReplicationConfig subject = conf.getObject(ReplicationConfig.class);
+
+ // THEN
+ assertEquals(OUTOFSERVICE_FACTOR_MIN,
+ subject.getOutOfServiceFactor(), 0.001);
+
+ // GIVEN
+ conf = new OzoneConfiguration();
+ conf.setDouble(REPLICATION_OUTOFSERVICE_FACTOR_KEY,
+ OUTOFSERVICE_FACTOR_MAX);
+
+ // WHEN
+ subject = conf.getObject(ReplicationConfig.class);
+
+ // THEN
+ assertEquals(OUTOFSERVICE_FACTOR_MAX,
subject.getOutOfServiceFactor(), 0.001);
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
index abfef6fbffdc..026ac6ac76ad 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
@@ -20,6 +20,7 @@
import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_MAINTENANCE;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
@@ -27,13 +28,11 @@
import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ReplicationCommandPriority.NORMAL;
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
import static org.apache.hadoop.ozone.container.replication.AbstractReplicationTask.Status.DONE;
-import static org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand.fromSources;
-import static org.assertj.core.api.Assertions.assertThat;
+import static org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand.toTarget;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.anyList;
import static org.mockito.Mockito.anyLong;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
@@ -42,13 +41,9 @@
import com.google.protobuf.UnsafeByteOperations;
import jakarta.annotation.Nonnull;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.time.Clock;
import java.time.Instant;
import java.time.ZoneId;
@@ -62,44 +57,28 @@
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.Semaphore;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
-import org.apache.commons.compress.archivers.ArchiveOutputStream;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ReplicationCommandPriority;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.security.symmetric.SecretKeySignerClient;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.metrics2.impl.MetricsCollectorImpl;
import org.apache.hadoop.ozone.container.checksum.DNContainerOperationClient;
import org.apache.hadoop.ozone.container.checksum.ReconcileContainerTask;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.impl.ContainerData;
-import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
-import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
-import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
-import org.apache.hadoop.ozone.container.common.volume.VolumeChoosingPolicyFactory;
import org.apache.hadoop.ozone.container.ec.reconstruction.ECReconstructionCommandInfo;
import org.apache.hadoop.ozone.container.ec.reconstruction.ECReconstructionCoordinator;
import org.apache.hadoop.ozone.container.ec.reconstruction.ECReconstructionCoordinatorTask;
@@ -112,8 +91,7 @@
import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
import org.apache.ozone.test.GenericTestUtils;
-import org.apache.ozone.test.GenericTestUtils.LogCapturer;
-import org.apache.ozone.test.TestClock;
+import org.apache.ozone.test.MockClock;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.io.TempDir;
@@ -148,16 +126,14 @@ public class TestReplicationSupervisor {
private ContainerLayoutVersion layoutVersion;
private StateContext context;
- private TestClock clock;
+ private MockClock clock;
private DatanodeDetails datanode;
private DNContainerOperationClient mockClient;
private ContainerController mockController;
- private VolumeChoosingPolicy volumeChoosingPolicy;
-
@BeforeEach
public void setUp() throws Exception {
- clock = new TestClock(Instant.now(), ZoneId.systemDefault());
+ clock = new MockClock(Instant.now(), ZoneId.systemDefault());
set = newContainerSet();
DatanodeStateMachine stateMachine = mock(DatanodeStateMachine.class);
context = new StateContext(
@@ -169,7 +145,6 @@ public void setUp() throws Exception {
mockClient = mock(DNContainerOperationClient.class);
mockController = mock(ContainerController.class);
when(stateMachine.getDatanodeDetails()).thenReturn(datanode);
- volumeChoosingPolicy = VolumeChoosingPolicyFactory.getPolicy(new OzoneConfiguration());
}
@AfterEach
@@ -262,7 +237,7 @@ public void failureHandling(ContainerLayoutVersion layout) {
}
@ContainerLayoutTestInfo.ContainerTest
- public void stalledDownload() {
+ public void stalledReplication() {
// GIVEN
ReplicationSupervisor supervisor = supervisorWith(__ -> noopReplicator,
new DiscardingExecutorService());
@@ -292,7 +267,7 @@ public void stalledDownload() {
}
@ContainerLayoutTestInfo.ContainerTest
- public void slowDownload() {
+ public void slowReplication() {
// GIVEN
ReplicationSupervisor supervisor = supervisorWith(__ -> slowReplicator,
new ThreadPoolExecutor(1, 1, 60, TimeUnit.SECONDS,
@@ -320,170 +295,41 @@ public void slowDownload() {
}
@ContainerLayoutTestInfo.ContainerTest
- public void testDownloadAndImportReplicatorFailure(ContainerLayoutVersion layout,
- @TempDir File tempFile) throws IOException {
- OzoneConfiguration conf = new OzoneConfiguration();
-
- ReplicationSupervisor supervisor = ReplicationSupervisor.newBuilder()
- .stateContext(context)
- .executor(newDirectExecutorService())
- .clock(clock)
- .build();
-
- // Mock to fetch an exception in the importContainer method.
- SimpleContainerDownloader moc =
- mock(SimpleContainerDownloader.class);
- Path res = Paths.get("file:/tmp/no-such-file");
- when(
- moc.getContainerDataFromReplicas(anyLong(), anyList(),
- any(Path.class), any()))
- .thenReturn(res);
-
- final String testDir = tempFile.getPath();
- MutableVolumeSet volumeSet = mock(MutableVolumeSet.class);
- when(volumeSet.getVolumesList())
- .thenReturn(singletonList(
- new HddsVolume.Builder(testDir).conf(conf).build()));
- ContainerController mockedCC =
- mock(ContainerController.class);
- ContainerImporter importer =
- new ContainerImporter(conf, set, mockedCC, volumeSet, volumeChoosingPolicy);
- ContainerReplicator replicator =
- new DownloadAndImportReplicator(conf, set, importer, moc);
-
- replicatorRef.set(replicator);
-
- LogCapturer logCapturer = LogCapturer.captureLogs(DownloadAndImportReplicator.class);
-
- supervisor.addTask(createTask(1L));
- assertEquals(1, supervisor.getReplicationFailureCount());
- assertEquals(0, supervisor.getReplicationSuccessCount());
- assertThat(logCapturer.getOutput())
- .contains("Container 1 replication was unsuccessful.");
- }
-
- @ContainerLayoutTestInfo.ContainerTest
- public void testReplicationImportReserveSpace(ContainerLayoutVersion layout)
- throws IOException, InterruptedException, TimeoutException {
- final long containerUsedSize = 100;
+ public void testPushReplicatorTargetReturnsError(ContainerLayoutVersion layout) throws IOException {
this.layoutVersion = layout;
+ // GIVEN a real PushReplicator wired to an uploader whose remote side rejects the push
+ // (e.g. the target datanode reports it has no space to import the container).
OzoneConfiguration conf = new OzoneConfiguration();
- conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, tempDir.getAbsolutePath());
-
- long containerMaxSize = (long) conf.getStorageSize(
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
-
ReplicationSupervisor supervisor = ReplicationSupervisor.newBuilder()
.stateContext(context)
.executor(newDirectExecutorService())
.clock(clock)
.build();
- MutableVolumeSet volumeSet = new MutableVolumeSet(datanode.getUuidString(), conf, null,
- StorageVolume.VolumeType.DATA_VOLUME, null);
-
- long containerId = 1;
- // create container
- KeyValueContainerData containerData = new KeyValueContainerData(containerId,
- ContainerLayoutVersion.FILE_PER_BLOCK, containerMaxSize, "test", "test");
- HddsVolume vol1 = (HddsVolume) volumeSet.getVolumesList().get(0);
- containerData.setVolume(vol1);
- // the container is not yet in HDDS, so only set its own size, leaving HddsVolume with used=0
- containerData.getStatistics().updateWrite(100, false);
- KeyValueContainer container = new KeyValueContainer(containerData, conf);
- ContainerController controllerMock = mock(ContainerController.class);
- Semaphore semaphore = new Semaphore(1);
- when(controllerMock.importContainer(any(), any(), any()))
- .thenAnswer((invocation) -> {
- semaphore.acquire();
- return container;
+ ContainerReplicationSource source = mock(ContainerReplicationSource.class);
+ ContainerUploader uploader = mock(ContainerUploader.class);
+ // Have the uploader's startUpload immediately complete the future exceptionally,
+ // simulating the target returning an error before any data is transferred.
+ when(uploader.startUpload(anyLong(), any(), any(), any()))
+ .thenAnswer(invocation -> {
+ CompletableFuture fut = invocation.getArgument(2);
+ fut.completeExceptionally(
+ new IOException("No space left on target datanode"));
+ return new ByteArrayOutputStream();
});
-
- File tarFile = containerTarFile(containerId, containerData);
-
- SimpleContainerDownloader moc =
- mock(SimpleContainerDownloader.class);
- when(
- moc.getContainerDataFromReplicas(anyLong(), anyList(),
- any(Path.class), any()))
- .thenReturn(tarFile.toPath());
-
- ContainerImporter importer =
- new ContainerImporter(conf, set, controllerMock, volumeSet, volumeChoosingPolicy);
-
- // Initially volume has 0 commit space
- assertEquals(0, vol1.getCommittedBytes());
- long usedSpace = vol1.getCurrentUsage().getUsedSpace();
- // Initially volume has 0 used space
- assertEquals(0, usedSpace);
- // Increase committed bytes so that volume has only remaining 3 times container size space
- long minFreeSpace =
- conf.getObject(DatanodeConfiguration.class).getHardLimitMinFreeSpace(vol1.getCurrentUsage().getCapacity());
- long initialCommittedBytes = vol1.getCurrentUsage().getCapacity() - containerMaxSize * 3 - minFreeSpace;
- vol1.incCommittedBytes(initialCommittedBytes);
- ContainerReplicator replicator =
- new DownloadAndImportReplicator(conf, set, importer, moc);
- replicatorRef.set(replicator);
-
- LogCapturer logCapturer = LogCapturer.captureLogs(DownloadAndImportReplicator.class);
-
- // Acquire semaphore so that container import will pause after reserving space.
- semaphore.acquire();
- CompletableFuture.runAsync(() -> {
- try {
- supervisor.addTask(createTask(containerId));
- } catch (Exception ex) {
- }
- });
-
- // Wait such that first container import reserve space
- GenericTestUtils.waitFor(() ->
- vol1.getCommittedBytes() > initialCommittedBytes,
- 1000, 50000);
-
- // Volume has reserved space of 2 * containerSize
- assertEquals(vol1.getCommittedBytes(), initialCommittedBytes + 2 * containerMaxSize);
- // Container 2 import will fail as container 1 has reserved space and no space left to import new container
- // New container import requires at least (2 * container size)
- long containerId2 = 2;
- supervisor.addTask(createTask(containerId2));
- GenericTestUtils.waitFor(() -> 1 == supervisor.getReplicationFailureCount(),
- 1000, 50000);
- assertThat(logCapturer.getOutput()).contains("No volumes have enough space for a new container");
- // Release semaphore so that first container import will pass
- semaphore.release();
- GenericTestUtils.waitFor(() ->
- 1 == supervisor.getReplicationSuccessCount(), 1000, 50000);
-
- usedSpace = vol1.getCurrentUsage().getUsedSpace();
- // After replication, volume used space should be increased by container used bytes
- assertEquals(containerUsedSize, usedSpace);
-
- // Volume committed bytes used for replication has been released, no need to reserve space for imported container
- // only closed container gets replicated, so no new data will be written it
- assertEquals(vol1.getCommittedBytes(), initialCommittedBytes);
- }
+ replicatorRef.set(new PushReplicator(conf, source, uploader));
- private File containerTarFile(
- long containerId, ContainerData containerData) throws IOException {
- File yamlFile = new File(tempDir, "container.yaml");
- ContainerDataYaml.createContainerFile(containerData,
- yamlFile);
- File tarFile = new File(tempDir,
- ContainerUtils.getContainerTarName(containerId));
- try (OutputStream output = Files.newOutputStream(tarFile.toPath())) {
- ArchiveOutputStream archive = new TarArchiveOutputStream(output);
- TarArchiveEntry entry = archive.createArchiveEntry(yamlFile,
- "container.yaml");
- archive.putArchiveEntry(entry);
- try (InputStream input = Files.newInputStream(yamlFile.toPath())) {
- IOUtils.copy(input, archive);
- }
- archive.closeArchiveEntry();
- }
- return tarFile;
+ // WHEN
+ ReplicationTask task = createTask(1L);
+ supervisor.addTask(task);
+
+ // THEN the supervisor records a clean failure.
+ assertEquals(1, supervisor.getReplicationRequestCount());
+ assertEquals(0, supervisor.getReplicationSuccessCount());
+ assertEquals(1, supervisor.getReplicationFailureCount());
+ assertEquals(0, supervisor.getTotalInFlightReplications());
+ assertEquals(ReplicationTask.Status.FAILED, task.getStatus());
}
@ContainerLayoutTestInfo.ContainerTest
@@ -527,15 +373,12 @@ public void testDatanodeOutOfService(ContainerLayoutVersion layout) {
datanode.setPersistedOpState(
HddsProtos.NodeOperationalState.DECOMMISSIONING);
- ReplicateContainerCommand pushCmd = ReplicateContainerCommand.toTarget(
- 1, MockDatanodeDetails.randomDatanodeDetails());
- pushCmd.setTerm(CURRENT_TERM);
- ReplicateContainerCommand pullCmd = createCommand(2);
+ // Push tasks always run regardless of the local datanode's operational state.
+ ReplicateContainerCommand cmd = createCommand(2);
- supervisor.addTask(new ReplicationTask(pushCmd, replicatorRef.get()));
- supervisor.addTask(new ReplicationTask(pullCmd, replicatorRef.get()));
+ supervisor.addTask(new ReplicationTask(cmd, replicatorRef.get()));
- assertEquals(2, supervisor.getReplicationRequestCount());
+ assertEquals(1, supervisor.getReplicationRequestCount());
assertEquals(1, supervisor.getReplicationSuccessCount());
assertEquals(0, supervisor.getReplicationFailureCount());
assertEquals(0, supervisor.getTotalInFlightReplications());
@@ -559,10 +402,8 @@ public void taskWithObsoleteTermIsDropped(ContainerLayoutVersion layout) {
}
@ContainerLayoutTestInfo.ContainerTest
- public void testMultipleReplication(ContainerLayoutVersion layout,
- @TempDir File tempFile) throws IOException {
+ public void testMultipleReplication(ContainerLayoutVersion layout) throws IOException {
this.layoutVersion = layout;
- OzoneConfiguration conf = new OzoneConfiguration();
// GIVEN
ReplicationSupervisor replicationSupervisor =
supervisorWithReplicator(FakeReplicator::new);
@@ -579,20 +420,7 @@ public void testMultipleReplication(ContainerLayoutVersion layout,
replicationSupervisor.addTask(createTask(3L));
ecReconstructionSupervisor.addTask(createECTaskWithCoordinator(4L));
- SimpleContainerDownloader moc = mock(SimpleContainerDownloader.class);
- Path res = Paths.get("file:/tmp/no-such-file");
- when(moc.getContainerDataFromReplicas(anyLong(), anyList(),
- any(Path.class), any())).thenReturn(res);
-
- final String testDir = tempFile.getPath();
- MutableVolumeSet volumeSet = mock(MutableVolumeSet.class);
- when(volumeSet.getVolumesList()).thenReturn(singletonList(
- new HddsVolume.Builder(testDir).conf(conf).build()));
- ContainerController mockedCC = mock(ContainerController.class);
- ContainerImporter importer = new ContainerImporter(conf, set, mockedCC, volumeSet, volumeChoosingPolicy);
- ContainerReplicator replicator = new DownloadAndImportReplicator(
- conf, set, importer, moc);
- replicatorRef.set(replicator);
+ replicatorRef.set(throwingReplicator);
replicationSupervisor.addTask(createTask(5L));
ReplicateContainerCommand cmd1 = createCommand(6L);
@@ -971,9 +799,9 @@ private ECReconstructionCoordinatorTask createECTaskWithCoordinator(long contain
ecReconstructionCommandInfo);
}
- private static ReplicateContainerCommand createCommand(long containerId) {
+ private ReplicateContainerCommand createCommand(long containerId) {
ReplicateContainerCommand cmd =
- ReplicateContainerCommand.forTest(containerId);
+ ReplicateContainerCommand.toTarget(containerId, datanode);
cmd.setTerm(CURRENT_TERM);
return cmd;
}
@@ -1139,6 +967,37 @@ public void poolSizeCanBeDecreased() {
}
}
+ @ContainerLayoutTestInfo.ContainerTest
+ public void poolSizeCanBeUpdatedByReplicationStreamsLimitReconfiguration() {
+ final int replicationMaxStreams = 5;
+ ReplicationServer.ReplicationConfig repConf =
+ new ReplicationServer.ReplicationConfig();
+ repConf.setReplicationMaxStreams(replicationMaxStreams);
+
+ AtomicInteger threadPoolSize = new AtomicInteger();
+
+ ReplicationSupervisor rs = ReplicationSupervisor.newBuilder()
+ .executor(new DiscardingExecutorService())
+ .executorThreadUpdater(threadPoolSize::set)
+ .replicationConfig(repConf)
+ .build();
+
+ rs.nodeStateUpdated(IN_SERVICE);
+ assertEquals(replicationMaxStreams, threadPoolSize.get());
+
+ rs.setReplicationMaxStreams(7);
+ assertEquals(7, threadPoolSize.get());
+
+ rs.nodeStateUpdated(DECOMMISSIONING);
+ assertEquals(repConf.scaleOutOfServiceLimit(7), threadPoolSize.get());
+
+ rs.setReplicationMaxStreams(3);
+ assertEquals(repConf.scaleOutOfServiceLimit(3), threadPoolSize.get());
+
+ rs.nodeStateUpdated(IN_SERVICE);
+ assertEquals(3, threadPoolSize.get());
+ }
+
@ContainerLayoutTestInfo.ContainerTest
public void testMaxQueueSize() {
List datanodes = new ArrayList<>();
@@ -1189,9 +1048,8 @@ public void testMaxQueueSize() {
private void scheduleTasks(
List datanodes, ReplicationSupervisor rs) {
for (int i = 0; i < 10; i++) {
- List sources =
- singletonList(datanodes.get(i % datanodes.size()));
- rs.addTask(new ReplicationTask(fromSources(i, sources), noopReplicator));
+ DatanodeDetails target = datanodes.get(i % datanodes.size());
+ rs.addTask(new ReplicationTask(toTarget(i, target), noopReplicator));
}
}
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestSendContainerRequestHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestSendContainerRequestHandler.java
index 4fb801532f0b..6a7bfa1063eb 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestSendContainerRequestHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestSendContainerRequestHandler.java
@@ -48,6 +48,7 @@
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
+import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
import org.junit.jupiter.api.BeforeEach;
@@ -104,6 +105,30 @@ public static Stream sizeProvider() {
);
}
+ @Test
+ void testNoSpaceOnTargetVolume() throws Exception {
+ long containerId = 1;
+
+ // Simulate the target datanode having no volume with enough space to
+ // import the incoming container by having the volume chooser throw.
+ DiskOutOfSpaceException noSpace =
+ new DiskOutOfSpaceException("No volumes have enough space for a new container");
+ doThrow(noSpace).when(importer).chooseNextVolume(anyLong());
+
+ doAnswer(invocation -> {
+ Object arg = invocation.getArgument(0);
+ assertEquals(noSpace, arg);
+ return null;
+ }).when(responseObserver).onError(any());
+
+ sendContainerRequestHandler.onNext(createRequest(containerId,
+ ByteString.copyFromUtf8("test"), 0, null));
+
+ // No volume was reserved, so no committed bytes should change on any volume.
+ HddsVolume volume = (HddsVolume) volumeSet.getVolumesList().get(0);
+ assertEquals(0, volume.getCommittedBytes());
+ }
+
@Test
void testReceiveDataForExistingContainer() throws Exception {
long containerId = 1;
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestSimpleContainerDownloader.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestSimpleContainerDownloader.java
deleted file mode 100644
index 076fb17c711b..000000000000
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestSimpleContainerDownloader.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.replication;
-
-import static org.apache.hadoop.ozone.container.replication.CopyContainerCompression.NO_COMPRESSION;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicReference;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-
-/**
- * Test SimpleContainerDownloader.
- */
-public class TestSimpleContainerDownloader {
-
- @TempDir
- private Path tempDir;
-
- @Test
- public void testGetContainerDataFromReplicasHappyPath() throws Exception {
-
- //GIVEN
- List datanodes = createDatanodes();
- TestingContainerDownloader downloader =
- TestingContainerDownloader.successful();
-
- //WHEN
- Path result = downloader.getContainerDataFromReplicas(1L, datanodes,
- tempDir, NO_COMPRESSION);
-
- //THEN
- assertEquals(datanodes.get(0).getUuidString(),
- result.toString());
- downloader.verifyAllClientsClosed();
- }
-
- @Test
- public void testGetContainerDataFromReplicasDirectFailure()
- throws Exception {
-
- //GIVEN
- List datanodes = createDatanodes();
-
- TestingContainerDownloader downloader =
- TestingContainerDownloader.immediateFailureFor(datanodes.get(0));
-
- //WHEN
- final Path result =
- downloader.getContainerDataFromReplicas(1L, datanodes,
- tempDir, NO_COMPRESSION);
-
- //THEN
- //first datanode is failed, second worked
- assertEquals(datanodes.get(1).getUuidString(),
- result.toString());
- downloader.verifyAllClientsClosed();
- }
-
- @Test
- public void testGetContainerDataFromReplicasAsyncFailure() throws Exception {
-
- //GIVEN
- List datanodes = createDatanodes();
-
- TestingContainerDownloader downloader =
- TestingContainerDownloader.delayedFailureFor(datanodes.get(0));
-
- //WHEN
- final Path result =
- downloader.getContainerDataFromReplicas(1L, datanodes,
- tempDir, NO_COMPRESSION);
-
- //THEN
- //first datanode is failed, second worked
- assertEquals(datanodes.get(1).getUuidString(),
- result.toString());
- downloader.verifyAllClientsClosed();
- }
-
- /**
- * Test if different datanode is used for each download attempt.
- */
- @Test
- public void testRandomSelection() throws Exception {
-
- //GIVEN
- final List datanodes = createDatanodes();
-
- TestingContainerDownloader downloader =
- TestingContainerDownloader.randomOrder();
-
- //WHEN executed, THEN at least once the second datanode should be
- //returned.
- for (int i = 0; i < 10000; i++) {
- Path path = downloader.getContainerDataFromReplicas(1L, datanodes,
- tempDir, NO_COMPRESSION);
- if (path.toString().equals(datanodes.get(1).getUuidString())) {
- return;
- }
- }
-
- //there is 1/3^10_000 chance for false positive, which is practically 0.
- fail(
- "Datanodes are selected 10000 times but second datanode was never "
- + "used.");
- downloader.verifyAllClientsClosed();
- }
-
- private List createDatanodes() {
- List datanodes = new ArrayList<>();
- datanodes.add(MockDatanodeDetails.randomDatanodeDetails());
- datanodes.add(MockDatanodeDetails.randomDatanodeDetails());
- datanodes.add(MockDatanodeDetails.randomDatanodeDetails());
- return datanodes;
- }
-
- private static final class TestingContainerDownloader
- extends SimpleContainerDownloader {
-
- private final List failedDatanodes;
- private final boolean disableShuffle;
- private final boolean directException;
- private final List clients = new LinkedList<>();
-
- private final AtomicReference datanodeRef =
- new AtomicReference<>();
-
- static TestingContainerDownloader randomOrder() {
- return new TestingContainerDownloader(false, false);
- }
-
- static TestingContainerDownloader successful() {
- return new TestingContainerDownloader(true, false);
- }
-
- static TestingContainerDownloader immediateFailureFor(
- DatanodeDetails... failedDatanodes) {
- return new TestingContainerDownloader(true, true, failedDatanodes);
- }
-
- static TestingContainerDownloader delayedFailureFor(
- DatanodeDetails... failedDatanodes) {
- return new TestingContainerDownloader(true, false, failedDatanodes);
- }
-
- /**
- * Creates downloader which fails with datanodes in the arguments.
- *
- * @param directException if false the exception will be wrapped in the
- * returning future.
- */
- private TestingContainerDownloader(
- boolean disableShuffle, boolean directException,
- DatanodeDetails... failedDatanodes) {
- super(new OzoneConfiguration(), null);
- this.disableShuffle = disableShuffle;
- this.directException = directException;
- this.failedDatanodes = Arrays.asList(failedDatanodes);
- }
-
- @Override
- protected List shuffleDatanodes(
- List sourceDatanodes
- ) {
- return disableShuffle ? sourceDatanodes //turn off randomization
- : super.shuffleDatanodes(sourceDatanodes);
- }
-
- @Override
- protected GrpcReplicationClient createReplicationClient(
- DatanodeDetails datanode, CopyContainerCompression compression) {
- datanodeRef.set(datanode);
- GrpcReplicationClient client = mock(GrpcReplicationClient.class);
- clients.add(client);
- return client;
- }
-
- @Override
- protected CompletableFuture downloadContainer(
- GrpcReplicationClient client,
- long containerId, Path downloadPath) {
-
- DatanodeDetails datanode = datanodeRef.get();
- assertNotNull(datanode);
-
- if (failedDatanodes.contains(datanode)) {
- if (directException) {
- throw new RuntimeException("Unavailable datanode");
- } else {
- return CompletableFuture.supplyAsync(() -> {
- throw new RuntimeException("Unavailable datanode");
- });
- }
- } else {
-
- //path includes the dn id to make it possible to assert.
- return CompletableFuture.completedFuture(
- Paths.get(datanode.getUuidString()));
- }
-
- }
-
- private void verifyAllClientsClosed() throws Exception {
- for (GrpcReplicationClient each : clients) {
- verify(each).close();
- }
- }
- }
-}
diff --git a/hadoop-hdds/crypto-api/pom.xml b/hadoop-hdds/crypto-api/pom.xml
index 801c7b0d036c..554da40f2eac 100644
--- a/hadoop-hdds/crypto-api/pom.xml
+++ b/hadoop-hdds/crypto-api/pom.xml
@@ -17,11 +17,11 @@
org.apache.ozone
hdds
- 2.2.0-SNAPSHOT
+ 2.3.0-SNAPSHOT
hdds-crypto-api
- 2.2.0-SNAPSHOT
+ 2.3.0-SNAPSHOT
Apache Ozone HDDS Crypto
Apache Ozone Distributed Data Store cryptographic functions
diff --git a/hadoop-hdds/crypto-default/pom.xml b/hadoop-hdds/crypto-default/pom.xml
index 49e7065476ef..92ee1a16e0ad 100644
--- a/hadoop-hdds/crypto-default/pom.xml
+++ b/hadoop-hdds/crypto-default/pom.xml
@@ -17,11 +17,11 @@
org.apache.ozone
hdds
- 2.2.0-SNAPSHOT
+ 2.3.0-SNAPSHOT
hdds-crypto-default
- 2.2.0-SNAPSHOT
+ 2.3.0-SNAPSHOT
Apache Ozone HDDS Crypto - Default
Default implementation of Apache Ozone Distributed Data Store's cryptographic functions
diff --git a/hadoop-hdds/docs/content/design/diskbalancer.md b/hadoop-hdds/docs/content/design/diskbalancer.md
index 6b1edefc62de..05ed92e77be6 100644
--- a/hadoop-hdds/docs/content/design/diskbalancer.md
+++ b/hadoop-hdds/docs/content/design/diskbalancer.md
@@ -69,7 +69,7 @@ Administrators use the `ozone admin datanode diskbalancer` CLI to manage and mon
- Update configuration parameters
- Query DiskBalancer status and volume density reports
* Each datanode performs its own **authentication** (via RPC) and **authorization** checks (using `OzoneAdmins` based on `ozone.administrators` configuration).
-* For batch operations, clients can use the `--in-service-datanodes` flag to automatically query SCM for all IN_SERVICE datanodes and execute commands on all of them.
+* For batch operations, clients can use the `--in-service-datanodes` flag to automatically query SCM for all IN_SERVICE and HEALTHY datanodes and execute commands on all of them.
**DN - DiskBalancer Service:**
@@ -81,7 +81,7 @@ A daemon thread, the **Scheduler**, runs periodically on each Datanode.
from the most over-utilized disk (source) to the least utilized disk (destination).
3. The scheduler dispatches these move tasks to a pool of **Worker** threads for parallel execution.
-**Note:** SCM is used **only** for datanode discovery when using the `--in-service-datanodes` flag. SCM provides a list of IN_SERVICE datanodes for batch operations but
+**Note:** SCM is used **only** for datanode discovery when using the `--in-service-datanodes` flag. SCM provides a list of IN_SERVICE and HEALTHY datanodes for batch operations but
does **not** participate in DiskBalancer control operations (start/stop/update/status/report). All DiskBalancer operations are performed directly between client and datanode.
## Container Move Process
@@ -165,7 +165,7 @@ This ensures DiskBalancer respects datanode lifecycle management and does not in
## Feature Flag
-The DiskBalancer feature is gated behind a feature flag (`hdds.datanode.disk.balancer.enabled`) to allow controlled rollout. By default, the feature is disabled. When disabled, the DiskBalancer service is not initialized on datanodes, and the CLI commands are hidden from the main help output to prevent accidental usage.
+The DiskBalancer feature is gated behind a feature flag (`hdds.datanode.disk.balancer.enabled`) to allow controlled rollout. By default, the feature is enabled. When disabled, the DiskBalancer service is not initialized on datanodes, and the CLI commands refuse to run until the flag is set back to true.
## DiskBalancer Metrics
diff --git a/hadoop-hdds/docs/content/design/efficient-snapdiff.md b/hadoop-hdds/docs/content/design/efficient-snapdiff.md
new file mode 100644
index 000000000000..6f646c124747
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/efficient-snapdiff.md
@@ -0,0 +1,254 @@
+---
+title: Snapshot Diff Optimization
+summary: Describe proposal for an optimized snapshot diff that uses mostly sequential reads and batch puts
+date: 2025-05-22
+jira: HDDS-9154
+status: draft
+author: Saketa Chalamchala
+---
+
+
+## 1. Introduction
+This document outlines the technical design, architectural choices, and algorithmic improvements to optimize Ozone's Snapshot Diff feature. The design addresses performance bottlenecks in both the **Full Diff** and **DAG-based Diff** paths. The primary goals are to reduce random I/O, minimize CPU overhead from deserialization, and streamline the classification of differences.
+
+ ## Goals
+ - Reduce random I/O.
+ - Minimize CPU cost of deserializing KeyInfo and DirectoryInfo for comparisons.
+ - Keep baseline diff semantics for CREATE/DELETE/RENAME/MODIFY where possible.
+
+---
+
+## 2. Core Design Choices & Optimizations
+
+### 2.1. Sequential Reads & Table Iterators
+**Baseline Issue:** Baseline full diff enumerates keys via SST readers (plus per-key `db.get` lookups), and the DAG-based diff relies heavily on random point lookups (`db.get()`) against the snapshot RocksDB instances to fetch the old and new states of keys identified in the delta SST files. For buckets with millions of keys, this random I/O degrades performance and thrashes the OS page cache.
+**Optimized Design:** The optimization shifts mostly to sequential reads. For the Full Diff path, it uses native RocksDB **Table Iterators** to scan the entire `directoryTable` and `fileTable` sequentially. For the DAG-based path, it uses a **K-way Merge Iterator** over the delta SST files to sequentially extract the latest visible versions without needing to query the main snapshot DBs. This sequential I/O pattern maximizes disk throughput and cache efficiency.
+
+### 2.2. Lightweight Parsing
+**Baseline Issue:** The baseline implementation fully deserializes `OmKeyInfo` and `OmDirectoryInfo` protobuf messages to compare objects, which is extremely CPU and memory intensive when scanning millions of keys.
+**Optimized Design:** Introduces a lightweight `SnapshotDiffValueParser` that reads the raw protobuf byte stream directly. It extracts only the required fields (like `updateID`, `parentID`, `name` and compare signature fields) without instantiating full Java objects. It dynamically builds a compare signature by hashing only meaningful fields (content-change: latest block layout, size, `fileChecksum` and metadata-change: ACLs, metadata, tags), skipping volatile fields like `modificationTime` or `creationTime` to identify modified entries.
+
+#### Pseudo-code: Selective Parsing and Signature
+```java
+ParsedObjectInfo parseRequiredKeyInfo(byte[] raw, boolean meaningfulOnly) {
+ ParsedObjectInfo parsed = new ParsedObjectInfo();
+ CodedInputStream input = CodedInputStream.newInstance(raw);
+ while (!input.isAtEnd()) {
+ int tag = input.readTag();
+ switch (WireFormat.getTagFieldNumber(tag)) {
+ case KEYINFO_OBJECT_ID_FIELD:
+ parsed.setObjectId(input.readUInt64());
+ break;
+ case KEYINFO_PARENT_ID_FIELD:
+ parsed.setParentId(input.readUInt64());
+ break;
+ case KEYINFO_KEY_NAME_FIELD:
+ parsed.setName(input.readString());
+ break;
+ case KEYINFO_UPDATE_ID_FIELD:
+ parsed.setUpdateId(input.readUInt64());
+ break;
+ default:
+ input.skipField(tag);
+ break;
+ }
+ }
+ return parsed;
+}
+
+ParsedObjectInfo parseSignatureKeyInfo(byte[] raw, boolean meaningfulOnly) {
+ ParsedObjectInfo parsed = new ParsedObjectInfo();
+ CodedInputStream input = CodedInputStream.newInstance(raw);
+ while (!input.isAtEnd()) {
+ int tag = input.readTag();
+ switch (WireFormat.getTagFieldNumber(tag)) {
+ case KEYINFO_METADATA_FIELD:
+ case KEYINFO_ACLS_FIELD:
+ case KEYINFO_TAGS_FIELD:
+ case KEYINFO_FILE_CHECKSUM_FIELD:
+ updateSignature(tag, input, parsed);
+ break;
+ case KEYINFO_BLOCK_LOCATIONS_FIELD:
+ updateSignature(extractLatestBlockInfo(tag, input), parsed);
+ default:
+ input.skipField(tag);
+ break;
+ }
+ }
+ return parsed;
+}
+```
+
+### 2.3. Sequence/UpdateID Gating
+**Baseline Issue:** The baseline performs full object comparisons including timestamps to detect modifications, which is susceptible to clock skew and is computationally expensive.
+**Optimized Design:** Use snapshot-specific gates that align with the transactional guarantees of the deployment mode.
+- **Full diff (w/ OM HA only):** `updateID > fromSnapshot.lastTransactionInfo.txIndex`. This compares two OM/Ratis log indices.
+- **DAG diff:** Extend raw SST iterators to expose internal sequence numbers, gate with `entry.sequence > fromSnapshot.dbTxSequenceNumber`.
+
+### 2.4. Deferred Classification & Path Resolution
+**Baseline Issue:** Baseline builds the diff key set first and then classifies entries during `generateDiffReport`, which requires resolving paths for all candidates. This causes unnecessary path lookups for entries that might ultimately be ignored.
+**Optimized Design:** Diff classification is strictly deferred to the final **Merge Join** stage. Path resolution is also deferred until an entry is definitively classified as a diff. This prevents wasting I/O and CPU on resolving paths for entries that might ultimately be ignored or unchanged.
+
+### 2.5. Batch Puts to Snapshot Diff DB
+**Baseline Issue:** Writing intermediate lists and final diff reports often relies on individual RocksDB `put` operations, incurring high JNI overhead.
+**Optimized Design:** The design advocates for using RocksDB `WriteBatch` operations. By batching writes to the `snap-diff-report-table` and intermediate `PersistentList`/`PersistentMap` structures, we significantly improve write throughput and reduce disk sync overhead.
+
+### 2.6. Delete Report Consistency
+**Baseline Issue:** With baseline full diff, deleting a directory emits `DELETE` entries for the directory but reports sub-directories and sub-files inconsistently depending on how far deep cleaning of the `toSnapshot` progressed. In DAG-based diff, only the deleted directory and any sub-directory/sub-file that was explicitly deleted before the top-level directory are reported. For the same snapshots, diff output can vary based on timing (before vs after deep cleaning) or mode (full diff vs DAG-based diff).
+**Optimized Design:** Only top level deleted directories are reported. This keeps diff results stable regardless of snapshot deep cleaning and which diff path was used.
+
+### 2.7. Dependency Ordered Reporting
+**Baseline Issue:** With baselines, diff report entries are ordered by diff type, `DELETES` are reported first followed by `RENAMES, CREATES, MODIFIES` in order. When the report is replayed this order does not safely cover all scenarios.
+
+For example,
+* Snapshot 1 has file `A/B` and directory `C`.
+* Snapshot 2 renames `A/B` to `C/B` and deletes directory `A`.
+* The diff entries are `RENAME A/B -> C/B` and `DELETE A`.
+If deletes are replayed first, `A/B` is removed before the rename and the rename fails. The correct replay order is `RENAME A/B -> C/B` followed by `DELETE A`.
+
+**Optimized Design:** Ensure the report can be replayed safely by ordering entries based on their dependencies rather than their diff type.
+
+**Dependency Rules:**
+1. Parents must appear before children for `CREATE/RENAME/MODIFY`.
+2. Children must appear before parents for `DELETE`.
+3. If a rename or create targets a path that is being deleted, the delete must come first.
+4. If a rename frees a source path that is re-created in the same diff, the rename must come first.
+
+**Building the dependency graph:**
+- Each diff entry becomes a node in a directed graph.
+- Add edges using the rules above:
+ - For hierarchy ordering, add edges from parent to child for `CREATE/RENAME/MODIFY`.
+ - For deletes, use the same parent-child edges but emit them in reverse order later.
+ - For path conflicts, add edges from the delete node to the rename/create node that reuses the deleted path, and from rename to create if the rename frees a path that is re-created.
+
+**Emitting entries using the graph:**
+- Run Kahn's algorithm on the graph to produce a topological order for `CREATE/RENAME/MODIFY`.
+- Emit all `CREATE/RENAME/MODIFY` entries in that order (parents before children, and conflict edges respected).
+- Emit `DELETE` entries in reverse topological order (children before parents) so deletes do not remove parents before their children.
+
+**Note on OBS Buckets:** Since OBS buckets lack a directory hierarchy, dependency ordering simplifies to path-conflict rules (Rules 3 and 4), ensuring renames and deletes occur in the correct sequence to avoid collisions or missing sources.
+
+---
+
+## 3. Data Structures and Algorithms
+
+- **oldList/newList maps**: `PersistentMap` keyed by `objectId`, storing `EntryValue` (`parentId`, `name`, `isDir`, `signature`).
+- **Directory path lookup**:
+ - **Persisted BFS**: RocksDB CFs for edges storing `(parentID, objectID) -> name` and resolved paths `objectID -> fullPath`, with an LRU cache for hot path lookups.
+- **DiffCandidateSet**: `Set/Set` captured by snapshot-specific gating rules mentioned in Section 2.3
+- **SHA-256 Hashing:** Used to generate compact, fixed-size compare signatures for object metadata.
+- **Delete retention sets (full diff only):** `deletedDirSet` and `deletedRootSet` to suppress redundant deletes.
+- **Dependency ordering graph:** adjacency list of `objectId -> children`, in-degree map, and a queue of zero in-degree nodes for Kahn's algorithm.
+- **Raw SST iterators**: `ManagedRawSSTFileIterator` yielding `(userKey, sequence, type, value)` tuples including tombstones used during DAG based diff delta SST scan.
+- **K-way merge heap**: Min-heap ordered by `(userKey ASC, sequence DESC)` to dedupe to the latest visible version per userKey. It guarantees $O(N \log K)$ time complexity for $N$ keys across $K$ SST files, ensuring sequential disk I/O.
+
+---
+
+## 4. Optimized DAG-Based Diff Implementation Stages
+
+The DAG-based diff optimizes the process by only looking at SST files that changed between snapshots. It identifies the set of SST files that differ between `fromSnapshot` and `toSnapshot` using the `RocksDBCheckpointDiffer` (compaction DAG).
+
+### Stage 1: Sequential Read Flow + Batched Point Lookups + Directory Scans
+**Baseline Issue:** Baseline reads these delta files and then performs random reads against the snapshot DBs to find the old/new state of the keys, causing severe I/O bottlenecks.
+**Optimized Design (in order):**
+1. **Sequential scan of `toSnapshot` diff SSTs:** Use native iterators (`ManagedRawSSTFileIterator`) and a K-way merge to scan the delta SSTs **only in `toSnapshot`**. This yields the latest visible versions for changed keys and populates `newList` (for non-tombstones) plus the `DiffCandidateSet` (all tombstones + all keys with `entry.sequence > fromSnapshot.dbTxSequenceNumber`).
+2. **Full table scan of `toSnapshot.directoryTable` (FSO only):** Use `tableIterator` to scan all directory entries sequentially and populate `jobId-to-edges`.
+3. **Full table scan of `fromSnapshot.directoryTable` (FSO only):** Use `tableIterator` to scan all directory entries sequentially.
+ * Populate `jobId-from-edges` with `(parentId, objectId) -> name`.
+ * For directory objectIds that are in the `DiffCandidateSet`, populate `oldList` (build signatures using the value read from the table).
+4. **Batch point lookups of `fromSnapshot.file/keyTable`:** Use `multiGet` for keys in the `DiffCandidateSet` that correspond to files and populate `oldList`.
+
+### Stage 2: Merge Join & Classification
+A synchronized sequential iteration (merge join) is performed over the `oldList` and `newList` based on `objectID`. Since `oldList` and `newList` are backed by RocksDB the iteration is ordered by the key `objectID`.
+* **Only in `newList`** → `CREATE`
+* **Only in `oldList`** → `DELETE`
+* **In both lists**:
+ * If `parentId` or `name` differs → `RENAME`
+ * If signatures differ → `MODIFY`
+ * Else → ignore
+
+### Stage 3: Deferred BFS with Early Stop + Dependency ordering + Final Write
+1. **Run persisted BFS (FSO only)** to resolve paths only for diff entries:
+ * Resolve CREATE + RENAME paths from `jobId-to-edges`.
+ * Resolve RENAME + MODIFY + DELETE paths from `jobId-from-edges`.
+ * Stop once all diff entries are resolved or the entire directory tree is traversed.
+ * Remove entries with unresolvable paths from diff lists
+3. **Write dependency ordered report to table**
+ * Build a dependency graph described in Section 2.7 using `parentId` for resolved entries.
+ * Write the topologically sorted report to reportTable.
+
+---
+
+## 5. Optimized Full Diff Implementation Stages
+
+The Full Diff path is used when compaction DAGs are unavailable or a full recalculation is forced.
+
+### Stage 1: Sequential Table Scanning & Filtering
+Instead of random lookups, the optimization uses native RocksDB **Table Iterators** to sequentially scan the `directoryTable` and `fileTable` of both snapshots while deferring path resolution until after classification.
+
+**1. `toSnapshot` Directory Scan (FSO only):**
+* Iterates sequentially through the `toSnapshot`'s `directoryTable`.
+* Extracts `updateID` using the lightweight parser. If `updateID <= fromSnapshot.lastTransactionInfo.txIndex`, the entry is unchanged (not created/renamed/modified) and is skipped. Otherwise, its compare signature is built and it is added to the `newList` and recorded in the `DiffCandidateSet`.
+* **Graph Construction:** Regardless of whether the entry is a candidate, the `parentID` and `name` are extracted to build the foundational edges of the `toSnapshot` directory structure graph. This is done by writing `(parentID, objectID) -> name` entries into a temporary RocksDB Column Family (`jobId-to-edges`).
+
+**2. `fromSnapshot` Directory Scan (FSO only):**
+* Iterates sequentially through the `fromSnapshot`'s `directoryTable`.
+* Only processes entries whose `objectID` is in `DiffCandidateSet` during the `toSnapshot` scan. Adds these to the `oldList`.
+* **Graph Construction:** Extracts `parentID` and `name` for all entries to build the `fromSnapshot` directory structure graph by writing to another temporary Column Family (`jobId-from-edges`).
+
+**3. `toSnapshot` Key Scan:**
+* Iterates sequentially through the `toSnapshot`'s `key/fileTable`.
+* Applies the same `updateID` gating logic: skips if `updateID <= fromSnapshot.lastTransactionInfo.txIndex`.
+* Builds the compare signature and adds to `newList`, recording these entries in the `DiffCandidateSet`. No parentID/path checks are performed at this stage.
+
+**4. `fromSnapshot` Key Scan:**
+* Iterates sequentially through the `fromSnapshot`'s `key/fileTable`.
+* Only builds compare signature for entries whose `objectID` was marked in `DiffCandidateSet` during the `toSnapshot` file scan. Adds these to the `oldList`.
+
+
+### Stage 2: Merge Join & Classification
+Same as Stage 2 of DAG based diff implementation.
+
+
+### Stage 3: Top level delete retention (FSO only)
+After merge join,
+* Build `deletedDirSet` for deleted directories.
+* Compute `deletedRootSet` by removing any directory whose parent is also deleted
+* Only report delete entries for the directories in `deletedRootSet`
+
+
+### Stage 4: Deferred BFS with Early Stop + Dependency ordering + Final Write
+Same as Stage 3 of DAG based diff implementation.
+
+---
+
+## 6. Comparison with Baseline & Trade-offs
+
+| Feature | Baseline Implementation | Optimized Implementation |
+| :--- | :--- |:--------------------------------------------------------------|
+| **Object Parsing** | Full Protobuf Deserialization (Heavy CPU/GC). | `SnapshotDiffValueParser` (Lightweight byte-stream parsing). |
+| **Modification Detection** | Full object equality. | Key `sequence`/`updateID` gating + selective field hashing. |
+| **DAG Diff I/O Pattern** | Random point lookups (`db.get()`) for delta keys. | Sequential reads with K-way merge of SST files. |
+| **Classification Timing** | During report generation. | Deferred until merge join. |
+| **Path Resolution** | During report generation for all candidates. | Deferred to diff entries only. |
+| **Delete Handling** | Emits deletes of descendants inconsistently. | Retains only top level directory deletes, dependency ordered. |
+| **Report Ordering** | Naive ordering based on Diff Type. | Dependency ordered with Kahn's algorithm. |
+
+### Trade-offs
+1. **Reliance on `updateID` in full diff:** The optimized snapdiff's speed in Full Diff relies heavily on `updateID`. If Ozone has bugs where `updateID` is not bumped during a meaningful metadata change (e.g., parent directory `modificationTime` updates during a child rename), the optimization will miss the modification. Baseline catches this via full comparison, albeit much slower.
+2. **K-way Merge Memory Overhead:** While the DAG optimization drastically reduces random I/O, maintaining a Priority Queue for K-way merging requires slightly more active memory and CPU comparison logic than simple iteration, though this is vastly outweighed by the I/O savings.
+3. **Signature Collisions:** Hash-based comparison assumes no SHA-256 collisions. While statistically negligible, baseline's exact object equality has zero collision risk.
+4. **Dependency Ordering Overhead:** Building and topologically sorting the dependency graph adds some CPU and memory overhead, especially for large delete sets.
+
+## 7. Conclusion
+The optimized implementation represents a shift from a compute-and-I/O-heavy approach to a streamlined, sequential, and deferred-evaluation model. By utilizing `SnapshotDiffValueParser` and entry `sequence`/`updateID` gating, CPU cycles and Garbage Collection pauses are drastically reduced. By replacing random reads in the DAG diff with a sequential K-way merge, disk I/O bottlenecks are eliminated. Deferred path resolution, batch RocksDB puts, and dependency ordered output ensure that resources are only spent on actual differences and replay remains consistent. Despite trade-offs around `updateID` reliance and graph ordering overhead, the optimization provides a scalable and accurate snapshot diff engine suitable for massive buckets.
diff --git a/hadoop-hdds/docs/content/design/ozone-sts.md b/hadoop-hdds/docs/content/design/ozone-sts.md
index fc335dc4de85..6cc94eadd4bc 100644
--- a/hadoop-hdds/docs/content/design/ozone-sts.md
+++ b/hadoop-hdds/docs/content/design/ozone-sts.md
@@ -81,7 +81,7 @@ subset of its capabilities. The restrictions are outlined below:
- The only supported prefix in ResourceArn is `arn:aws:s3:::` - all others will be rejected. **Note**: a ResourceArn
of `*` is supported as well.
-- The only supported Condition operator is `StringEquals` - all others will be rejected.
+- The only supported Condition operators are `StringEquals` and `StringLike` - all others will be rejected.
- The only supported Condition key is `s3:prefix` - all others will be rejected.
- Only one Condition operator per Statement is supported - a Statement with more than one Condition will be rejected.
- The only supported Effect is `Allow` - all others will be rejected.
@@ -135,9 +135,9 @@ to make S3 API calls
- sessionPolicy - when using the RangerOzoneAuthorizer, if Ranger successfully authorizes the AssumeRole call,
it will return a String representing the role the token was authorized for. Furthermore, if an AWS IAM Session Policy
was included with the AssumeRole request, the String return value will also include resources (i.e. buckets, keys, etc.)
-and permissions (i.e. ACLType) corresponding to the AWS IAM Session Policy. These resources and permissions, if present,
-would further limit the scope of the permissions and resources granted by the role in Ranger, such that the temporary
-credential will have the permissions comprising the intersection of the role permissions and the sessionPolicy permissions.
+, permissions (i.e. ACLType - for legacy purposes), and actions (i.e. GetObject, GetObjectTagging, etc.) corresponding to the AWS IAM Session Policy. These resources, permissions and actions, if present,
+would further limit the scope of the permissions, resources and actions granted by the role in Ranger, such that the temporary
+credential will have the permissions and actions comprising the intersection of the role permissions and actions and the sessionPolicy permissions and actions.
- HMAC-SHA256 signature - used to ensure the sessionToken was created by Ozone and was not altered since it was created.
- expiration time of the token (via `ShortLivedTokenIdentifier#getExpiry()`)
- UUID of the OzoneManager secret key used to sign the sessionToken and encrypt the secretAccessKey (via `ShortLivedTokenIdentifier#getSecretKeyId()`)
@@ -189,14 +189,24 @@ components:
The grants parameter is optional, and would only be present if the AssumeRole API call had an IAM session policy JSON
parameter supplied. A conversion utility, `IamSessionPolicyResolver` will process the IAM policy and convert it to a
`Set`, in effect translating from S3 nomenclature for resources and actions to Ozone nomenclature of
-`IOzoneObj` and `ACLType`. Ranger would use all of this information to determine if the AssumeRole call should be
+`IOzoneObj`, `ACLType` and actions without the s3: prefix (such as GetObject or PutObject). Ranger would use all of this information to determine if the AssumeRole call should be
successfully authorized, and if so, it will return a String representation of the granted permissions and paths.
The format of this String is entirely up to the Ranger team. What is required from the Ozone side is to supply this String to Ranger when any
subsequent S3 API calls are made that use STS tokens. In order to achieve this, the sessionPolicy String from Ranger will
be included in the sessionToken response to the AssumeRole API call (as mentioned above), and Ozone will supply this String
to Ranger whenever STS tokens are used on S3 API calls via a new `RequestContext.sessionPolicy` field in the
-`IAccessAuthorizer#checkAccess(IOzoneObj, RequestContext)` call.
+`IAccessAuthorizer#checkAccess(IOzoneObj, RequestContext)` call. Another requirement from the Ozone side is to pass the action (without the s3: prefix) corresponding to the S3 api call into the `RequestContext.s3Action` field.
+
+### 3.6.2 Additional Context on Permissions and Actions
+
+In a prior iteration of this design, only permissions corresponding to Ozone `ACLType` (i.e. read, write, create, read_acl, etc.) were included in Ranger roles and session policies.
+However, after testing against AWS, it was found that ACLs used by Ozone and Ranger are not granular enough. For example, read on volume, read on bucket, and write on key can be used by either the S3 PutObjectTagging api (requiring `s3:PutObjectTagging` action) or the S3 DeleteObjectTagging api (requiring `s3:DeleteObjectTagging` action).
+Similarly, because the S3 PutObject api (`s3:PutObject` action) requires read on volume, read on bucket, and create and write on key, someone with `s3:PutObject` access could previously also call the S3 PutObjectTagging api, even though they did not have access to the `s3:PutObjectTagging` action (as an example).
+AWS does not allow an STS token that is restricted for one action to issue calls to an api that is associated with a different action. To prevent having more access than requested (or different access than requested), ACL permissions can be constrained further by S3 actions.
+
+To do this constraining, the `RequestContext.s3Action` field is introduced so that if populated, the RangerOzoneAuthorizer would further restrict the permissions according to the action.
+Additionally, the OzoneGrant would contain a Set representing the S3 actions that are allowed for an inline policy. If all actions are allowed, then the Set would be empty or null.
## 3.7 Overall Flow
diff --git a/hadoop-hdds/docs/content/feature/ContainerBalancer.md b/hadoop-hdds/docs/content/feature/ContainerBalancer.md
index 7faa99b0e1ec..848e1b998ff4 100644
--- a/hadoop-hdds/docs/content/feature/ContainerBalancer.md
+++ b/hadoop-hdds/docs/content/feature/ContainerBalancer.md
@@ -53,10 +53,10 @@ ozone admin containerbalancer start [options]
|-------------------------------------------------------| -------------------------------------------------------------------------------------------------------------------------------------- |
| `-t`, `--threshold` | The percentage deviation from the average utilization of the cluster after which a datanode will be rebalanced. Default is 10%. |
| `-i`, `--iterations` | The maximum number of consecutive iterations the balancer will run for. Default is 10. Use -1 for infinite iterations. |
-| `-d`, `--maxDatanodesPercentageToInvolvePerIteration` | The maximum percentage of healthy, in-service datanodes that can be involved in balancing in one iteration. Default is 20%. |
-| `-s`, `--maxSizeToMovePerIterationInGB` | The maximum size of data in GB to be moved in one iteration. Default is 500GB. |
-| `-e`, `--maxSizeEnteringTargetInGB` | The maximum size in GB that can enter a target datanode in one iteration. Default is 26GB. |
-| `-l`, `--maxSizeLeavingSourceInGB` | The maximum size in GB that can leave a source datanode in one iteration. Default is 26GB. |
+| `-d`, `--max-datanodes-percentage-to-involve-per-iteration` | The maximum percentage of healthy, in-service datanodes that can be involved in balancing in one iteration. Default is 20%. |
+| `-s`, `--max-size-to-move-per-iteration-in-gb` | The maximum size of data in GB to be moved in one iteration. Default is 500GB. |
+| `-e`, `--max-size-entering-target-in-gb` | The maximum size in GB that can enter a target datanode in one iteration. Default is 26GB. |
+| `-l`, `--max-size-leaving-source-in-gb` | The maximum size in GB that can leave a source datanode in one iteration. Default is 26GB. |
| `--balancing-iteration-interval-minutes` | The interval in minutes between each iteration of the Container Balancer. Default is 70 minutes. |
| `--move-timeout-minutes` | The time in minutes to allow a single container to move from source to target. Default is 65 minutes. |
| `--move-replication-timeout-minutes` | The time in minutes to allow a single container's replication from source to target as part of a container move. Default is 50 minutes. |
diff --git a/hadoop-hdds/docs/content/feature/DiskBalancer.md b/hadoop-hdds/docs/content/feature/DiskBalancer.md
index a022c6968433..b88f1710028b 100644
--- a/hadoop-hdds/docs/content/feature/DiskBalancer.md
+++ b/hadoop-hdds/docs/content/feature/DiskBalancer.md
@@ -43,9 +43,9 @@ A disk is considered a candidate for balancing if its
## Feature Flag
-The Disk Balancer feature is introduced with a feature flag. By default, this feature is disabled.
+The Disk Balancer feature is introduced with a feature flag. By default, this feature is enabled.
-The feature can be **enabled** by setting the following property to `true` in the `ozone-site.xml` configuration file:
+The feature can be **disabled** by setting the following property to `false` in the `ozone-site.xml` configuration file:
`hdds.datanode.disk.balancer.enabled = false`
### Authentication and Authorization
@@ -119,8 +119,8 @@ restart the datanode service for the changes to take effect.
## Command Line Usage
The DiskBalancer is managed through the `ozone admin datanode diskbalancer` command.
-**Note:** This command is hidden from the main help message (`ozone admin datanode --help`). This is because the feature
-is currently considered experimental and is disabled by default. The command is, however, fully functional for those who wish to enable and use the feature.
+**Note:** DiskBalancer is enabled by default on datanodes. Use `hdds.datanode.disk.balancer.enabled=false` in
+`ozone-site.xml` to disable the service on datanodes and prevent CLI commands from running.
### Command Syntax
@@ -154,13 +154,13 @@ ozone admin datanode diskbalancer report [ ...] [--in-service-
| Option | Description | Example |
|-------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------|
| `` | One or more datanode addresses as positional arguments. Addresses can be:
- Hostname (e.g., `DN-1`) - uses default CLIENT_RPC port (19864)
- Hostname with port (e.g., `DN-1:19864`)
- IP address (e.g., `192.168.1.10`)
- IP address with port (e.g., `192.168.1.10:19864`)
- Stdin (`-`) - reads datanode addresses from standard input, one per line | `DN-1`
`DN-1:19864`
`192.168.1.10`
`-` |
-| `--in-service-datanodes` | It queries SCM for all IN_SERVICE datanodes and executes the command on all of them. | `--in-service-datanodes` |
+| `--in-service-datanodes` | It queries SCM for all IN_SERVICE and HEALTHY datanodes and executes the command on all of them. | `--in-service-datanodes` |
| `--json` | Format output as JSON. | `--json` |
| `-t/--threshold-percentage` | Volume density threshold percentage (default: 10.0). Used with `start` and `update` commands. | `-t 5`
`--threshold-percentage 5.0` |
| `-b/--bandwidth-in-mb` | Maximum disk bandwidth in MB/s (default: 10). Used with `start` and `update` commands. | `-b 20`
`--bandwidth-in-mb 50` |
-| `-p/--parallel-thread` | Number of parallel threads (default: 1). Used with `start` and `update` commands. | `-p 5`
`--parallel-thread 10` |
+| `-p/--parallel-thread` | Number of parallel threads (default: 5). Used with `start` and `update` commands. | `-p 5`
`--parallel-thread 10` |
| `-s/--stop-after-disk-even` | Stop automatically after disks are balanced (default: true). Used with `start` and `update` commands. | `-s false`
`--stop-after-disk-even true` |
-| `-c/--container-states` | Comma-separated container lifecycle state names that may be moved between disks . Used with `start` and `update` commands. | `-c CLOSED,QUASI_CLOSED`
`--container-states OPEN,CLOSED` |
+| `-c/--container-states` | Comma-separated container lifecycle state names that may be moved between disks. Used with `start` and `update` commands. | `-c CLOSED,QUASI_CLOSED`
`--container-states CLOSED` |
### Examples
@@ -169,7 +169,7 @@ ozone admin datanode diskbalancer report [ ...] [--in-service-
# Start DiskBalancer on multiple datanodes
ozone admin datanode diskbalancer start DN-1 DN-2 DN-3
-# Start DiskBalancer on all IN_SERVICE datanodes
+# Start DiskBalancer on all IN_SERVICE and HEALTHY datanodes
ozone admin datanode diskbalancer start --in-service-datanodes
# Start DiskBalancer with configuration parameters
@@ -189,7 +189,7 @@ ozone admin datanode diskbalancer start DN-1 --json
# Stop DiskBalancer on multiple datanodes
ozone admin datanode diskbalancer stop DN-1 DN-2 DN-3
-# Stop DiskBalancer on all IN_SERVICE datanodes
+# Stop DiskBalancer on all IN_SERVICE and HEALTHY datanodes
ozone admin datanode diskbalancer stop --in-service-datanodes
# Stop DiskBalancer with json output
@@ -202,7 +202,7 @@ ozone admin datanode diskbalancer stop DN-1 --json
# Update multiple parameters
ozone admin datanode diskbalancer update DN-1 -t 5 -b 50 -p 10
-# Update on all IN_SERVICE datanodes
+# Update on all IN_SERVICE and HEALTHY datanodes
ozone admin datanode diskbalancer update --in-service-datanodes -t 5
# Or using the long form:
ozone admin datanode diskbalancer update --in-service-datanodes --threshold-percentage 5
@@ -216,7 +216,7 @@ ozone admin datanode diskbalancer update DN-1 -b 50 --json
# Get status from multiple datanodes
ozone admin datanode diskbalancer status DN-1 DN-2 DN-3
-# Get status from all IN_SERVICE datanodes
+# Get status from all IN_SERVICE and HEALTHY datanodes
ozone admin datanode diskbalancer status --in-service-datanodes
# Get status as JSON
@@ -228,7 +228,7 @@ ozone admin datanode diskbalancer status --in-service-datanodes --json
# Get report from multiple datanodes
ozone admin datanode diskbalancer report DN-1 DN-2 DN-3
-# Get report from all IN_SERVICE datanodes
+# Get report from all IN_SERVICE and HEALTHY datanodes
ozone admin datanode diskbalancer report --in-service-datanodes
# Get report as JSON
@@ -241,14 +241,14 @@ The DiskBalancer's behavior can be controlled using the following configuration
| Property | Default Value | Description |
|----------------------------------------------------------------|----------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `hdds.datanode.disk.balancer.enabled` | `false` | If false, the DiskBalancer service on the Datanode is disabled. Configure it to true for diskBalancer to be enabled. |
+| `hdds.datanode.disk.balancer.enabled` | `true` | If false, the DiskBalancer service on the Datanode is disabled. By default, DiskBalancer is enabled on datanodes. |
| `hdds.datanode.disk.balancer.volume.density.threshold.percent` | `10.0` | A percentage (0-100). A datanode is considered balanced if for each volume, its utilization differs from the average datanode utilization by no more than this threshold. |
| `hdds.datanode.disk.balancer.max.disk.throughputInMBPerSec` | `10` | The maximum bandwidth (in MB/s) that the balancer can use for moving data, to avoid impacting client I/O. |
| `hdds.datanode.disk.balancer.parallel.thread` | `5` | The number of worker threads to use for moving containers in parallel. |
| `hdds.datanode.disk.balancer.service.interval` | `60s` | The time interval at which the Datanode DiskBalancer service checks for imbalance and updates its configuration. |
| `hdds.datanode.disk.balancer.stop.after.disk.even` | `true` | If true, the DiskBalancer will automatically stop its balancing activity once disks are considered balanced (i.e., all volume densities are within the threshold). |
| `hdds.datanode.disk.balancer.replica.deletion.delay` | `5m` | The delay after a container is successfully moved from source volume to destination volume before the source container replica is deleted. This lazy deletion provides a grace period before failing the read thread holding the old container replica. Unit: ns, ms, s, m, h, d. |
-| `hdds.datanode.disk.balancer.container.states` | `CLOSED,QUASI_CLOSED` | Comma-separated container lifecycle state names that may be moved between disks (must match enum names exactly, uppercase). Default includes **CLOSED** and **QUASI_CLOSED**; extend the list when additional states are needed to be balanced. All defined container states are OPEN, CLOSING, QUASI_CLOSED, CLOSED, UNHEALTHY, INVALID, DELETED, RECOVERING. |
+| `hdds.datanode.disk.balancer.container.states` | `CLOSED,QUASI_CLOSED` | Comma-separated container lifecycle state names that may be moved between disks (must match enum names exactly, uppercase). Default includes **CLOSED** and **QUASI_CLOSED**; extend the list when additional states are needed to be balanced. All defined container states which are eligibile to move QUASI_CLOSED, CLOSED, UNHEALTHY, INVALID. |
| `hdds.datanode.disk.balancer.container.choosing.policy` | `org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultContainerChoosingPolicy` | The policy for selecting source/destination volumes and which containers to move. |
| `hdds.datanode.disk.balancer.service.timeout` | `300s` | Timeout for the Datanode DiskBalancer service operations. |
| `hdds.datanode.disk.balancer.should.run.default` | `false` | If the balancer fails to read its persisted configuration, this value determines if the service should run by default. |
diff --git a/hadoop-hdds/docs/content/feature/DiskBalancer.zh.md b/hadoop-hdds/docs/content/feature/DiskBalancer.zh.md
index fb8c704e2035..585d1535d29a 100644
--- a/hadoop-hdds/docs/content/feature/DiskBalancer.zh.md
+++ b/hadoop-hdds/docs/content/feature/DiskBalancer.zh.md
@@ -39,9 +39,9 @@ summary: 数据节点的磁盘平衡器.
## 功能标志
-磁盘平衡器功能已通过功能标志引入。默认情况下，此功能处于禁用状态。
+磁盘平衡器功能已通过功能标志引入。默认情况下，此功能处于启用状态。
-可以通过在“ozone-site.xml”配置文件中将以下属性设置为“true”来**启用**该功能：
+可以通过在“ozone-site.xml”配置文件中将以下属性设置为“false”来**禁用**该功能：
`hdds.datanode.disk.balancer.enabled = false`
### 身份验证和授权
@@ -115,8 +115,8 @@ DiskBalancer 命令通过 RPC 直接与数据节点通信，因此需要进行
## 命令行用法
DiskBalancer 通过 `ozone admin datanode diskbalancer` 命令进行管理。
-**注意：**此命令在主帮助信息（`ozone admin datanode --help`）中隐藏。这是因为该功能目前处于实验阶段，默认禁用。隐藏该命令可防止意外使用，
-并为普通用户提供清晰的帮助输出。但是，对于希望启用和使用该功能的用户，该命令仍然完全可用。
+**注意：**DiskBalancer 在数据节点上默认启用。在 `ozone-site.xml` 中使用 `hdds.datanode.disk.balancer.enabled=false`
+可禁用数据节点上的服务并阻止 CLI 命令运行。
### 命令语法
**启动 DiskBalancer：**
@@ -149,13 +149,13 @@ ozone admin datanode diskbalancer report [ ...] [--in-service-
| Option | Description | Example |
|-------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------|
| `` | 一个或多个数据节点地址作为位置参数。地址可以是：
- 主机名（例如，`DN-1`）- 使用默认的 CLIENT_RPC 端口 (19864)
- 带端口的主机名（例如，`DN-1:19864`）
- IP 地址（例如，`192.168.1.10`）
- 带端口的 IP 地址（例如，`192.168.1.10:19864`）
- 标准输入 (`-`) - 从标准输入读取数据节点地址，每行一个 | `DN-1`
`DN-1:19864`
`192.168.1.10`
`-` |
-| `--in-service-datanodes` | 它向 SCM 查询所有 IN_SERVICE 数据节点，并在所有这些数据节点上执行该命令。 | `--in-service-datanodes` |
+| `--in-service-datanodes` | 它向 SCM 查询所有 IN_SERVICE 且 HEALTHY 的数据节点，并在所有这些数据节点上执行该命令。 | `--in-service-datanodes` |
| `--json` | 输出格式设置为JSON。 | `--json` |
| `-t/--threshold-percentage` | 磁盘使用率阈值百分比（默认值：10.0）。与 `start` 和 `update` 命令一起使用。 | `-t 5`
`--threshold-percentage 5.0` |
| `-b/--bandwidth-in-mb` | 最大磁盘带宽，单位为 MB/s（默认值：10）。与 `start` 和 `update` 命令一起使用。 | `-b 20`
`--bandwidth-in-mb 50` |
-| `-p/--parallel-thread` | 并行线程数（默认值：1）。与 `start` 和 `update` 命令一起使用。 | `-p 5`
`--parallel-thread 10` |
-| `-s/--stop-after-disk-even` | 磁盘平衡完成后自动停止（默认值：false）。与 `start` 和 `update` 命令一起使用。 | `-s false`
`--stop-after-disk-even true` |
-| `-c/--container-states` | 以逗号分隔的容器生命周期状态名称，表示可在磁盘之间移动的状态。配合 `start` 和 `update` 命令使用。 | `-c CLOSED,QUASI_CLOSED`
`--container-states OPEN,CLOSED` |
+| `-p/--parallel-thread` | 并行线程数（默认值：5）。与 `start` 和 `update` 命令一起使用。 | `-p 5`
`--parallel-thread 10` |
+| `-s/--stop-after-disk-even` | 磁盘平衡完成后自动停止（默认值：true）。与 `start` 和 `update` 命令一起使用。 | `-s false`
`--stop-after-disk-even true` |
+| `-c/--container-states` | 以逗号分隔的容器生命周期状态名称，表示可在磁盘之间移动的状态。配合 `start` 和 `update` 命令使用。 | `-c CLOSED,QUASI_CLOSED`
`--container-states CLOSED` |
### 示例
**启动 DiskBalancer：**
@@ -164,7 +164,7 @@ ozone admin datanode diskbalancer report [ ...] [--in-service-
# 在多个数据节点上启动 DiskBalancer
ozone admin datanode diskbalancer start DN-1 DN-2 DN-3
-# 在所有运行中的数据节点上启动 DiskBalancer
+# 在所有 IN_SERVICE 且 HEALTHY 的数据节点上启动 DiskBalancer
ozone admin datanode diskbalancer start --in-service-datanodes
# 使用配置参数启动 DiskBalancer
@@ -183,7 +183,7 @@ ozone admin datanode diskbalancer start DN-1 --json
# 在多个数据节点上停止 DiskBalancer
ozone admin datanode diskbalancer stop DN-1 DN-2 DN-3
-# 在所有运行中的数据节点上停止 DiskBalancer
+# 在所有 IN_SERVICE 且 HEALTHY 的数据节点上停止 DiskBalancer
ozone admin datanode diskbalancer stop --in-service-datanodes
# 停止 DiskBalancer 并输出 JSON 信息
@@ -195,7 +195,7 @@ ozone admin datanode diskbalancer stop DN-1 --json
# 更新多个参数
ozone admin datanode diskbalancer update DN-1 -t 5 -b 50 -p 10
-# 更新所有 IN_SERVICE 数据节点
+# 更新所有 IN_SERVICE 且 HEALTHY 的数据节点
ozone admin datanode diskbalancer update --in-service-datanodes -t 5
# 更新并输出 JSON 格式
@@ -208,7 +208,7 @@ ozone admin datanode diskbalancer update DN-1 -b 50 --json
# 从多个数据节点获取状态
ozone admin datanode diskbalancer status DN-1 DN-2 DN-3
-# 从所有处于服务状态的数据节点获取状态
+# 从所有 IN_SERVICE 且 HEALTHY 的数据节点获取状态
ozone admin datanode diskbalancer status --in-service-datanodes
# 以 JSON 格式获取状态
@@ -220,7 +220,7 @@ ozone admin datanode diskbalancer status --in-service-datanodes --json
# 从多个数据节点获取报告
ozone admin datanode diskbalancer report DN-1 DN-2 DN-3
-# 从所有处于服务状态的数据节点获取报告
+# 从所有 IN_SERVICE 且 HEALTHY 的数据节点获取报告
ozone admin datanode diskbalancer report --in-service-datanodes
# 以 JSON 格式获取报告
@@ -233,14 +233,14 @@ The DiskBalancer's behavior can be controlled using the following configuration
| Property | Default Value | Description |
|-------------------------------------------------------------|----------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `hdds.datanode.disk.balancer.enabled` | `false` | 如果为 false，则 Datanode 上的 DiskBalancer 服务将被禁用。将其配置为 true 可启用 DiskBalancer。 | | | |
+| `hdds.datanode.disk.balancer.enabled` | `true` | 如果为 false，则 Datanode 上的 DiskBalancer 服务将被禁用。默认情况下，DiskBalancer 在 Datanode 上启用。 | | | |
| `hdds.datanode.disk.balancer.volume.density.threshold.percent` | `10.0` | 百分比（0-100）。如果对于每个卷，其利用率与平均数据节点利用率之差不超过此阈值，则认为数据节点处于平衡状态。 |
| `hdds.datanode.disk.balancer.max.disk.throughputInMBPerSec` | `10` | 平衡器可用于移动数据的最大带宽（以 MB/s 为单位），以避免影响客户端 I/O。 |
| `hdds.datanode.disk.balancer.parallel.thread` | `5` | 用于并行移动容器的工作线程数。 |
| `hdds.datanode.disk.balancer.service.interval` | `60s` | Datanode DiskBalancer 服务检查不平衡并更新其配置的时间间隔。 |
| `hdds.datanode.disk.balancer.stop.after.disk.even` | `true` | 如果为真，则一旦磁盘被视为平衡（即所有卷密度都在阈值内），DiskBalancer 将自动停止其平衡活动。 |
| `hdds.datanode.disk.balancer.replica.deletion.delay` | `5m` | 容器成功从源卷移动到目标卷后，源容器副本被删除前的延迟时间。这种延迟删除机制旨在避免旧副本的即时删除导致持有旧容器副本的线程数据读取失败。单位：ns、ms、s、m、h、d。|
-| `hdds.datanode.disk.balancer.container.states` | `CLOSED,QUASI_CLOSED` | 以逗号分隔的容器生命周期状态名称列表，指定了允许在不同磁盘之间移动的容器状态（须与枚举名完全一致，使用大写）。默认包含 **CLOSED** 和 **QUASI_CLOSED**；若需对更多状态的容器进行负载均衡，请扩展此列表。所有已定义的容器状态包括：OPEN、CLOSING、QUASI_CLOSED、CLOSED、UNHEALTHY、INVALID、DELETED 和 RECOVERING。 |
+| `hdds.datanode.disk.balancer.container.states` | `CLOSED,QUASI_CLOSED` | 以逗号分隔的容器生命周期状态名称列表，指定可在不同磁盘之间移动的容器状态（须与枚举名完全一致，使用大写）。默认包含 **CLOSED** 和 **QUASI_CLOSED**；若需对更多状态的容器进行负载均衡，请扩展此列表。可移动的已定义容器状态包括：QUASI_CLOSED、CLOSED、UNHEALTHY、INVALID。 |
| `hdds.datanode.disk.balancer.container.choosing.policy` | `org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultContainerChoosingPolicy` | 用于选择源/目标卷以及要移动的容器的策略。 |
| `hdds.datanode.disk.balancer.service.timeout` | `300s` | Datanode DiskBalancer 服务操作超时。 |
| `hdds.datanode.disk.balancer.should.run.default` | `false` | 如果平衡器无法读取其持久配置，则该值决定服务是否应默认运行。 |
diff --git a/hadoop-hdds/docs/content/feature/OM-HA.md b/hadoop-hdds/docs/content/feature/OM-HA.md
index f055b2958490..d0978c7d939d 100644
--- a/hadoop-hdds/docs/content/feature/OM-HA.md
+++ b/hadoop-hdds/docs/content/feature/OM-HA.md
@@ -137,7 +137,7 @@ ozone admin om transfer -id -r
```
* `-id, --service-id`: Specifies the Ozone Manager Service ID.
-* `-n, --newLeaderId, --new-leader-id`: The node ID of the OM to which leadership will be transferred (e.g., `om1`).
+* `-n, --new-leader-id`: The node ID of the OM to which leadership will be transferred (e.g., `om1`).
* `-r, --random`: Randomly chooses a follower to transfer leadership to.
### Example
diff --git a/hadoop-hdds/docs/content/feature/SCM-HA.md b/hadoop-hdds/docs/content/feature/SCM-HA.md
index 7f9396fafe69..0703ccba2fd8 100644
--- a/hadoop-hdds/docs/content/feature/SCM-HA.md
+++ b/hadoop-hdds/docs/content/feature/SCM-HA.md
@@ -104,7 +104,7 @@ ozone admin scm transfer -id -r
```
* `-id, --service-id`: Specifies the SCM Service ID.
-* `-n, --newLeaderId, --new-leader-id`: The SCM UUID (Raft peer ID) of the SCM to which leadership will be transferred (e.g., `e6877ce5-56cd-4f0b-ad60-4c8ef9000882`).
+* `-n, --new-leader-id`: The SCM UUID (Raft peer ID) of the SCM to which leadership will be transferred (e.g., `e6877ce5-56cd-4f0b-ad60-4c8ef9000882`).
* `-r, --random`: Randomly chooses a follower to transfer leadership to.
### Example
@@ -291,7 +291,7 @@ layoutVersion=0
You can also create data and double check with `ozone debug` tool if all the container metadata is replicated.
```shell
-bin/ozone freon randomkeys --numOfVolumes=1 --numOfBuckets=1 --numOfKeys=10000 --keySize=524288 --replicationType=RATIS --numOfThreads=8 --factor=THREE --bufferSize=1048576
+bin/ozone freon randomkeys --num-of-volumes=1 --num-of-buckets=1 --num-of-keys=10000 --key-size=524288 --type=RATIS --num-of-threads=8 --factor=THREE --buffer-size=1048576
# use debug ldb to check scm.db on all the machines
diff --git a/hadoop-hdds/docs/content/feature/SCM-HA.zh.md b/hadoop-hdds/docs/content/feature/SCM-HA.zh.md
index 66d2b885fbee..2169ae475636 100644
--- a/hadoop-hdds/docs/content/feature/SCM-HA.zh.md
+++ b/hadoop-hdds/docs/content/feature/SCM-HA.zh.md
@@ -191,7 +191,7 @@ layoutVersion=0
如果所有的容器元数据都已复制，您还可以创建数据并使用 `ozone debug` 工具进行双重检查。
```shell
-bin/ozone freon randomkeys --numOfVolumes=1 --numOfBuckets=1 --numOfKeys=10000 --keySize=524288 --replicationType=RATIS --numOfThreads=8 --factor=THREE --bufferSize=1048576
+bin/ozone freon randomkeys --num-of-volumes=1 --num-of-buckets=1 --num-of-keys=10000 --key-size=524288 --type=RATIS --num-of-threads=8 --factor=THREE --buffer-size=1048576
# 使用 debug ldb 工具逐一检查各机上的 scm.db
diff --git a/hadoop-hdds/docs/content/feature/Snapshot-Configuration-Properties.md b/hadoop-hdds/docs/content/feature/Snapshot-Configuration-Properties.md
index 3b530e19958e..802a9fb6bd39 100644
--- a/hadoop-hdds/docs/content/feature/Snapshot-Configuration-Properties.md
+++ b/hadoop-hdds/docs/content/feature/Snapshot-Configuration-Properties.md
@@ -42,10 +42,10 @@ These parameters, defined in `ozone-site.xml`, control how Ozone manages snapsho
* `ozone.om.snapshot.diff.db.dir`: Directory for SnapshotDiff job data. Defaults to OM metadata dir. Use a spacious location for large diffs.
* `ozone.om.snapshot.force.full.diff`: Force a full diff for all snapshot diff jobs (Default: false).
* `ozone.om.snapshot.diff.disable.native.libs`: Disable native libraries for snapshot diff (Default: false).
- * `ozone.om.snapshot.diff.max.page.size`: Maximum page size for snapshot diff (Default: 1000).
+ * `ozone.om.snapshot.diff.max.page.size`: Maximum page size for snapshot diff (Default: 5000).
* `ozone.om.snapshot.diff.thread.pool.size`: Thread pool size for snapshot diff (Default: 10).
* `ozone.om.snapshot.diff.job.default.wait.time`: Default wait time for a snapshot diff job (Default: 1m).
- * `ozone.om.snapshot.diff.max.allowed.keys.changed.per.job`: Maximum number of keys allowed to be changed per snapshot diff job (Default: 10000000).
+ * `ozone.om.snapshot.diff.max.allowed.keys.changed.per.job`: Maximum number of keys allowed to be changed per snapshot diff job (Default: 1000000000).
* **Snapshot Compaction and Cleanup**
* `ozone.snapshot.key.deleting.limit.per.task`: The maximum number of keys scanned by the snapshot deleting service in a single run (Default: 20000).
diff --git a/hadoop-hdds/docs/content/feature/Snapshot.md b/hadoop-hdds/docs/content/feature/Snapshot.md
index 3ac1d931d497..5a1cee340e7f 100644
--- a/hadoop-hdds/docs/content/feature/Snapshot.md
+++ b/hadoop-hdds/docs/content/feature/Snapshot.md
@@ -48,6 +48,24 @@ When keys are changed or deleted in the live bucket, their data blocks are retai
**Snapshot Data Storage:** Snapshot metadata resides in OM's RocksDB. Diff job data is stored in `ozone.om.snapshot.diff.db.dir` (defaults to OM metadata directory).
+### Snapshot Space & Size Tracking
+
+When a snapshot is created, it references the state of the bucket at that point in time. Over time, as keys are deleted or overwritten in the active namespace, the data blocks are kept alive by the snapshots. Ozone tracks space usage for snapshots using the following metrics:
+
+#### Referenced Size
+* **`referencedSize`**: The total logical data size (in bytes, unreplicated) of all active keys/files in the bucket at the moment the snapshot was created.
+* **`referencedReplicatedSize`**: The total replicated data size (in bytes, replicated) referenced by the snapshot at its creation point.
+
+#### Exclusive Size
+As mutations occur in the active bucket or other snapshots, some blocks become exclusively held by a single snapshot. Ozone tracks this exclusive size using two separate asynchronous background services:
+
+1. **`KeyDeletingService` (Key Deep Cleaning)**: Processes deleted keys to find blocks exclusively held by the snapshot. It sets `exclusiveSize` (unreplicated) and `exclusiveReplicatedSize` (replicated).
+2. **`SnapshotDirectoryCleaningService` (Directory Deep Cleaning)**: Recursively processes deleted directories. To avoid write conflicts and overwriting between these two independent services, it stores its results separately in `exclusiveSizeDeltaFromDirDeepCleaning` (unreplicated) and `exclusiveReplicatedSizeDeltaFromDirDeepCleaning` (replicated).
+
+The actual total exclusive size of a snapshot is the sum of these fields:
+* **Total Exclusive Size**: `exclusiveSize` + `exclusiveSizeDeltaFromDirDeepCleaning`
+* **Total Exclusive Replicated Size**: `exclusiveReplicatedSize` + `exclusiveReplicatedSizeDeltaFromDirDeepCleaning`
+
For more details, see Prashant Pogde’s [Introducing Apache Ozone Snapshots](https://medium.com/@prashantpogde/introducing-apache-ozone-snapshots-af82e976142f).
## User Tutorial
diff --git a/hadoop-hdds/docs/content/interface/ReconApi.md b/hadoop-hdds/docs/content/interface/ReconApi.md
index e2df65d168b6..c338908b33f4 100644
--- a/hadoop-hdds/docs/content/interface/ReconApi.md
+++ b/hadoop-hdds/docs/content/interface/ReconApi.md
@@ -96,36 +96,40 @@ Returns all the ContainerMetadata objects.
**Returns**
-Returns all the KeyMetadata objects for the given ContainerID.
-
+Returns all the KeyMetadata objects for the given ContainerID. `lastKey` is the final key seen in
+this page: pass it back as `prevKey` to continue paginating.
+
```json
{
- "totalCount":7,
+ "totalCount": 7,
+ "lastKey": "/vol-1-73141/bucket-3-35816/key-0-43637",
"keys": [
{
- "Volume":"vol-1-73141",
- "Bucket":"bucket-3-35816",
- "Key":"key-0-43637",
- "DataSize":1000,
- "Versions":[0],
+ "Volume": "vol-1-73141",
+ "Bucket": "bucket-3-35816",
+ "Key": "key-0-43637",
+ "CompletePath": "/vol-1-73141/bucket-3-35816/dir1/dir2/key-0-43637",
+ "DataSize": 1000,
+ "Versions": [0],
"Blocks": {
"0": [
{
- "containerID":1,
- "localID":105232659753992201
+ "containerID": 1,
+ "localID": 105232659753992201
}
]
},
- "CreationTime":"2020-11-18T18:09:17.722Z",
- "ModificationTime":"2020-11-18T18:09:30.405Z"
- },
- ...
+ "CreationTime": "2020-11-18T18:09:17.722Z",
+ "ModificationTime": "2020-11-18T18:09:30.405Z"
+ }
]
}
```
### GET /api/v1/containers/missing
+> **Deprecated.** Use `/api/v1/containers/unhealthy/MISSING` instead.
+
**Parameters**
* limit (optional)
@@ -159,6 +163,58 @@ Returns the MissingContainerMetadata objects for all the missing containers.
}
```
+### GET /api/v1/containers/quasiClosed
+
+**Parameters**
+
+* limit (optional)
+
+ Maximum number of containers to return. Default is 1000.
+
+* minContainerId (optional)
+
+ Cursor. Returns containers with ID greater than this value, in ascending order. Pass the
+ previous response's `lastKey` to fetch the next page. Default is 0.
+
+**Returns**
+
+Returns containers currently in the `QUASI_CLOSED` lifecycle state. `quasiClosedCount` is the
+cluster-wide total (not just the current page). When the page is empty, both `firstKey` and
+`lastKey` echo back the `minContainerId` cursor.
+
+```json
+{
+ "quasiClosedCount": 42,
+ "firstKey": 100,
+ "lastKey": 199,
+ "containers": [
+ {
+ "containerID": 100,
+ "pipelineID": "88646d32-a1aa-4e1a-a8d5-aa1e7dd3f5cc",
+ "keys": 17,
+ "stateEnterTime": 1718640123456,
+ "expectedReplicaCount": 3,
+ "actualReplicaCount": 2,
+ "replicas": [
+ {
+ "containerID": 100,
+ "datanodeUuid": "841be80f-0454-47df-b676",
+ "datanodeHost": "localhost-1",
+ "firstSeenTime": 1605724047057,
+ "lastSeenTime": 1605731201301,
+ "lastBcsId": 123,
+ "state": "QUASI_CLOSED"
+ }
+ ]
+ }
+ ]
+}
+```
+
+Responses:
+
+* `400 Bad Request`: `limit` or `minContainerId` is negative.
+
### GET /api/v1/containers/:id/replicaHistory
**Parameters**
@@ -183,22 +239,26 @@ Returns all the ContainerHistory objects for the given ContainerID.
### GET /api/v1/containers/unhealthy
-
-**Parameters**
-* batchNum (optional)
+**Parameters**
- The batch number (like "page number") of results to return.
- Passing 1, will return records 1 to limit. 2 will return
- limit + 1 to 2 * limit, etc.
-
* limit (optional)
- Only returns the limited number of results. The default limit is 1000.
+ Only returns the limited number of results. The default limit is 1000.
+
+* maxContainerId (optional)
+
+ Upper bound for container IDs (exclusive). When specified, returns containers with IDs less
+ than this value in descending order. Use it for backward pagination.
+
+* minContainerId (optional)
+
+ Lower bound for container IDs (exclusive). When `maxContainerId` is not specified, returns
+ containers with IDs greater than this value in ascending order. Use it for forward pagination.
**Returns**
-Returns the UnhealthyContainerMetadata objects for all the unhealthycontainers.
+Returns the UnhealthyContainerMetadata objects for all the unhealthy containers.
```json
{
@@ -231,26 +291,99 @@ Returns the UnhealthyContainerMetadata objects for all the unhealthycontainers.
```
### GET /api/v1/containers/unhealthy/:state
-
+
**Parameters**
-* batchNum (optional)
-
- The batch number (like "page number") of results to return.
- Passing 1, will return records 1 to limit. 2 will return
- limit + 1 to 2 * limit, etc.
-
* limit (optional)
- Only returns the limited number of results. The default limit is 1000.
+ Only returns the limited number of results. The default limit is 1000.
+
+* maxContainerId (optional)
+
+ Upper bound for container IDs (exclusive). When specified, returns containers with IDs less
+ than this value in descending order. Use it for backward pagination.
+
+* minContainerId (optional)
+
+ Lower bound for container IDs (exclusive). When `maxContainerId` is not specified, returns
+ containers with IDs greater than this value in ascending order. Use it for forward pagination.
**Returns**
Returns the UnhealthyContainerMetadata objects for the containers in the given state.
-Possible unhealthy container states are `MISSING`, `MIS_REPLICATED`,`UNDER_REPLICATED`, `OVER_REPLICATED`.
+Possible unhealthy container states are `MISSING`, `MIS_REPLICATED`, `UNDER_REPLICATED`, `OVER_REPLICATED`.
The response structure is same as `/containers/unhealthy`.
+### GET /api/v1/containers/unhealthy/export
+
+**Returns**
+
+Lists every unhealthy-container export job currently tracked by Recon, in any status.
+Items are `ExportJob` objects (see schema below).
+
+```json
+[
+ {
+ "jobId": "4f7a8b9c-1234-5678-9abc-def012345678",
+ "state": "MISSING",
+ "status": "RUNNING",
+ "submittedAt": 1718640123456,
+ "startedAt": 1718640124000,
+ "completedAt": 0,
+ "totalRecords": 250,
+ "estimatedTotal": 1000,
+ "fileName": "",
+ "errorMessage": null,
+ "progressPercent": 25,
+ "queuePosition": 0,
+ "downloadCount": 0,
+ "downloadsRemaining": 3
+ }
+]
+```
+
+### POST /api/v1/containers/unhealthy/export
+
+**Parameters**
+
+* state (required)
+
+ One of `MISSING`, `MIS_REPLICATED`, `UNDER_REPLICATED`, `OVER_REPLICATED`.
+
+**Returns**
+
+Submits a new CSV export job and returns the `ExportJob` with the assigned `jobId`.
+The job initially has `status: QUEUED`.
+
+* `400 Bad Request`: `state` is missing or not a valid unhealthy state.
+* `429 Too Many Requests`: the export queue is full; retry later. Body: `{ "error": "Too Many Requests", "message": "" }`.
+
+### GET /api/v1/containers/unhealthy/export/:jobId
+
+**Returns**
+
+Returns the current `ExportJob` for the given `jobId`. `404 Not Found` if no job has that id.
+
+### GET /api/v1/containers/unhealthy/export/:jobId/download
+
+**Returns**
+
+Streams the TAR archive produced by the export job. Response `Content-Type` is `application/x-tar` with
+a `Content-Disposition: attachment` header carrying the export filename.
+
+* `404 Not Found`: `jobId` is unknown or the on-disk file was removed.
+* `409 Conflict`: the job has not reached `COMPLETED` status yet.
+* `429 Too Many Requests`: the per-job download limit has been reached. Body: `{ "error": "Download limit reached", "message": "" }` (schema `DownloadLimitReachedError`).
+
+### DELETE /api/v1/containers/unhealthy/export/:jobId
+
+**Returns**
+
+Cancels the export job. `200 OK` with empty body on success. `404 Not Found` if the job cannot be
+cancelled (for example, it has already reached a terminal state).
+
+
### GET /api/v1/containers/mismatch
**Returns**
@@ -306,6 +439,41 @@ list of keys mapped to such DELETED state containers.
]
```
+### GET /api/v1/containers/deleted
+
+**Parameters**
+
+* limit (optional)
+
+ Maximum number of DELETED containers to return. Default 1000.
+
+* prevKey (optional)
+
+ Previous container ID to skip. Use the last returned `containerId` to fetch the next page.
+ Default 0.
+
+**Returns**
+
+Returns all DELETED containers in SCM along with their pipeline and replication info.
+
+```json
+[
+ {
+ "containerId": 12,
+ "pipelineID": { "id": "1202e6bb-b7c1-4a85-8067-61374b069adb" },
+ "containerState": "DELETED",
+ "stateEnterTime": 1716123456789,
+ "lastUsed": 1716123456789,
+ "replicationConfig": {
+ "replicationType": "RATIS",
+ "replicationFactor": "THREE",
+ "replicationNodes": 3
+ },
+ "replicationFactor": "THREE"
+ }
+]
+```
+
### GET /api/v1/keys/open
@@ -320,60 +488,98 @@ list of keys mapped to such DELETED state containers.
Only returns the limited number of results. The default limit is 1000.
+* startPrefix (optional)
+
+ Restricts the listing to keys matching this prefix. Must be at bucket level or deeper
+ (e.g. `/vol1/bucket1` or `/vol1/bucket1/dir1`); shallower prefixes return `400 Bad Request`.
+
+* includeFso (optional)
+
+ Boolean, default `false`. Include keys/files from FSO buckets in the result.
+
+* includeNonFso (optional)
+
+ Boolean, default `false`. Include keys/files from non-FSO (OBS / LEGACY) buckets.
+
+If neither `includeFso` nor `includeNonFso` is `true`, the response will be empty.
+
**Returns**
-Returns set of keys/files which are open.
+Returns set of keys/files which are open. FSO and non-FSO keys are reported in separate arrays.
```json
{
"lastKey": "/vol1/fso-bucket/dir1/dir2/file2",
- "replicatedTotal": 13824,
- "unreplicatedTotal": 4608,
- "entities": [
+ "replicatedDataSize": 13824,
+ "unreplicatedDataSize": 4608,
+ "status": "OK",
+ "fso": [
{
- "path": "/vol1/bucket1/key1",
- "keyState": "Open",
+ "key": "/-9223372036854775552/-9223372036854774016/file1",
+ "path": "/vol1/fso-bucket/dir1/file1",
"inStateSince": 1667564193026,
"size": 1024,
"replicatedSize": 3072,
- "unreplicatedSize": 1024,
- "replicationType": "RATIS",
- "replicationFactor": "THREE"
- },
- {
- "path": "/vol1/bucket1/key2",
- "keyState": "Open",
- "inStateSince": 1667564193026,
- "size": 512,
- "replicatedSize": 1536,
- "unreplicatedSize": 512,
- "replicationType": "RATIS",
- "replicationFactor": "THREE"
- },
+ "replicationInfo": {
+ "replicationFactor": "THREE",
+ "requiredNodes": 3,
+ "replicationType": "RATIS"
+ },
+ "creationTime": 1667564000000,
+ "modificationTime": 1667564193026,
+ "isKey": true
+ }
+ ],
+ "nonFSO": [
{
- "path": "/vol1/fso-bucket/dir1/file1",
- "keyState": "Open",
+ "key": "/vol1/bucket1/key1",
+ "path": "/vol1/bucket1/key1",
"inStateSince": 1667564193026,
"size": 1024,
"replicatedSize": 3072,
- "unreplicatedSize": 1024,
- "replicationType": "RATIS",
- "replicationFactor": "THREE"
- },
- {
- "path": "/vol1/fso-bucket/dir1/dir2/file2",
- "keyState": "Open",
- "inStateSince": 1667564193026,
- "size": 2048,
- "replicatedSize": 6144,
- "unreplicatedSize": 2048,
- "replicationType": "RATIS",
- "replicationFactor": "THREE"
+ "replicationInfo": {
+ "replicationFactor": "THREE",
+ "requiredNodes": 3,
+ "replicationType": "RATIS"
+ },
+ "creationTime": 1667564000000,
+ "modificationTime": 1667564193026,
+ "isKey": true
}
]
}
```
+### GET /api/v1/keys/open/summary
+
+**Returns**
+
+Returns a flat summary of all currently-open keys across the cluster.
+
+```json
+{
+ "totalOpenKeys": 8,
+ "totalReplicatedDataSize": 90000,
+ "totalUnreplicatedDataSize": 30000
+}
+```
+
+### GET /api/v1/keys/open/mpu/summary
+
+**Returns**
+
+Returns a flat summary of all currently-open multipart-upload keys across the cluster. Note that
+the unreplicated total is reported as `totalDataSize` (not `totalUnreplicatedDataSize`): the
+naming differs from `/keys/open/summary`.
+
+```json
+{
+ "totalOpenMPUKeys": 2,
+ "totalReplicatedDataSize": 90000,
+ "totalDataSize": 30000
+}
+```
+
### GET /api/v1/keys/deletePending
@@ -388,48 +594,41 @@ Returns set of keys/files which are open.
Only returns the limited number of results. The default limit is 1000.
+* startPrefix (optional)
+
+ Restricts the listing to keys matching this prefix. Must be at bucket level or deeper
+ (e.g. `/vol1/bucket1` or `/vol1/bucket1/dir1`); shallower prefixes return `400 Bad Request`.
+
**Returns**
-Returns set of keys/files pending for deletion.
+Returns the set of keys/files pending deletion, paired with aggregated size totals. Each item in
+`deletedKeyInfo` is a `RepeatedOmKeyInfo` (a wrapper around one or more `OmKeyInfo` entries).
```json
{
"lastKey": "sampleVol/bucketOne/key_one",
- "replicatedTotal": -1530804718628866300,
- "unreplicatedTotal": -1530804718628866300,
- "deletedkeyinfo": [
+ "replicatedDataSize": 1800000,
+ "unreplicatedDataSize": 600000,
+ "deletedKeyInfo": [
{
"omKeyInfoList": [
{
- "metadata": {},
- "objectID": 0,
- "updateID": 0,
- "parentObjectID": 0,
"volumeName": "sampleVol",
"bucketName": "bucketOne",
"keyName": "key_one",
- "dataSize": -1530804718628866300,
- "keyLocationVersions": [],
- "creationTime": 0,
- "modificationTime": 0,
+ "dataSize": 200000,
+ "replicatedSize": 600000,
"replicationConfig": {
- "replicationFactor": "ONE",
- "requiredNodes": 1,
- "replicationType": "STANDALONE"
+ "replicationFactor": "THREE",
+ "requiredNodes": 3,
+ "replicationType": "RATIS"
},
- "fileChecksum": null,
- "fileName": "key_one",
- "acls": [],
- "path": "0/key_one",
- "file": false,
- "latestVersionLocations": null,
- "replicatedSize": -1530804718628866300,
- "fileEncryptionInfo": null,
- "objectInfo": "OMKeyInfo{volume='sampleVol', bucket='bucketOne', key='key_one', dataSize='-1530804718628866186', creationTime='0', objectID='0', parentID='0', replication='STANDALONE/ONE', fileChecksum='null}",
- "updateIDset": false
+ "creationTime": 1717000000000,
+ "modificationTime": 1717100000000
}
]
- }
+ },
+ ...
],
"status": "OK"
}
@@ -451,51 +650,127 @@ Returns set of keys/files pending for deletion.
**Returns**
- Returns set of directories pending for deletion.
+Returns the set of directories pending for deletion. Each entry in `deletedDirInfo` is a
+`KeyEntityInfo` describing one pending-delete directory (not a `RepeatedOmKeyInfo` like
+`/keys/deletePending`).
```json
{
- "lastKey": "vol1/bucket1/bucket1/dir1",
- "replicatedTotal": -1530804718628866300,
- "unreplicatedTotal": -1530804718628866300,
- "deletedkeyinfo": [
+ "lastKey": "/vol1/bucket1/dir1",
+ "replicatedDataSize": 13824,
+ "unreplicatedDataSize": 4608,
+ "deletedDirInfo": [
{
- "omKeyInfoList": [
- {
- "metadata": {},
- "objectID": 0,
- "updateID": 0,
- "parentObjectID": 0,
- "volumeName": "sampleVol",
- "bucketName": "bucketOne",
- "keyName": "key_one",
- "dataSize": -1530804718628866300,
- "keyLocationVersions": [],
- "creationTime": 0,
- "modificationTime": 0,
- "replicationConfig": {
- "replicationFactor": "ONE",
- "requiredNodes": 1,
- "replicationType": "STANDALONE"
- },
- "fileChecksum": null,
- "fileName": "key_one",
- "acls": [],
- "path": "0/key_one",
- "file": false,
- "latestVersionLocations": null,
- "replicatedSize": -1530804718628866300,
- "fileEncryptionInfo": null,
- "objectInfo": "OMKeyInfo{volume='sampleVol', bucket='bucketOne', key='key_one', dataSize='-1530804718628866186', creationTime='0', objectID='0', parentID='0', replication='STANDALONE/ONE', fileChecksum='null}",
- "updateIDset": false
- }
- ]
+ "key": "/-9223372036854775552/-9223372036854774016/dir1",
+ "path": "/vol1/bucket1/dir1",
+ "inStateSince": 1717000000000,
+ "size": 4608,
+ "replicatedSize": 13824,
+ "replicationInfo": {
+ "replicationFactor": "THREE",
+ "requiredNodes": 3,
+ "replicationType": "RATIS"
+ },
+ "creationTime": 1716900000000,
+ "modificationTime": 1716999999999,
+ "isKey": false
}
],
"status": "OK"
}
```
+### GET /api/v1/keys/deletePending/summary
+
+**Returns**
+
+Returns a flat summary of all keys pending deletion across the cluster.
+
+```json
+{
+ "totalDeletedKeys": 8,
+ "totalReplicatedDataSize": 90000,
+ "totalUnreplicatedDataSize": 30000
+}
+```
+
+### GET /api/v1/keys/deletePending/dirs/summary
+
+**Returns**
+
+Returns the total count of directories pending deletion.
+
+```json
+{
+ "totalDeletedDirectories": 5
+}
+```
+
+### GET /api/v1/keys/listKeys
+
+**Parameters**
+
+* startPrefix (optional, but effectively required)
+
+ Bucket-level or deeper prefix (e.g. `/vol1/bucket1` or `/vol1/bucket1/dir1`). HTTP-level the
+ parameter is optional (defaults to `/`), but the handler rejects anything shallower than
+ bucket level with `400 Bad Request`, so in practice callers must supply one.
+
+* replicationType (optional)
+
+ Filter by replication type (e.g. `RATIS`, `EC`).
+
+* creationDate (optional)
+
+ Filter by creation date; only keys created on or after this date are returned.
+
+* keySize (optional)
+
+ Filter to keys with data size at least this many bytes. Default 0.
+
+* prevKey (optional)
+
+ Pagination cursor. Pass back the `lastKey` from the previous response to continue iteration.
+
+* limit (optional)
+
+ Maximum number of keys to return. Default 1000.
+
+**Returns**
+
+Returns committed keys (and files in FSO buckets) under the given prefix.
+
+* `200 OK` with a `ListKeysResponse` body.
+* `204 No Content` when no keys matched the given filters.
+* `400 Bad Request` when `startPrefix` is missing or shallower than bucket level.
+* `503 Service Unavailable` while Recon is still bootstrapping OM DB; response body status is `INITIALIZING`.
+
+```json
+{
+ "status": "OK",
+ "path": "/vol1/bucket1",
+ "replicatedDataSize": 600000,
+ "unReplicatedDataSize": 200000,
+ "lastKey": "/vol1/bucket1/dir1/file42",
+ "keys": [
+ {
+ "key": "/vol1/bucket1/dir1/file42",
+ "path": "/vol1/bucket1/dir1/file42",
+ "size": 1048576,
+ "replicatedSize": 3145728,
+ "replicationInfo": {
+ "replicationFactor": "THREE",
+ "requiredNodes": 3,
+ "replicationType": "RATIS"
+ },
+ "creationTime": 1717000000000,
+ "modificationTime": 1717100000000,
+ "isKey": true
+ }
+ ]
+}
+```
+
## Blocks Metadata (admin only)
### GET /api/v1/blocks/deletePending
@@ -761,20 +1036,33 @@ No parameters.
Returns a summary of the current state of the Ozone cluster.
```json
- {
- "pipelines": 5,
- "totalDatanodes": 4,
- "healthyDatanodes": 4,
- "storageReport": {
- "capacity": 1081719668736,
- "used": 1309212672,
- "remaining": 597361258496
- },
- "containers": 26,
- "volumes": 6,
- "buckets": 26,
- "keys": 25
- }
+{
+ "pipelines": 5,
+ "totalDatanodes": 4,
+ "healthyDatanodes": 4,
+ "storageReport": {
+ "capacity": 1081719668736,
+ "used": 1309212672,
+ "remaining": 597361258496,
+ "committed": 27007111,
+ "reserved": 31457280,
+ "minimumFreeSpace": 20480,
+ "filesystemCapacity": 1081730000000,
+ "filesystemUsed": 1310000000,
+ "filesystemAvailable": 597361258496
+ },
+ "containers": 26,
+ "missingContainers": 0,
+ "openContainers": 5,
+ "deletedContainers": 1,
+ "volumes": 6,
+ "buckets": 26,
+ "keys": 25,
+ "keysPendingDeletion": 0,
+ "deletedDirs": 0,
+ "scmServiceId": "scmservice",
+ "omServiceId": "omservice"
+}
```
## Volumes (admin only)
@@ -898,35 +1186,42 @@ No parameters.
Returns all the datanodes in the cluster.
```json
- {
- "totalCount": 4,
- "datanodes": [{
- "uuid": "f8f8cb45-3ab2-4123",
- "hostname": "localhost-1",
- "state": "HEALTHY",
- "lastHeartbeat": 1605738400544,
- "storageReport": {
- "capacity": 270429917184,
- "used": 358805504,
- "remaining": 119648149504
- },
- "pipelines": [{
- "pipelineID": "b9415b20-b9bd-4225",
- "replicationType": "RATIS",
- "replicationFactor": 3,
- "leaderNode": "localhost-2"
- }, {
- "pipelineID": "3bf4a9e9-69cc-4d20",
- "replicationType": "RATIS",
- "replicationFactor": 1,
- "leaderNode": "localhost-1"
- }],
- "containers": 17,
- "leaderCount": 1
- },
- ...
- ]
- }
+{
+ "totalCount": 4,
+ "datanodes": [
+ {
+ "uuid": "f8f8cb45-3ab2-4123",
+ "hostname": "localhost-1",
+ "state": "HEALTHY",
+ "opState": "IN_SERVICE",
+ "lastHeartbeat": 1605738400544,
+ "storageReport": {
+ "capacity": 270429917184,
+ "used": 358805504,
+ "remaining": 270071111680,
+ "committed": 27007111,
+ "reserved": 31457280,
+ "minimumFreeSpace": 20480,
+ "filesystemCapacity": 270461374464,
+ "filesystemUsed": 390262784,
+ "filesystemAvailable": 270071111680
+ },
+ "pipelines": [
+ { "pipelineID": "b9415b20-b9bd-4225", "replicationType": "RATIS", "replicationFactor": 3, "leaderNode": "localhost-2" },
+ { "pipelineID": "3bf4a9e9-69cc-4d20", "replicationType": "RATIS", "replicationFactor": 1, "leaderNode": "localhost-1" }
+ ],
+ "containers": 17,
+ "openContainers": 4,
+ "leaderCount": 1,
+ "version": "2.0.0",
+ "setupTime": 1605700000000,
+ "revision": "abcdef1",
+ "layoutVersion": 6,
+ "networkLocation": "/default-rack"
+ },
+ ...
+ ]
+}
```
### PUT /api/v1/datanodes/remove
@@ -938,30 +1233,99 @@ Returns all the datanodes in the cluster.
```json
[
"50ca4c95-2ef3-4430-b944-97d2442c3daf"
-]
+]
```
**Returns**
-Returns the list of datanodes which are removed successfully and list of datanodes which were not found.
+Returns a `datanodesResponseMap` keyed by the outcome category. Each value is a `DatanodesResponse`
+(same shape as `GET /api/v1/datanodes`). Categories that have no entries for a given request are
+omitted (not present as empty arrays).
+
+* `removedDatanodes`: successfully removed.
+* `failedDatanodes`: pre-checks failed (e.g. node is not DEAD, or still has open containers/pipelines). Includes `totalCount` and a per-uuid `errors` map describing the failure reason; `datanodes` is empty.
+* `notFoundDatanodes`: uuid did not match any known datanode.
```json
{
- "removedNodes": {
- "totalCount": 1,
- "datanodes": [
- {
- "uuid": "50ca4c95-2ef3-4430-b944-97d2442c3daf",
- "hostname": "ozone-datanode-4.ozone_default",
- "state": "DEAD",
- "pipelines": null
+ "datanodesResponseMap": {
+ "removedDatanodes": {
+ "totalCount": 1,
+ "datanodes": [
+ {
+ "uuid": "50ca4c95-2ef3-4430-b944-97d2442c3daf",
+ "hostname": "ozone-datanode-4.ozone_default",
+ "state": "DEAD"
+ }
+ ]
+ },
+ "failedDatanodes": {
+ "totalCount": 1,
+ "datanodes": [],
+ "errors": {
+ "60ca4c95-...": "Open Containers/Pipelines"
}
- ],
- "message": "Success"
+ }
}
-}
+}
```
-
+
+### GET /api/v1/datanodes/decommission/info
+
+**Parameters**
+
+No parameters.
+
+**Returns**
+
+Returns info for every datanode currently in the `DECOMMISSIONING` state. Each entry wraps the
+datanode details, the per-state container list, and decommission metrics from the SCM JMX bean
+`Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics`.
+
+```json
+{
+ "DatanodesDecommissionInfo": [
+ {
+ "datanodeDetails": {
+ "uuid": "f8f8cb45-3ab2-4123",
+ "hostName": "ozone-datanode-3",
+ "ipAddress": "10.0.0.13",
+ "persistedOpState": "DECOMMISSIONING"
+ },
+ "metrics": {
+ "decommissionStartTime": "2024-05-01T10:00:00Z",
+ "numOfUnclosedContainers": 2,
+ "numOfUnclosedPipelines": 0,
+ "numOfUnderReplicatedContainers": 1
+ },
+ "containers": {
+ "OPEN": ["#1234"],
+ "CLOSED": ["#1235", "#1236"]
+ }
+ }
+ ]
+}
+```
+
+### GET /api/v1/datanodes/decommission/info/datanode
+
+Returns info for a single decommissioning datanode. Provide either `uuid` or `ipAddress`. If both
+are passed, `uuid` wins. Omitting both returns an error.
+
+**Parameters**
+
+* uuid (optional)
+
+ UUID of the decommissioning datanode.
+
+* ipAddress (optional)
+
+ IP address of the decommissioning datanode. Used when `uuid` is not provided.
+
+**Returns**
+
+Same shape as `/api/v1/datanodes/decommission/info`, but the array contains at most one entry.
+
## Pipelines
### GET /api/v1/pipelines
@@ -1124,4 +1488,272 @@ Example: /api/v1/metrics/query?query=ratis_leader_election_electionCount
}
}
```
-
+
+## Storage Distribution (admin only)
+
+### GET /api/v1/storageDistribution
+
+**Parameters**
+
+No parameters.
+
+**Returns**
+
+Aggregated storage capacity distribution across the cluster, including the global storage hierarchy
+(filesystem capacity, Ozone capacity, used/free/reserved/committed space), namespace totals, a
+breakdown of used space (open vs finalized), and per-datanode storage reports.
+
+`500 Internal Server Error` (text/plain body) is returned if the report cannot be produced.
+
+```json
+{
+ "globalStorage": {
+ "totalFileSystemCapacity": 270461374464,
+ "totalReservedSpace": 31457280,
+ "totalOzoneCapacity": 270429917184,
+ "totalOzoneUsedSpace": 358805504,
+ "totalOzoneFreeSpace": 270071111680,
+ "totalOzoneCommittedSpace": 27007111,
+ "totalMinimumFreeSpace": 20480
+ },
+ "globalNamespace": {
+ "totalUsedSpace": 500000000,
+ "totalKeys": 10000
+ },
+ "usedSpaceBreakdown": {
+ "openKeyBytes": {
+ "openKeyAndFileBytes": 13824,
+ "multipartOpenKeyBytes": 4096,
+ "totalOpenKeyBytes": 17920
+ },
+ "finalizedKeyBytes": 450000000
+ },
+ "dataNodeUsage": [
+ {
+ "datanodeUuid": "841be80f-0454-47df-b676",
+ "hostName": "ozone-datanode-1",
+ "capacity": 270429917184,
+ "used": 358805504,
+ "remaining": 270071111680,
+ "committed": 27007111,
+ "minimumFreeSpace": 20480,
+ "reserved": 31457280,
+ "filesystemCapacity": 270461374464,
+ "filesystemUsed": 390262784,
+ "filesystemAvailable": 270071111680
+ }
+ ]
+}
+```
+
+### GET /api/v1/storageDistribution/download
+
+**Parameters**
+
+No parameters.
+
+**Returns**
+
+Triggers or polls a background per-datanode metrics collection. The response varies by collection
+state:
+
+* `200 OK` (`text/csv`) when collection is FINISHED. The CSV columns are HostName, Datanode UUID,
+ Filesystem Capacity, Filesystem Used Space, Filesystem Remaining Space, Ozone Capacity, Ozone Used
+ Space, Ozone Remaining Space, PreAllocated Container Space, Reserved Space, Minimum Free Space,
+ Pending Block Size. A `Content-Disposition: attachment` header carries the file name.
+* `202 Accepted` (`application/json`, body matches `DataNodeMetricsServiceResponse`) when collection
+ is NOT_STARTED or IN_PROGRESS. Poll the endpoint again until status is FINISHED.
+* `500 Internal Server Error` (`text/plain`) if collection is marked FINISHED but the metrics data
+ is missing.
+
+## Pending Deletion (admin only)
+
+### GET /api/v1/pendingDeletion
+
+Returns pending-deletion statistics for one of the three Ozone components.
+
+**Parameters**
+
+* component (required)
+
+ One of `scm`, `om`, `dn`. Selects the source whose pending-deletion data should be returned.
+
+* limit (optional)
+
+ Maximum number of per-datanode entries to return. Only applies when `component=dn`. Must be at
+ least 1.
+
+**Returns**
+
+The response body depends on `component`:
+
+* `component=scm`
+ * `200 OK` with a `ScmPendingDeletion` object (`totalBlocksize`, `totalReplicatedBlockSize`,
+ `totalBlocksCount`).
+ * `204 No Content` if SCM has no pending-deletion summary yet.
+* `component=om`
+ * `200 OK` with a map keyed by category (typical keys: `pendingDirectorySize`,
+ `pendingKeySize`). Values are byte counts.
+* `component=dn`
+ * `200 OK` with a `DataNodeMetricsServiceResponse` body when the background metrics collection
+ has FINISHED.
+ * `202 Accepted` with the same shape while collection is NOT_STARTED or IN_PROGRESS; poll until
+ `status` becomes `FINISHED`.
+
+`400 Bad Request` (text/plain) is returned when `component` is missing/invalid, or when
+`component=dn` and `limit < 1`.
+
+```json
+{
+ "totalBlocksize": 10485760,
+ "totalReplicatedBlockSize": 31457280,
+ "totalBlocksCount": 500
+}
+```
+
+## Heat Map (admin only)
+
+Read-access heatmap data is feature-gated. If the HeatMap feature is listed by
+`/api/v1/features/disabledFeatures`, `/api/v1/heatmap/readaccess` returns `404 Not Found`.
+
+### GET /api/v1/heatmap/readaccess
+
+**Parameters**
+
+* startDate (optional)
+
+ Look-back window for access aggregation. Default `24H`.
+
+* entityType (optional)
+
+ Entity granularity. Default `key`.
+
+* path (optional)
+
+ Restrict the heatmap to this path prefix.
+
+**Returns**
+
+A nested `EntityReadAccessHeatMap` tree. The root represents `/`; children represent volumes, then
+buckets, then directories, then keys. Each node carries `size`, `accessCount`,
+`minAccessCount`/`maxAccessCount`, and a normalized `color` value.
+
+```json
+{
+ "label": "root",
+ "path": "/",
+ "size": 12345678,
+ "accessCount": 1000,
+ "minAccessCount": 0,
+ "maxAccessCount": 250,
+ "color": 0.5,
+ "children": [
+ {
+ "label": "vol1",
+ "path": "/vol1",
+ "size": 8345678,
+ "accessCount": 750,
+ "color": 0.75,
+ "children": []
+ }
+ ]
+}
+```
+
+### GET /api/v1/heatmap/healthCheck
+
+**Returns**
+
+Health-check response from the configured HeatMap provider. The body shape depends on the provider
+implementation.
+
+## Features (admin only)
+
+### GET /api/v1/features/disabledFeatures
+
+**Returns**
+
+JSON array of feature enum names that are currently disabled. The only feature name in use today
+is `HEATMAP`. Useful for the UI to decide whether to show or grey out feature-gated controls.
+
+```json
+["HEATMAP"]
+```
+
+## Admin Utilities (admin only)
+
+### GET /api/v1/triggerdbsync/om
+
+**Returns**
+
+Requests Recon to start an immediate sync from the Ozone Manager DB. Returns a boolean indicating
+whether the sync request was accepted by the OM service provider.
+
+```json
+true
+```
+
+### POST /api/v1/triggerdbsync/scm/snapshot
+
+**Returns**
+
+Starts a one-shot SCM DB snapshot sync in the background. Idempotent. The response always carries
+the current `ScmDbSnapshotSyncStatus` so callers can distinguish "accepted and started" from
+"rejected because another sync is already in progress".
+
+* `202 Accepted`: sync accepted and started. Body has `accepted: true`.
+* `409 Conflict`: another SCM DB sync is already running. Body has `accepted: false`.
+
+```json
+{
+ "accepted": true,
+ "status": "IN_PROGRESS",
+ "message": "SCM DB snapshot sync started."
+}
+```
+
+### GET /api/v1/triggerdbsync/scm/snapshot/status
+
+**Returns**
+
+Current status of the triggered SCM DB snapshot sync. Always returns 200, even when no sync has
+ever run (status will be `IDLE`, phase `NONE`, `startedAt`/`finishedAt` zero).
+
+* `status`: one of `IDLE`, `IN_PROGRESS`, `SUCCESS`, `FAILED`, `CANCELLED`.
+* `phase`: one of `NONE`, `DOWNLOADING_CHECKPOINT`, `INITIALIZING_DB`, `SWAPPING_DB`,
+ `COMPLETED`, `FAILED`, `CANCELLED`.
+* `cancelAllowed`: true only while in `DOWNLOADING_CHECKPOINT`. Once the phase advances to
+ `INITIALIZING_DB`, cancellation is no longer honored.
+* `durationMs`: elapsed time in millis; for a running sync, computed against `now()`.
+
+```json
+{
+ "status": "IN_PROGRESS",
+ "phase": "DOWNLOADING_CHECKPOINT",
+ "startedAt": 1718640123456,
+ "finishedAt": 0,
+ "durationMs": 12345,
+ "cancelAllowed": true,
+ "lastError": null
+}
+```
+
+### POST /api/v1/triggerdbsync/scm/snapshot/cancel
+
+**Returns**
+
+Cancels an in-progress SCM DB snapshot sync. Only honored while `status == IN_PROGRESS` and
+`cancelAllowed == true` (see `/triggerdbsync/scm/snapshot/status`).
+
+* `200 OK`: cancellation accepted and the sync has been cancelled. Body has `cancelled: true`.
+* `409 Conflict`: no sync is running, or the sync has passed the cancellable phase. Body has
+ `cancelled: false` and `message` explains which.
+
+```json
+{
+ "cancelled": true,
+ "status": "CANCELLED",
+ "phase": "CANCELLED",
+ "message": "SCM DB snapshot sync cancelled."
+}
+```
diff --git a/hadoop-hdds/docs/content/interface/S3.md b/hadoop-hdds/docs/content/interface/S3.md
index 1edc89f809d4..b43abb13db59 100644
--- a/hadoop-hdds/docs/content/interface/S3.md
+++ b/hadoop-hdds/docs/content/interface/S3.md
@@ -68,7 +68,6 @@ The Ozone S3 Gateway implements a substantial subset of the Amazon S3 REST API.
| ✅ [CreateBucket](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html) | Creates a new bucket. | **Non-compliant behavior:** The default bucket ACL may include extra group permissions instead of being strictly private. Bucket names must adhere to S3 naming conventions. |
| ✅ [HeadBucket](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadBucket.html) | Checks for the existence of a bucket. | Returns a 200 status if the bucket exists. |
| ✅ [DeleteBucket](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucket.html) | Deletes a bucket. | Bucket must be empty before deletion. |
-| ✅ [GetBucketLocation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLocation.html) | Retrieves the location (region) of a bucket. | Typically returns a default region (e.g., `us-east-1`), which may differ from AWS if region-specific responses are expected. |
### Object Operations
diff --git a/hadoop-hdds/docs/content/tools/Repair.md b/hadoop-hdds/docs/content/tools/Repair.md
index d3368e8b40b8..fdddd47eee15 100644
--- a/hadoop-hdds/docs/content/tools/Repair.md
+++ b/hadoop-hdds/docs/content/tools/Repair.md
@@ -65,7 +65,7 @@ CLI to compact a column-family in the DB while the service is offline.
Note: If om.db is compacted with this tool then it will negatively impact the
Ozone Manager\'s efficient snapshot diff. The corresponding OM, SCM or Datanode
role should be stopped for this tool.
- --cf, --column-family, --column_family=
+ --cf, --column-family=
Column family name
--db= Database File Path
```
@@ -189,7 +189,7 @@ Usage: ozone repair om compact [-hV] [--dry-run] [--force] [--verbose]
[--service-id=]
CLI to compact a column family in the om.db. The compaction happens
asynchronously. Requires admin privileges. OM should be running for this tool.
- --cf, --column-family, --column_family=
+ --cf, --column-family=
Column family name
--node-id= NodeID of the OM for which db needs to be compacted.
--service-id, --om-service-id=
diff --git a/hadoop-hdds/docs/content/tools/TestTools.md b/hadoop-hdds/docs/content/tools/TestTools.md
index bde400d84aef..e678db47ad0d 100644
--- a/hadoop-hdds/docs/content/tools/TestTools.md
+++ b/hadoop-hdds/docs/content/tools/TestTools.md
@@ -27,7 +27,7 @@ Note: we have more tests (like TCP-DS, TCP-H tests via Spark or Hive) which are
## Unit test
-As every almost every java project we have the good old unit tests inside each of our projects.
+As with every Java project, we have the good old unit tests within each of our projects.
## Integration test (JUnit)
@@ -56,20 +56,16 @@ cd compose/ozone
[Blockade](https://github.com/worstcase/blockade) is a tool to test network failures and partitions (it's inspired by the legendary [Jepsen tests](https://jepsen.io/analyses)).
-Blockade tests are implemented with the help of tests and can be started from the `./blockade` directory of the distribution.
+The Blockade test suite is shipped as Python tests under `tests/blockade`. After you build or unpack the distribution, run the commands below from that directory:
```
-cd blockade
-pip install pytest==2.8.7,blockade
+cd tests/blockade
+pip install pytest==2.8.7 blockade
python -m pytest -s .
```
See the README in the blockade directory for more details.
-## MiniChaosOzoneCluster
-
-This is a way to get [chaos](https://en.wikipedia.org/wiki/Chaos_engineering) in your machine. It can be started from the source code and a MiniOzoneCluster (which starts real daemons) will be started and killed randomly.
-
## Freon
Freon is a command line application which is included in the Ozone distribution. It's a load generator which is used in our stress tests.
@@ -82,7 +78,7 @@ The number of volumes/buckets/keys can be configured. The replication type and f
For more information use:
-bin/ozone freon --help
+ozone freon --help
For example:
diff --git a/hadoop-hdds/docs/content/tools/TestTools.zh.md b/hadoop-hdds/docs/content/tools/TestTools.zh.md
index b6b647c90c7b..a528c303764e 100644
--- a/hadoop-hdds/docs/content/tools/TestTools.zh.md
+++ b/hadoop-hdds/docs/content/tools/TestTools.zh.md
@@ -56,21 +56,16 @@ cd compose/ozone
[Blockade](https://github.com/worstcase/blockade) 是一个测试网络故障和分片的工具（灵感来自于大名鼎鼎的[Jepsen 测试](https://jepsen.io/analyses)）。
-Blockade 测试在其它测试的基础上实现，可以在分发包中的 `./blockade` 目录下进行测试。
+Blockade 测试以 Python 脚本形式包含在 `tests/blockade` 目录中。构建或解压发行包后，进入该目录并运行下面的命令：
```
-cd blockade
-pip install pytest==2.8.7,blockade
+cd tests/blockade
+pip install pytest==2.8.7 blockade
python -m pytest -s .
```
更多细节查看 blockade 目录下的 README。
-## MiniChaosOzoneCluster
-
-这是一种在你的机器上获得[混沌](https://en.wikipedia.org/wiki/Chaos_engineering)的方法。它可以直接从源码启动一个 MiniOzoneCluster
-（会启动真实的守护进程），并随机杀死它。
-
## Freon
Freon 是 Ozone 发行包中包含的命令行应用，它是一个负载生成器，用于压力测试。
@@ -83,7 +78,7 @@ volume/bucket/key的数量是可以配置的。副本type和factor(例如: 3个
更多信息，可使用如下命令查看:
-bin/ozone freon --help
+ozone freon --help
例如：
diff --git a/hadoop-hdds/docs/content/tools/_index.md b/hadoop-hdds/docs/content/tools/_index.md
index c44be17effa1..aa708e263b81 100644
--- a/hadoop-hdds/docs/content/tools/_index.md
+++ b/hadoop-hdds/docs/content/tools/_index.md
@@ -54,7 +54,7 @@ Admin commands:
* **classpath** - Prints the class path needed to get the hadoop jar and the
required libraries.
* **dtutil** - Operations related to delegation tokens
- * **envvars** - Display computed Hadoop environment variables.
+ * **envvars** - Display computed Ozone environment variables.
* **getconf** - Reads ozone config values from configuration.
* **genconf** - Generate minimally required ozone configs and output to
ozone-site.xml.
diff --git a/hadoop-hdds/docs/content/tools/debug/Ldb.md b/hadoop-hdds/docs/content/tools/debug/Ldb.md
index a6190b6c6d7a..55a4cbb35416 100644
--- a/hadoop-hdds/docs/content/tools/debug/Ldb.md
+++ b/hadoop-hdds/docs/content/tools/debug/Ldb.md
@@ -74,7 +74,7 @@ Usage: ozone debug ldb scan [--compact] [--count] [--with-keys]
Parse specified metadataTable
--batch-size=
Batch size for processing DB data.
- --cf, --column_family, --column-family=
+ --cf, --column-family=
Table name
--cid, --container-id=
Container ID. Applicable if datanode DB Schema is V3
@@ -82,7 +82,7 @@ Parse specified metadataTable
--count, --show-count
Get estimated key count for the given DB column family
Default: false
- -d, --dnSchema, --dn-schema=
+ -d, --dn-schema=
Datanode DB Schema Version: V1/V2/V3
-e, --ek, --endkey=
Key at which iteration of the DB ends
diff --git a/hadoop-hdds/docs/content/tools/debug/Ldb.zh.md b/hadoop-hdds/docs/content/tools/debug/Ldb.zh.md
index 3f3238dd84bc..85c9765c4321 100644
--- a/hadoop-hdds/docs/content/tools/debug/Ldb.zh.md
+++ b/hadoop-hdds/docs/content/tools/debug/Ldb.zh.md
@@ -101,7 +101,7 @@ Usage: ozone debug ldb scan [--compact] [--count] [--with-keys]
Parse specified metadataTable
--batch-size=
Batch size for processing DB data.
- --cf, --column_family, --column-family=
+ --cf, --column-family=
Table name
--cid, --container-id=
Container ID. Applicable if datanode DB Schema is V3
@@ -109,7 +109,7 @@ Parse specified metadataTable
--count, --show-count
Get estimated key count for the given DB column family
Default: false
- -d, --dnSchema, --dn-schema=