diff --git a/README.md b/README.md index dccb8cfa25..59e5d06645 100644 --- a/README.md +++ b/README.md @@ -548,27 +548,27 @@ Let's explain the lines one by one. Each section within the `tests.yaml` file contains properties for the environment the defined tests for that section should be conducted in. -| Property | Example value | Description | -|----------------------------------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------| -| `name` | `read_only` | The name of the test suite (overriden by test-specific values) | -| `repetitions` | `1` | The number of times this test suite should run (1 or more) | -| `duration` | `300s` | The amount of time this test suite should run for (45m, 1h, 2d, etc.) | -| `clients` | `1` | The number of loadgenerator workers to use in this test suite (hosted on `loadgenerator_hosts` | -| `members` | `1` | The number of Hazelcast Members to use in this test suite (hosted on `node_hosts`) | -| `loadgenerator_hosts` | `loadgenerators` | Defines the host for Clients, based on either `loadgenerators` or `nodes`, allowing both separate and mixed client/member setups | -| `node_hosts` | `nodes` | Defines the host for Members - this should generally always be `nodes`, and only `loadgenerator_hosts` should be changed for mixed testing. | -| `driver` | `hazelcast5` | The Hazelcast Driver to use - for 5.0+ testing, this is either `hazelcast5` or `hazelcast-enterprise5` for OS or EE respectively | -| `version` | `maven=5.1` | The Hazelcast version to use - typically provided by maven, i.e. `maven=5.3.0-SNAPSHOT` | -| `client_args` | `-Xms3g -Xmx3g` | The command-line Java parameters passed to all clients in this test suite | -| `member_args` | `-Xms3g -Xmx3g` | The command-line Java parameters passed to all members in this test suite | -| `performance_monitor_interval_seconds` | `1` | The interval of the Simulator performance monitor | -| `verify_enabled` | `True` | Defines whether tests should be verified after completion or not (default true) | -| `warmup_seconds` | `0` | The number of seconds from the start of the test to exclude in reporting (only used for report generation) | -| `cooldown_seconds` | `0` | The number of seconds before the end of the test to exclude in reporting (only used for report generation) | -| `license_key` | `your_ee_key` | The Hazelcast Enterprise Edition license to use in your test, if using `hazelcast-enterprise5` drivers | -| `parallel` | `True` | Defines whether tests should be run in parallel when multiple tests are defined within 1 suite (default false) | -| `cp_priorities` | 
- address: internalIp
  priority: 1
| Defines the leadership priority of the CP Subsystem members in the cluster. Use the internal IP address of the agent(s) you wish to configure. | -| `clients_per_loadgenerator` | `1` | The number of Hazelcast client instances per loadgenerator worker (default 1) | +| Property | Example value | Description | +|----------------------------------------|--------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `name` | `read_only` | The name of the test suite (overriden by test-specific values) | +| `repetitions` | `1` | The number of times this test suite should run (1 or more) | +| `duration` | `300s` | The amount of time this test suite should run for (45m, 1h, 2d, etc.) | +| `clients` | `1` | The number of loadgenerator workers to use in this test suite (hosted on `loadgenerator_hosts` | +| `members` | `1` | The number of Hazelcast Members to use in this test suite (hosted on `node_hosts`) | +| `loadgenerator_hosts` | `loadgenerators` | Defines the host for Clients, based on either `loadgenerators` or `nodes`, allowing both separate and mixed client/member setups | +| `node_hosts` | `nodes` | Defines the host for Members - this should generally always be `nodes`, and only `loadgenerator_hosts` should be changed for mixed testing. | +| `driver` | `hazelcast5` | The Hazelcast Driver to use - for 5.0+ testing, this is either `hazelcast5` or `hazelcast-enterprise5` for OS or EE respectively | +| `version` | `maven=5.1` | The Hazelcast version to use - typically provided by maven, i.e. `maven=5.3.0-SNAPSHOT` | +| `client_args` | `-Xms3g -Xmx3g` | The command-line Java parameters passed to all clients in this test suite | +| `member_args` | `-Xms3g -Xmx3g` | The command-line Java parameters passed to all members in this test suite | +| `performance_monitor_interval_seconds` | `1` | The interval of the Simulator performance monitor | +| `verify_enabled` | `True` | Defines whether tests should be verified after completion or not (default true) | +| `warmup_seconds` | `0` | The number of seconds from the start of the test to exclude in reporting (only used for report generation) | +| `cooldown_seconds` | `0` | The number of seconds before the end of the test to exclude in reporting (only used for report generation) | +| `license_key` | `your_ee_key` | The Hazelcast Enterprise Edition license to use in your test, if using `hazelcast-enterprise5` drivers. License key can be also set using `HZ_LICENSEKEY` environment variable. | +| `parallel` | `True` | Defines whether tests should be run in parallel when multiple tests are defined within 1 suite (default false) | +| `cp_priorities` | 
- address: internalIp
  priority: 1
| Defines the leadership priority of the CP Subsystem members in the cluster. Use the internal IP address of the agent(s) you wish to configure. | +| `clients_per_loadgenerator` | `1` | The number of Hazelcast client instances per loadgenerator worker (default 1) | ### Specify test class(es) and number of threads per worker @@ -580,6 +580,7 @@ properties shown in the above example are built-in "magic" properties of Simulat | `class` | `com.hazelcast.simulator.tests.map.IntByteMapTest` | Defines the fully qualified class name for the Simulator Test. Used to create the test class instance on the Simulator Worker. This is the only mandatory property which has to be defined. | | `name` | `MyByteTest` | Defines a unique name for this Test. This property is only required when running multiple tests on the same test class, without it only 1 test will run per class type (as the class name is used as the name if not defined here). The tests are ordered alphabetically by name. | | `threadCount` | `40` | Defines how many threads are running the Test methods in parallel. In other words, defines the number of worker threads for Simulator Tests which use the `@RunWithWorker` annotation. | +| `sequential` | `true` | By default test operations are chosen randomly based on probabilities (`sequential` not configured or `false`). Sequential mode causes test operations to be executed in round-robin, deterministic order, sorted alphabetically. | > :books: For details about available values for `class`, refer to the provided classes in the [drivers](java/drivers) > directory or the [Writing a Simulator test](#writing-a-simulator-test) section. diff --git a/java/drivers/driver-hazelcast4plus/conf/configure.py b/java/drivers/driver-hazelcast4plus/conf/configure.py index dd5a9917cf..09e1681b3b 100644 --- a/java/drivers/driver-hazelcast4plus/conf/configure.py +++ b/java/drivers/driver-hazelcast4plus/conf/configure.py @@ -17,7 +17,7 @@ def _configure_hazelcast_xml(nodes, args: DriverConfigureArgs, is_lite_member: b config = read_file(src_file) # configure - license_key = args.test.get("license_key") + license_key = args.test.get("license_key") or os.getenv("HZ_LICENSEKEY") if license_key is not None: license_key_config = f"{license_key}" config = config.replace("", license_key_config) diff --git a/java/drivers/driver-hazelcast4plus/src/main/java/com/hazelcast/simulator/tests/map/LongByteArrayMapTest.java b/java/drivers/driver-hazelcast4plus/src/main/java/com/hazelcast/simulator/tests/map/LongByteArrayMapTest.java index 82c88457d3..d26eddfdb6 100644 --- a/java/drivers/driver-hazelcast4plus/src/main/java/com/hazelcast/simulator/tests/map/LongByteArrayMapTest.java +++ b/java/drivers/driver-hazelcast4plus/src/main/java/com/hazelcast/simulator/tests/map/LongByteArrayMapTest.java @@ -153,6 +153,24 @@ public void deleteAndPut(ThreadState state) { map.put(key, state.randomValue()); } + /** + * {@link #deleteNewCurrent(ThreadState)} and {@link #setCurrent(ThreadState)} form a sequential scenario, + * that ensures that entry exists before delete (if filled during setup) and does not exist before insertion. + */ + @TimeStep(prob = 0) + public void deleteNewCurrent(ThreadState state) { + var map = getRandomMap(); + long key = state.newRandomCurrentKey(); + map.delete(key); + } + + @TimeStep(prob = 0) + public void setCurrent(ThreadState state) { + var map = getRandomMap(); + long key = state.currentKey(); + map.set(key, state.randomValue()); + } + @TimeStep(prob = 0.0) public CompletableFuture putAsync(ThreadState state) { return getRandomMap().putAsync(state.randomKey(), state.randomValue()).toCompletableFuture(); @@ -204,6 +222,7 @@ public class ThreadState extends BaseThreadState { public static final int HIGHEST_PROBABILITY = 100; private Pipelining pipeline; private int i; + private long currentKey = randomKey(); private long fixedKeyOrRandom() { if (fixedKeyDomain > 0 && fixedKeyDomain < keyDomain && fixedKeyProbability > 0 && @@ -220,6 +239,14 @@ private long randomKey() { private byte[] randomValue() { return values[randomInt(values.length)]; } + + public long currentKey() { + return currentKey; + } + + public long newRandomCurrentKey() { + return currentKey = randomKey(); + } } private static final class UpdateEntryProcessor implements EntryProcessor { diff --git a/java/pom.xml b/java/pom.xml index ff85cd0ffe..e21d6d905c 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -169,6 +169,12 @@ ${mockito.version} test + + org.assertj + assertj-core + 3.27.7 + test + diff --git a/java/simulator/src/main/java/com/hazelcast/simulator/worker/testcontainer/TimeStepLoopCodeGenerator.java b/java/simulator/src/main/java/com/hazelcast/simulator/worker/testcontainer/TimeStepLoopCodeGenerator.java index 6be9222ed2..2019fc7ae3 100644 --- a/java/simulator/src/main/java/com/hazelcast/simulator/worker/testcontainer/TimeStepLoopCodeGenerator.java +++ b/java/simulator/src/main/java/com/hazelcast/simulator/worker/testcontainer/TimeStepLoopCodeGenerator.java @@ -42,6 +42,7 @@ import java.net.URL; import java.net.URLClassLoader; import java.security.PrivilegedAction; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -66,7 +67,8 @@ Class compile( Class probeClass, long logFrequency, long logRateMs, - boolean hasIterationCap) { + boolean hasIterationCap, + boolean sequential) { ensureExistingDirectory(targetDirectory); @@ -81,7 +83,7 @@ Class compile( className += testCaseId; } JavaFileObject file = createJavaFileObject( - className, executionGroup, metronomeClass, timeStepModel, probeClass, logFrequency, logRateMs, hasIterationCap); + className, executionGroup, metronomeClass, timeStepModel, probeClass, logFrequency, logRateMs, hasIterationCap, sequential); return compile(javaCompiler, file, className); } @@ -123,7 +125,8 @@ Class compile(JavaCompiler compiler, JavaFileObject file, final String className }); } - private JavaFileObject createJavaFileObject( + // visible for tests + JavaFileObject createJavaFileObject( String className, String executionGroup, Class metronomeClass, @@ -131,7 +134,8 @@ private JavaFileObject createJavaFileObject( Class probeClass, long logFrequency, long logRateMs, - boolean hasIterationCap) { + boolean hasIterationCap, + boolean sequential) { try { Configuration cfg = new Configuration(Configuration.VERSION_2_3_24); cfg.setClassForTemplateLoading(this.getClass(), "/"); @@ -142,7 +146,11 @@ private JavaFileObject createJavaFileObject( Map root = new HashMap<>(); root.put("testInstanceClass", getClassName(timeStepModel.getTestClass())); root.put("metronomeClass", getMetronomeClass(metronomeClass)); - root.put("timeStepMethods", timeStepModel.getActiveTimeStepMethods(executionGroup)); + List activeTimeStepMethods = timeStepModel.getActiveTimeStepMethods(executionGroup); + if (sequential) { + activeTimeStepMethods.sort(Comparator.comparing(Method::getName)); + } + root.put("timeStepMethods", activeTimeStepMethods); root.put("probeClass", getClassName(probeClass)); root.put("isStartNanos", new IsStartNanos(timeStepModel)); root.put("isAssignableFrom", new IsAssignableFromMethod()); @@ -162,6 +170,9 @@ private JavaFileObject createJavaFileObject( if (hasIterationCap) { root.put("hasIterationCap", "true"); } + if (sequential) { + root.put("sequential", "true"); + } Template temp = cfg.getTemplate("TimeStepLoop.ftl"); StringWriter out = new StringWriter(); diff --git a/java/simulator/src/main/java/com/hazelcast/simulator/worker/testcontainer/TimeStepRunner.java b/java/simulator/src/main/java/com/hazelcast/simulator/worker/testcontainer/TimeStepRunner.java index 3c8333b96f..308cf149d9 100644 --- a/java/simulator/src/main/java/com/hazelcast/simulator/worker/testcontainer/TimeStepRunner.java +++ b/java/simulator/src/main/java/com/hazelcast/simulator/worker/testcontainer/TimeStepRunner.java @@ -79,6 +79,8 @@ class TimeStepRunner extends TestRunner { long iterations = binding.loadAsLong(toPropertyName(executionGroup, "iterations"), 0); runIterationMap.put(executionGroup, iterations); + boolean sequential = binding.loadAsBoolean(toPropertyName(executionGroup, "sequential"), false); + Class loopClass = new TimeStepLoopCodeGenerator().compile( testContainer.getTestCase().getId(), executionGroup, @@ -87,7 +89,8 @@ class TimeStepRunner extends TestRunner { binding.getProbeClass(), logFrequency, logRateMs, - iterations > 0); + iterations > 0, + sequential); loopClassMap.put(executionGroup, loopClass); } diff --git a/java/simulator/src/main/resources/TimeStepLoop.ftl b/java/simulator/src/main/resources/TimeStepLoop.ftl index b8ef9cc192..8405fdaff9 100644 --- a/java/simulator/src/main/resources/TimeStepLoop.ftl +++ b/java/simulator/src/main/resources/TimeStepLoop.ftl @@ -46,7 +46,7 @@ public class ${className} extends TimeStepLoop { final ThrottlingLogger throttlingLogger = new ThrottlingLogger(logger, ${logRateMs}); -<#if timeStepMethods?size gt 1> +<#if timeStepMethods?size gt 1 && !sequential??> final byte[] probs = this.timeStepProbabilities; @@ -82,8 +82,11 @@ public class ${className} extends TimeStepLoop { ${method.name}Probe.recordValue(System.nanoTime() - startNanos); <#else> - + <#if sequential??> + switch((int) (iteration % ${timeStepMethods?size})){ + <#else> switch(probs[random.nextInt(probs.length)]){ + <#list timeStepMethods as method> <#assign index = method?counter-1> case ${index}: diff --git a/java/simulator/src/test/java/com/hazelcast/simulator/worker/testcontainer/TimeStepLoopCodeGeneratorTest.java b/java/simulator/src/test/java/com/hazelcast/simulator/worker/testcontainer/TimeStepLoopCodeGeneratorTest.java index 2006e130dc..b7942ac7bb 100644 --- a/java/simulator/src/test/java/com/hazelcast/simulator/worker/testcontainer/TimeStepLoopCodeGeneratorTest.java +++ b/java/simulator/src/test/java/com/hazelcast/simulator/worker/testcontainer/TimeStepLoopCodeGeneratorTest.java @@ -1,14 +1,29 @@ package com.hazelcast.simulator.worker.testcontainer; +import com.hazelcast.simulator.common.TestCase; +import com.hazelcast.simulator.test.annotations.TimeStep; +import com.hazelcast.simulator.worker.metronome.EmptyMetronome; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.tools.JavaFileObject; +import javax.tools.ToolProvider; +import java.io.IOException; +import java.util.Map; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; import static org.mockito.Mockito.mock; public class TimeStepLoopCodeGeneratorTest { - private TimeStepLoopCodeGenerator codeGenerator = new TimeStepLoopCodeGenerator(); + private static final String RANDOM_SELECTOR = "probs[random.nextInt(probs.length)]"; + private static final String SEQUENTIAL_SELECTOR = "iteration % 2"; + private static final String SEQUENTIAL_SELECTOR_3 = "iteration % 3"; + private static final Logger log = LoggerFactory.getLogger(TimeStepLoopCodeGeneratorTest.class); + + private final TimeStepLoopCodeGenerator codeGenerator = new TimeStepLoopCodeGenerator(); @Test(expected = IllegalStateException.class) public void testCompile_whenCompilerIsNull_thenThrowIllegalStateException() { @@ -16,4 +31,87 @@ public void testCompile_whenCompilerIsNull_thenThrowIllegalStateException() { codeGenerator.compile(null, javaFileObject, "className"); } + + public static class TestClass1 { + @TimeStep(prob = 0) + public void method3() { + } + + @TimeStep(prob = 0) + public void method1() { + } + + @TimeStep(prob = 0) + public void method2() { + } + } + + @Test + public void testGenerateSingleMethodRandom() throws IOException { + testGenerateSingleMethod(false); + } + + @Test + public void testGenerateSingleMethodSequential() throws IOException { + testGenerateSingleMethod(true); + } + + private void testGenerateSingleMethod(boolean sequential) throws IOException { + var generatedCode = generateTestClass(Map.of("method1Prob", "1"), sequential); + + assertThat(generatedCode) + .as("should not select methods") + .doesNotContain(RANDOM_SELECTOR).doesNotContain(SEQUENTIAL_SELECTOR) + .as("should invoke configured method") + .contains("method1( )").doesNotContain("method2( )"); + } + + @Test + public void testGenerateRandom() throws IOException { + var generatedCode = generateTestClass(Map.of("method1Prob", "0.5", "method2Prob", "0.5"), false); + + assertThat(generatedCode) + .as("should invoke methods in sequence") + .contains(RANDOM_SELECTOR) + .as("should invoke all methods") + .contains("method1( )", "method2( )"); + } + + @Test + public void testGenerateSequential() throws IOException { + var generatedCode = generateTestClass(Map.of("method1Prob", "0.5", "method2Prob", "0.5"), true); + + assertThat(generatedCode) + .as("should invoke methods in sequence") + .contains(SEQUENTIAL_SELECTOR) + .as("should sort methods alphabetically") + .containsSubsequence("method1( )", "method2( )"); + } + + @Test + public void testGenerateSequential_3methods() throws IOException { + var generatedCode = generateTestClass(Map.of("method1Prob", "0.5", "method2Prob", "0.25", "method3Prob", "0.25"), true); + + assertThat(generatedCode) + .as("should invoke methods in sequence") + .contains(SEQUENTIAL_SELECTOR_3) + .as("should sort methods alphabetically") + .containsSubsequence("method1( )", "method2( )", "method3( )"); + } + + private CharSequence generateTestClass(Map properties, boolean sequential) throws IOException { + // minimal scaffolding required for code generation + var testCase = new TestCase("someTest", properties); + var model = new TimeStepModel(TestClass1.class, new PropertyBinding(testCase)); + + var generatedCode = codeGenerator.createJavaFileObject(model.getTestClass().getSimpleName() + "Loop", "", EmptyMetronome.class, + model, null, 0, 0, false, sequential); + log.info("generated code: {}", generatedCode.getCharContent(false)); + + assertThatNoException() + .as("generated code should be valid") + .isThrownBy(() -> codeGenerator.compile(ToolProvider.getSystemJavaCompiler(), generatedCode, model.getTestClass().getName())); + + return generatedCode.getCharContent(false); + } } diff --git a/java/simulator/src/test/resources/log4j.xml b/java/simulator/src/test/resources/log4j.xml deleted file mode 100644 index b79b661c1e..0000000000 --- a/java/simulator/src/test/resources/log4j.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - - - diff --git a/java/simulator/src/test/resources/log4j2.xml b/java/simulator/src/test/resources/log4j2.xml new file mode 100644 index 0000000000..8731b5a24f --- /dev/null +++ b/java/simulator/src/test/resources/log4j2.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file