diff --git a/maestro-engine/src/main/java/com/netflix/maestro/engine/params/DefaultParamManager.java b/maestro-engine/src/main/java/com/netflix/maestro/engine/params/DefaultParamManager.java index 03022b5f..09078ea9 100644 --- a/maestro-engine/src/main/java/com/netflix/maestro/engine/params/DefaultParamManager.java +++ b/maestro-engine/src/main/java/com/netflix/maestro/engine/params/DefaultParamManager.java @@ -19,6 +19,7 @@ import com.netflix.maestro.models.parameter.ParamDefinition; import java.io.IOException; import java.net.URL; +import java.util.Collections; import java.util.HashMap; import java.util.Locale; import java.util.Map; @@ -33,7 +34,17 @@ public class DefaultParamManager { private static final String STEP_TYPE_PARAMS_FILE = "defaultparams/default-%s-step-params.yaml"; private static final String DRY_RUN_PARAMS_FILE = "defaultparams/default-dry-run-params.yaml"; + /** Override key for the default workflow (system) params. */ + public static final String WORKFLOW_OVERRIDE_KEY = "workflow"; + + /** Override key for the default step params. */ + public static final String STEP_OVERRIDE_KEY = "step"; + + /** Override key for the default dry-run params. */ + public static final String DRY_RUN_OVERRIDE_KEY = "dry-run"; + private final ObjectMapper objectMapper; + private final Map paramOverrides; private final TypeReference> typeRef = new TypeReference<>() {}; private Map defaultSystemParams; private Map defaultStepParams; @@ -46,23 +57,42 @@ public class DefaultParamManager { * @param objectMapper object mapper */ public DefaultParamManager(ObjectMapper objectMapper) { + this(objectMapper, Collections.emptyMap()); + } + + /** + * Constructor with param overrides supplied from configuration. + * + * @param objectMapper object mapper + * @param paramOverrides map of override key to a YAML blob of param definitions, merged on top of + * the bundled defaults by param name + */ + public DefaultParamManager(ObjectMapper objectMapper, Map paramOverrides) { this.objectMapper = objectMapper; + this.paramOverrides = paramOverrides == null ? Collections.emptyMap() : paramOverrides; } /** Postconstruct initialization for DefaultParamManager. */ public void init() throws IOException { - defaultSystemParams = loadParamsFromFile(WORKFLOW_PARAMS_FILE); - defaultStepParams = loadParamsFromFile(NETFLIX_PARAMS_FILE); - defaultDryRunParams = loadParamsFromFile(DRY_RUN_PARAMS_FILE); + defaultSystemParams = + applyOverride(loadParamsFromFile(WORKFLOW_PARAMS_FILE), WORKFLOW_OVERRIDE_KEY); + defaultStepParams = applyOverride(loadParamsFromFile(NETFLIX_PARAMS_FILE), STEP_OVERRIDE_KEY); + defaultDryRunParams = + applyOverride(loadParamsFromFile(DRY_RUN_PARAMS_FILE), DRY_RUN_OVERRIDE_KEY); defaultTypeParams = new HashMap<>(); for (StepType stepType : StepType.values()) { String stepName = stepType.toString().toLowerCase(Locale.US); String defaultFile = String.format(STEP_TYPE_PARAMS_FILE, stepName); try { + Map typeParams = Collections.emptyMap(); URL filename = Thread.currentThread().getContextClassLoader().getResource(defaultFile); if (filename != null) { LOG.info("Loading default param file for {} step", stepName); - defaultTypeParams.put(stepName, loadParamsFromFile(defaultFile)); + typeParams = loadParamsFromFile(defaultFile); + } + typeParams = applyOverride(typeParams, stepName); + if (!typeParams.isEmpty()) { + defaultTypeParams.put(stepName, typeParams); } } catch (Exception e) { throw new MaestroRuntimeException("Error processing step default file " + defaultFile, e); @@ -119,6 +149,22 @@ private Map loadParamsFromFile(String paramsFile) throw Thread.currentThread().getContextClassLoader().getResourceAsStream(paramsFile), typeRef); } + /** + * Merge a configured override blob on top of the bundled defaults, by param name. Overrides + * replace matching params and add new ones; params absent from the override are kept as-is. + */ + private Map applyOverride( + Map base, String overrideKey) throws IOException { + String blob = paramOverrides.get(overrideKey); + if (blob == null || blob.isBlank()) { + return base; + } + LOG.info("Applying default param override for [{}]", overrideKey); + Map merged = new HashMap<>(base); + merged.putAll(objectMapper.readValue(blob, typeRef)); + return merged; + } + private Map preprocessParams(Map params) { if (params == null) { return null; diff --git a/maestro-engine/src/test/java/com/netflix/maestro/engine/params/DefaultParamManagerTest.java b/maestro-engine/src/test/java/com/netflix/maestro/engine/params/DefaultParamManagerTest.java index 6b18f007..31526a59 100644 --- a/maestro-engine/src/test/java/com/netflix/maestro/engine/params/DefaultParamManagerTest.java +++ b/maestro-engine/src/test/java/com/netflix/maestro/engine/params/DefaultParamManagerTest.java @@ -12,6 +12,7 @@ */ package com.netflix.maestro.engine.params; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -113,4 +114,46 @@ public void testStepTypeParamsMutate() { .put("TEST", ParamDefinition.buildParamDefinition("TEST", "123")); assertNull(defaultParamManager.getDefaultParamsForType(StepType.FOREACH).get().get("TEST")); } + + @Test + public void testWorkflowParamOverrideMergesAndAdds() throws IOException { + Map overrides = + Map.of( + DefaultParamManager.WORKFLOW_OVERRIDE_KEY, + "TARGET_RUN_HOUR:\n" + + " value: OVERRIDDEN\n" + + " type: STRING\n" + + "MY_ORG_PARAM:\n" + + " value: hello\n" + + " type: STRING\n"); + DefaultParamManager manager = new DefaultParamManager(YAML_MAPPER, overrides); + manager.init(); + + Map params = manager.getDefaultWorkflowParams(); + assertNotNull(params.get("TARGET_RUN_DATE")); + assertEquals("OVERRIDDEN", params.get("TARGET_RUN_HOUR").getValue()); + assertEquals("hello", params.get("MY_ORG_PARAM").getValue()); + } + + @Test + public void testByTypeOverrideAddsParamsForTypeWithoutBundledFile() throws IOException { + Map overrides = + Map.of("titus", "my_titus_param:\n value: v\n type: STRING\n"); + DefaultParamManager manager = new DefaultParamManager(YAML_MAPPER, overrides); + manager.init(); + + assertTrue(manager.getDefaultParamsForType(StepType.TITUS).isPresent()); + assertEquals( + "v", + manager.getDefaultParamsForType(StepType.TITUS).get().get("my_titus_param").getValue()); + } + + @Test + public void testNoOverrideLeavesDefaultsUnchanged() throws IOException { + DefaultParamManager manager = new DefaultParamManager(YAML_MAPPER, null); + manager.init(); + + assertNotNull(manager.getDefaultWorkflowParams().get("TARGET_RUN_HOUR").getName()); + assertFalse(manager.getDefaultParamsForType(StepType.TITUS).isPresent()); + } } diff --git a/maestro-server/src/main/java/com/netflix/maestro/server/config/MaestroEngineConfiguration.java b/maestro-server/src/main/java/com/netflix/maestro/server/config/MaestroEngineConfiguration.java index e21ec335..1a21265f 100644 --- a/maestro-server/src/main/java/com/netflix/maestro/server/config/MaestroEngineConfiguration.java +++ b/maestro-server/src/main/java/com/netflix/maestro/server/config/MaestroEngineConfiguration.java @@ -31,11 +31,15 @@ import com.netflix.maestro.engine.utils.WorkflowHelper; import com.netflix.maestro.engine.validations.DryRunValidator; import com.netflix.maestro.models.Constants; +import com.netflix.maestro.server.properties.DefaultParamsProperties; import com.netflix.maestro.server.properties.MaestroProperties; import com.netflix.maestro.server.properties.StepRuntimeProperties; import com.netflix.maestro.utils.JsonHelper; import com.netflix.maestro.utils.StepParamSeparator; import com.netflix.spectator.api.DefaultRegistry; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -47,7 +51,11 @@ /** beans for maestro engine related classes. */ @Configuration @Slf4j -@EnableConfigurationProperties({MaestroProperties.class, StepRuntimeProperties.class}) +@EnableConfigurationProperties({ + MaestroProperties.class, + StepRuntimeProperties.class, + DefaultParamsProperties.class +}) public class MaestroEngineConfiguration { private static final String OBJECT_MAPPER_WITH_YAML_QUALIFIER = "ObjectMapperWithYaml"; @@ -115,9 +123,29 @@ public InstanceStepConcurrencyHandler noopInstanceStepConcurrencyHandler() { @Bean(initMethod = "init") public DefaultParamManager defaultParamManager( - @Qualifier(OBJECT_MAPPER_WITH_YAML_QUALIFIER) ObjectMapper objectMapper) { + @Qualifier(OBJECT_MAPPER_WITH_YAML_QUALIFIER) ObjectMapper objectMapper, + DefaultParamsProperties defaultParamsProperties) { LOG.info("Creating DefaultParamManager within Spring boot..."); - return new DefaultParamManager(objectMapper); + return new DefaultParamManager(objectMapper, toParamOverrides(defaultParamsProperties)); + } + + private static Map toParamOverrides(DefaultParamsProperties properties) { + Map overrides = new HashMap<>(); + if (properties.getWorkflow() != null) { + overrides.put(DefaultParamManager.WORKFLOW_OVERRIDE_KEY, properties.getWorkflow()); + } + if (properties.getStep() != null) { + overrides.put(DefaultParamManager.STEP_OVERRIDE_KEY, properties.getStep()); + } + if (properties.getDryRun() != null) { + overrides.put(DefaultParamManager.DRY_RUN_OVERRIDE_KEY, properties.getDryRun()); + } + if (properties.getByType() != null) { + properties + .getByType() + .forEach((type, blob) -> overrides.put(type.toLowerCase(Locale.US), blob)); + } + return overrides; } @Bean diff --git a/maestro-server/src/main/java/com/netflix/maestro/server/properties/DefaultParamsProperties.java b/maestro-server/src/main/java/com/netflix/maestro/server/properties/DefaultParamsProperties.java new file mode 100644 index 00000000..2a3f11b6 --- /dev/null +++ b/maestro-server/src/main/java/com/netflix/maestro/server/properties/DefaultParamsProperties.java @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.maestro.server.properties; + +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Properties for overriding or extending the bundled default parameters via application config. + * + *

Each field carries a YAML document (as text) with the same shape as the bundled {@code + * defaultparams/*.yaml} resources. Values are parsed by the maestro YAML {@code ObjectMapper} and + * merged, by param name, on top of the bundled defaults. Absent fields leave the bundled defaults + * untouched. + */ +@AllArgsConstructor +@Getter +@ConfigurationProperties(prefix = "maestro.default-params") +public class DefaultParamsProperties { + /** Override blob for the default workflow (system) params. */ + private final String workflow; + + /** Override blob for the default step params. */ + private final String step; + + /** Override blob for the default dry-run params. */ + private final String dryRun; + + /** Override blobs keyed by step-type name (e.g. {@code subworkflow}, {@code foreach}). */ + private final Map byType; +}