Support configurable default parameters via application config

maestro-engine/src/main/java/com/netflix/maestro/engine/params/DefaultParamManager.java

@@ -19,6 +19,7 @@
 import com.netflix.maestro.models.parameter.ParamDefinition;
 import java.io.IOException;
 import java.net.URL;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
@@ -33,7 +34,17 @@ public class DefaultParamManager {
   private static final String STEP_TYPE_PARAMS_FILE = "defaultparams/default-%s-step-params.yaml";
   private static final String DRY_RUN_PARAMS_FILE = "defaultparams/default-dry-run-params.yaml";
 
+  /** Override key for the default workflow (system) params. */
+  public static final String WORKFLOW_OVERRIDE_KEY = "workflow";
+
+  /** Override key for the default step params. */
+  public static final String STEP_OVERRIDE_KEY = "step";
+
+  /** Override key for the default dry-run params. */
+  public static final String DRY_RUN_OVERRIDE_KEY = "dry-run";
+
   private final ObjectMapper objectMapper;
+  private final Map<String, String> paramOverrides;
   private final TypeReference<Map<String, ParamDefinition>> typeRef = new TypeReference<>() {};
   private Map<String, ParamDefinition> defaultSystemParams;
   private Map<String, ParamDefinition> defaultStepParams;
@@ -46,23 +57,42 @@ public class DefaultParamManager {
    * @param objectMapper object mapper
    */
   public DefaultParamManager(ObjectMapper objectMapper) {
+    this(objectMapper, Collections.emptyMap());
+  }
+
+  /**
+   * Constructor with param overrides supplied from configuration.
+   *
+   * @param objectMapper object mapper
+   * @param paramOverrides map of override key to a YAML blob of param definitions, merged on top of
+   *     the bundled defaults by param name
+   */
+  public DefaultParamManager(ObjectMapper objectMapper, Map<String, String> paramOverrides) {
     this.objectMapper = objectMapper;
+    this.paramOverrides = paramOverrides == null ? Collections.emptyMap() : paramOverrides;
   }
 
   /** Postconstruct initialization for DefaultParamManager. */
   public void init() throws IOException {
-    defaultSystemParams = loadParamsFromFile(WORKFLOW_PARAMS_FILE);
-    defaultStepParams = loadParamsFromFile(NETFLIX_PARAMS_FILE);
-    defaultDryRunParams = loadParamsFromFile(DRY_RUN_PARAMS_FILE);
+    defaultSystemParams =
+        applyOverride(loadParamsFromFile(WORKFLOW_PARAMS_FILE), WORKFLOW_OVERRIDE_KEY);
+    defaultStepParams = applyOverride(loadParamsFromFile(NETFLIX_PARAMS_FILE), STEP_OVERRIDE_KEY);
+    defaultDryRunParams =
+        applyOverride(loadParamsFromFile(DRY_RUN_PARAMS_FILE), DRY_RUN_OVERRIDE_KEY);
     defaultTypeParams = new HashMap<>();
     for (StepType stepType : StepType.values()) {
       String stepName = stepType.toString().toLowerCase(Locale.US);
       String defaultFile = String.format(STEP_TYPE_PARAMS_FILE, stepName);
       try {
+        Map<String, ParamDefinition> typeParams = Collections.emptyMap();
         URL filename = Thread.currentThread().getContextClassLoader().getResource(defaultFile);
         if (filename != null) {
           LOG.info("Loading default param file for {} step", stepName);
-          defaultTypeParams.put(stepName, loadParamsFromFile(defaultFile));
+          typeParams = loadParamsFromFile(defaultFile);
+        }
+        typeParams = applyOverride(typeParams, stepName);
+        if (!typeParams.isEmpty()) {
+          defaultTypeParams.put(stepName, typeParams);
         }
       } catch (Exception e) {
         throw new MaestroRuntimeException("Error processing step default file " + defaultFile, e);
@@ -119,6 +149,22 @@ private Map<String, ParamDefinition> loadParamsFromFile(String paramsFile) throw
         Thread.currentThread().getContextClassLoader().getResourceAsStream(paramsFile), typeRef);
   }
 
+  /**
+   * Merge a configured override blob on top of the bundled defaults, by param name. Overrides
+   * replace matching params and add new ones; params absent from the override are kept as-is.
+   */
+  private Map<String, ParamDefinition> applyOverride(
+      Map<String, ParamDefinition> base, String overrideKey) throws IOException {
+    String blob = paramOverrides.get(overrideKey);
+    if (blob == null || blob.isBlank()) {
+      return base;
+    }
+    LOG.info("Applying default param override for [{}]", overrideKey);
+    Map<String, ParamDefinition> merged = new HashMap<>(base);
+    merged.putAll(objectMapper.readValue(blob, typeRef));
+    return merged;
+  }
+
   private Map<String, ParamDefinition> preprocessParams(Map<String, ParamDefinition> params) {
     if (params == null) {
       return null;

maestro-engine/src/test/java/com/netflix/maestro/engine/params/DefaultParamManagerTest.java

@@ -12,6 +12,7 @@
  */
 package com.netflix.maestro.engine.params;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
@@ -113,4 +114,46 @@ public void testStepTypeParamsMutate() {
         .put("TEST", ParamDefinition.buildParamDefinition("TEST", "123"));
     assertNull(defaultParamManager.getDefaultParamsForType(StepType.FOREACH).get().get("TEST"));
   }
+
+  @Test
+  public void testWorkflowParamOverrideMergesAndAdds() throws IOException {
+    Map<String, String> overrides =
+        Map.of(
+            DefaultParamManager.WORKFLOW_OVERRIDE_KEY,
+            "TARGET_RUN_HOUR:\n"
+                + "  value: OVERRIDDEN\n"
+                + "  type: STRING\n"
+                + "MY_ORG_PARAM:\n"
+                + "  value: hello\n"
+                + "  type: STRING\n");
+    DefaultParamManager manager = new DefaultParamManager(YAML_MAPPER, overrides);
+    manager.init();
+
+    Map<String, ParamDefinition> params = manager.getDefaultWorkflowParams();
+    assertNotNull(params.get("TARGET_RUN_DATE"));
+    assertEquals("OVERRIDDEN", params.get("TARGET_RUN_HOUR").getValue());
+    assertEquals("hello", params.get("MY_ORG_PARAM").getValue());
+  }
+
+  @Test
+  public void testByTypeOverrideAddsParamsForTypeWithoutBundledFile() throws IOException {
+    Map<String, String> overrides =
+        Map.of("titus", "my_titus_param:\n  value: v\n  type: STRING\n");
+    DefaultParamManager manager = new DefaultParamManager(YAML_MAPPER, overrides);
+    manager.init();
+
+    assertTrue(manager.getDefaultParamsForType(StepType.TITUS).isPresent());
+    assertEquals(
+        "v",
+        manager.getDefaultParamsForType(StepType.TITUS).get().get("my_titus_param").getValue());
+  }
+
+  @Test
+  public void testNoOverrideLeavesDefaultsUnchanged() throws IOException {
+    DefaultParamManager manager = new DefaultParamManager(YAML_MAPPER, null);
+    manager.init();
+
+    assertNotNull(manager.getDefaultWorkflowParams().get("TARGET_RUN_HOUR").getName());
+    assertFalse(manager.getDefaultParamsForType(StepType.TITUS).isPresent());
+  }
 }

maestro-server/src/main/java/com/netflix/maestro/server/config/MaestroEngineConfiguration.java

@@ -31,11 +31,15 @@
 import com.netflix.maestro.engine.utils.WorkflowHelper;
 import com.netflix.maestro.engine.validations.DryRunValidator;
 import com.netflix.maestro.models.Constants;
+import com.netflix.maestro.server.properties.DefaultParamsProperties;
 import com.netflix.maestro.server.properties.MaestroProperties;
 import com.netflix.maestro.server.properties.StepRuntimeProperties;
 import com.netflix.maestro.utils.JsonHelper;
 import com.netflix.maestro.utils.StepParamSeparator;
 import com.netflix.spectator.api.DefaultRegistry;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -47,7 +51,11 @@
 /** beans for maestro engine related classes. */
 @Configuration
 @Slf4j
-@EnableConfigurationProperties({MaestroProperties.class, StepRuntimeProperties.class})
+@EnableConfigurationProperties({
+  MaestroProperties.class,
+  StepRuntimeProperties.class,
+  DefaultParamsProperties.class
+})
 public class MaestroEngineConfiguration {
   private static final String OBJECT_MAPPER_WITH_YAML_QUALIFIER = "ObjectMapperWithYaml";
 
@@ -115,9 +123,29 @@ public InstanceStepConcurrencyHandler noopInstanceStepConcurrencyHandler() {
 
   @Bean(initMethod = "init")
   public DefaultParamManager defaultParamManager(
-      @Qualifier(OBJECT_MAPPER_WITH_YAML_QUALIFIER) ObjectMapper objectMapper) {
+      @Qualifier(OBJECT_MAPPER_WITH_YAML_QUALIFIER) ObjectMapper objectMapper,
+      DefaultParamsProperties defaultParamsProperties) {
     LOG.info("Creating DefaultParamManager within Spring boot...");
-    return new DefaultParamManager(objectMapper);
+    return new DefaultParamManager(objectMapper, toParamOverrides(defaultParamsProperties));
+  }
+
+  private static Map<String, String> toParamOverrides(DefaultParamsProperties properties) {
+    Map<String, String> overrides = new HashMap<>();
+    if (properties.getWorkflow() != null) {
+      overrides.put(DefaultParamManager.WORKFLOW_OVERRIDE_KEY, properties.getWorkflow());
+    }
+    if (properties.getStep() != null) {
+      overrides.put(DefaultParamManager.STEP_OVERRIDE_KEY, properties.getStep());
+    }
+    if (properties.getDryRun() != null) {
+      overrides.put(DefaultParamManager.DRY_RUN_OVERRIDE_KEY, properties.getDryRun());
+    }
+    if (properties.getByType() != null) {
+      properties
+          .getByType()
+          .forEach((type, blob) -> overrides.put(type.toLowerCase(Locale.US), blob));
+    }
+    return overrides;
   }
 
   @Bean

maestro-server/src/main/java/com/netflix/maestro/server/properties/DefaultParamsProperties.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2024 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.maestro.server.properties;
+
+import java.util.Map;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+/**
+ * Properties for overriding or extending the bundled default parameters via application config.
+ *
+ * <p>Each field carries a YAML document (as text) with the same shape as the bundled {@code
+ * defaultparams/*.yaml} resources. Values are parsed by the maestro YAML {@code ObjectMapper} and
+ * merged, by param name, on top of the bundled defaults. Absent fields leave the bundled defaults
+ * untouched.
+ */
+@AllArgsConstructor
+@Getter
+@ConfigurationProperties(prefix = "maestro.default-params")
+public class DefaultParamsProperties {
+  /** Override blob for the default workflow (system) params. */
+  private final String workflow;
+
+  /** Override blob for the default step params. */
+  private final String step;
+
+  /** Override blob for the default dry-run params. */
+  private final String dryRun;
+
+  /** Override blobs keyed by step-type name (e.g. {@code subworkflow}, {@code foreach}). */
+  private final Map<String, String> byType;
+}