From 80460e71d3a927c18767d9e927474b99c1791234 Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 10 Nov 2025 02:40:50 +0100 Subject: [PATCH 01/39] Add JSON source infrastructure and Markov factory --- build.gradle | 2 + .../io/connectors/JsonFileConnector.java | 53 +++ .../markov/MarkovLoadModelFactory.java | 317 ++++++++++++++++++ .../io/factory/markov/MarkovModelData.java | 26 ++ .../edu/ie3/datamodel/io/file/FileType.java | 3 +- .../io/source/json/JsonDataSource.java | 142 ++++++++ .../source/json/JsonMarkovProfileSource.java | 117 +++++++ .../profile/markov/MarkovLoadModel.java | 63 ++++ 8 files changed, 722 insertions(+), 1 deletion(-) create mode 100644 src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java create mode 100644 src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java create mode 100644 src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java create mode 100644 src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java create mode 100644 src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java create mode 100644 src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java diff --git a/build.gradle b/build.gradle index bf0966b5ae..e420517cde 100644 --- a/build.gradle +++ b/build.gradle @@ -105,6 +105,8 @@ dependencies { implementation 'commons-io:commons-io:2.20.0' // I/O functionalities implementation 'commons-codec:commons-codec:1.20.0' // needed by commons-compress implementation 'org.apache.commons:commons-compress:1.28.0' // I/O functionalities + + implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.2' } tasks.withType(JavaCompile) { diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java new file mode 100644 index 0000000000..40764d5511 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java @@ -0,0 +1,53 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.connectors; + +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.function.Function; + +/** Connector for JSON-based sources and sinks. */ +public class JsonFileConnector extends FileConnector { + private static final String FILE_ENDING = ".json"; + + public JsonFileConnector(Path baseDirectory) { + super(baseDirectory); + } + + public JsonFileConnector(Path baseDirectory, Function customInputStream) { + super(baseDirectory, customInputStream); + } + + /** + * Opens a buffered reader for the given JSON file, using UTF-8 decoding. + * + * @param filePath relative path without ending + * @return buffered reader referencing the JSON file + */ + public BufferedReader initReader(Path filePath) throws FileNotFoundException { + InputStream inputStream = openInputStream(filePath); + return new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8), 16384); + } + + /** + * Opens an input stream for the given JSON file. + * + * @param filePath relative path without ending + * @return input stream for the file + */ + public InputStream initInputStream(Path filePath) throws FileNotFoundException { + return openInputStream(filePath); + } + + @Override + protected String getFileEnding() { + return FILE_ENDING; + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java new file mode 100644 index 0000000000..aeabdffe2a --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -0,0 +1,317 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.factory.markov; + +import com.fasterxml.jackson.databind.JsonNode; +import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.io.factory.Factory; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.*; +import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; + +/** Factory turning Markov JSON data into {@link MarkovLoadModel}s. */ +public class MarkovLoadModelFactory + extends Factory { + + public MarkovLoadModelFactory() { + super(MarkovLoadModel.class); + } + + @Override + protected MarkovLoadModel buildModel(MarkovModelData data) { + JsonNode root = data.getRoot(); + String schema = requireText(root, "schema"); + ZonedDateTime generatedAt = parseTimestamp(requireText(root, "generated_at")); + Generator generator = parseGenerator(requireNode(root, "generator")); + TimeModel timeModel = parseTimeModel(requireNode(root, "time_model")); + ValueModel valueModel = parseValueModel(requireNode(root, "value_model")); + Parameters parameters = parseParameters(root.path("parameters")); + + JsonNode dataNode = requireNode(root, "data"); + TransitionData transitionData = + parseTransitions(dataNode, timeModel.bucketCount(), valueModel.discretization().states()); + Optional gmmBuckets = parseGmmBuckets(dataNode.path("gmms")); + + return new MarkovLoadModel( + schema, + generatedAt, + generator, + timeModel, + valueModel, + parameters, + transitionData, + gmmBuckets); + } + + @Override + protected List> getFields(Class entityClass) { + Set requiredFields = + newSet( + "schema", + "generated_at", + "generator.name", + "generator.version", + "time_model.bucket_count", + "time_model.bucket_encoding.formula", + "time_model.sampling_interval_minutes", + "time_model.timezone", + "value_model.value_unit", + "value_model.normalization.method", + "value_model.discretization.states", + "value_model.discretization.thresholds_right", + "data.transitions.shape", + "data.transitions.values", + "data.gmms.buckets"); + return List.of(requiredFields); + } + + private static Generator parseGenerator(JsonNode generatorNode) { + String name = requireText(generatorNode, "name"); + String version = requireText(generatorNode, "version"); + Map config = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + JsonNode configNode = generatorNode.path("config"); + if (configNode.isObject()) { + Iterator> fields = configNode.fields(); + while (fields.hasNext()) { + Map.Entry entry = fields.next(); + config.put(entry.getKey(), entry.getValue().asText()); + } + } + return new Generator(name, version, config); + } + + private static TimeModel parseTimeModel(JsonNode timeNode) { + int bucketCount = requireInt(timeNode, "bucket_count"); + String formula = requireNode(timeNode, "bucket_encoding").path("formula").asText(""); + if (formula.isEmpty()) { + throw new FactoryException("Missing bucket encoding formula"); + } + int samplingInterval = requireInt(timeNode, "sampling_interval_minutes"); + String timezone = requireText(timeNode, "timezone"); + return new TimeModel(bucketCount, formula, samplingInterval, timezone); + } + + private static ValueModel parseValueModel(JsonNode valueNode) { + String valueUnit = requireText(valueNode, "value_unit"); + JsonNode normalizationNode = requireNode(valueNode, "normalization"); + String normalizationMethod = requireText(normalizationNode, "method"); + ValueModel.Normalization normalization = new ValueModel.Normalization(normalizationMethod); + + JsonNode discretizationNode = requireNode(valueNode, "discretization"); + int states = requireInt(discretizationNode, "states"); + List thresholds = new ArrayList<>(); + JsonNode thresholdsNode = requireNode(discretizationNode, "thresholds_right"); + if (!thresholdsNode.isArray()) { + throw new FactoryException("thresholds_right must be an array"); + } + thresholdsNode.forEach(element -> thresholds.add(element.asDouble())); + ValueModel.Discretization discretization = + new ValueModel.Discretization(states, List.copyOf(thresholds)); + + return new ValueModel(valueUnit, normalization, discretization); + } + + private static Parameters parseParameters(JsonNode parametersNode) { + Parameters.TransitionParameters transitions = + new Parameters.TransitionParameters( + parametersNode.path("transitions").path("empty_row_strategy").asText("")); + if (transitions.emptyRowStrategy().isEmpty()) { + transitions = null; + } + + JsonNode gmmNode = parametersNode.path("gmm"); + Parameters.GmmParameters gmm = + gmmNode.isMissingNode() || gmmNode.isNull() || gmmNode.size() == 0 + ? null + : new Parameters.GmmParameters( + gmmNode.path("value_col").asText(""), + optionalInt(gmmNode, "verbose"), + optionalInt(gmmNode, "heartbeat_seconds")); + + return new Parameters(transitions, gmm); + } + + private static Optional optionalInt(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isNull()) return Optional.empty(); + return Optional.of(value.asInt()); + } + + private static TransitionData parseTransitions( + JsonNode dataNode, int expectedBucketCount, int stateCount) { + JsonNode transitionsNode = requireNode(dataNode, "transitions"); + String dtype = requireText(transitionsNode, "dtype"); + String encoding = requireText(transitionsNode, "encoding"); + + JsonNode shapeNode = requireNode(transitionsNode, "shape"); + if (!shapeNode.isArray() || shapeNode.size() != 3) { + throw new FactoryException("Transition shape must contain three dimensions"); + } + int buckets = shapeNode.get(0).asInt(); + int rows = shapeNode.get(1).asInt(); + int columns = shapeNode.get(2).asInt(); + if (buckets != expectedBucketCount) { + throw new FactoryException( + "Transition bucket count mismatch. Expected " + + expectedBucketCount + + " but was " + + buckets); + } + if (rows != stateCount || columns != stateCount) { + throw new FactoryException( + "Transition state dimension mismatch. Expected " + + stateCount + + " but was rows=" + + rows + + ", columns=" + + columns); + } + + JsonNode valuesNode = requireNode(transitionsNode, "values"); + if (!valuesNode.isArray()) { + throw new FactoryException("Transition values must be a three dimensional array"); + } + + double[][][] values = new double[buckets][stateCount][stateCount]; + int bucketIndex = 0; + for (JsonNode bucketNode : valuesNode) { + if (bucketIndex >= buckets) { + throw new FactoryException("More transition buckets present than specified in shape"); + } + int rowIndex = 0; + for (JsonNode rowNode : bucketNode) { + if (rowIndex >= stateCount) { + throw new FactoryException( + "Too many rows in transition matrix for bucket " + bucketIndex); + } + int columnIndex = 0; + for (JsonNode probNode : rowNode) { + if (columnIndex >= stateCount) { + throw new FactoryException( + "Too many columns in transition matrix for bucket " + + bucketIndex + + ", row " + + rowIndex); + } + values[bucketIndex][rowIndex][columnIndex] = probNode.asDouble(); + columnIndex++; + } + if (columnIndex != stateCount) { + throw new FactoryException( + "Row " + + rowIndex + + " in bucket " + + bucketIndex + + " had " + + columnIndex + + " columns. Expected " + + stateCount); + } + rowIndex++; + } + if (rowIndex != stateCount) { + throw new FactoryException( + "Bucket " + bucketIndex + " contained " + rowIndex + " rows. Expected " + stateCount); + } + bucketIndex++; + } + if (bucketIndex != buckets) { + throw new FactoryException( + "Transition values provided only " + bucketIndex + " buckets. Expected " + buckets); + } + + return new TransitionData(dtype, encoding, values); + } + + private static Optional parseGmmBuckets(JsonNode gmmsNode) { + if (gmmsNode == null + || gmmsNode.isMissingNode() + || gmmsNode.isNull() + || !gmmsNode.has("buckets")) { + return Optional.empty(); + } + JsonNode bucketsNode = gmmsNode.get("buckets"); + if (!bucketsNode.isArray()) { + throw new FactoryException("data.gmms.buckets must be an array"); + } + List buckets = new ArrayList<>(); + for (JsonNode bucketNode : bucketsNode) { + JsonNode statesNode = bucketNode.get("states"); + if (statesNode == null || !statesNode.isArray()) { + throw new FactoryException("Each GMM bucket must contain an array 'states'"); + } + List> states = new ArrayList<>(); + for (JsonNode stateNode : statesNode) { + if (stateNode == null || stateNode.isNull()) { + states.add(Optional.empty()); + continue; + } + List weights = readDoubleArray(stateNode, "weights"); + List means = readDoubleArray(stateNode, "means"); + List variances = readDoubleArray(stateNode, "variances"); + states.add(Optional.of(new GmmBuckets.GmmState(weights, means, variances))); + } + buckets.add(new GmmBuckets.GmmBucket(List.copyOf(states))); + } + return Optional.of(new GmmBuckets(List.copyOf(buckets))); + } + + private static List readDoubleArray(JsonNode node, String field) { + JsonNode arrayNode = node.get(field); + if (arrayNode == null || !arrayNode.isArray()) { + throw new FactoryException("Field '" + field + "' must be an array"); + } + List values = new ArrayList<>(); + arrayNode.forEach(element -> values.add(element.asDouble())); + return List.copyOf(values); + } + + private static JsonNode requireNode(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode()) { + throw new FactoryException("Missing field '" + field + "'"); + } + return value; + } + + private static String requireText(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode() || value.isNull()) { + throw new FactoryException("Missing field '" + field + "'"); + } + if (!value.isTextual()) { + throw new FactoryException("Field '" + field + "' must be textual"); + } + return value.asText(); + } + + private static int requireInt(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode() || value.isNull()) { + throw new FactoryException("Missing field '" + field + "'"); + } + if (!value.canConvertToInt()) { + throw new FactoryException("Field '" + field + "' must be an integer"); + } + return value.asInt(); + } + + private static ZonedDateTime parseTimestamp(String timestamp) { + try { + return ZonedDateTime.parse(timestamp); + } catch (DateTimeParseException e) { + throw new FactoryException("Unable to parse generated_at timestamp '" + timestamp + "'", e); + } + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java new file mode 100644 index 0000000000..e1cbdf3b0e --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java @@ -0,0 +1,26 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.factory.markov; + +import com.fasterxml.jackson.databind.JsonNode; +import edu.ie3.datamodel.io.factory.FactoryData; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; +import java.util.Collections; +import java.util.Objects; + +/** Factory data wrapper around a parsed Markov-load JSON tree. */ +public class MarkovModelData extends FactoryData { + private final JsonNode root; + + public MarkovModelData(JsonNode root) { + super(Collections.emptyMap(), MarkovLoadModel.class); + this.root = Objects.requireNonNull(root, "root"); + } + + public JsonNode getRoot() { + return root; + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/file/FileType.java b/src/main/java/edu/ie3/datamodel/io/file/FileType.java index b9ac1f3f7e..08886b255d 100644 --- a/src/main/java/edu/ie3/datamodel/io/file/FileType.java +++ b/src/main/java/edu/ie3/datamodel/io/file/FileType.java @@ -10,7 +10,8 @@ import java.util.stream.Collectors; public enum FileType { - CSV(".csv"); + CSV(".csv"), + JSON(".json"); public final String fileEnding; diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java new file mode 100644 index 0000000000..945a3b11a9 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java @@ -0,0 +1,142 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.source.json; + +import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.io.connectors.JsonFileConnector; +import edu.ie3.datamodel.io.naming.FileNamingStrategy; +import edu.ie3.datamodel.io.source.file.FileDataSource; +import edu.ie3.datamodel.models.Entity; +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Stream; + +/** Data source abstraction for JSON files. */ +public class JsonDataSource extends FileDataSource { + + private final JsonFileConnector connector; + + public JsonDataSource(Path directoryPath, FileNamingStrategy fileNamingStrategy) { + this(new JsonFileConnector(directoryPath), fileNamingStrategy); + } + + public JsonDataSource(JsonFileConnector connector, FileNamingStrategy fileNamingStrategy) { + super(connector.getBaseDirectory(), fileNamingStrategy); + this.connector = connector; + } + + /** + * Opens a buffered reader for the provided file path. + * + * @param filePath relative path without ending + * @return buffered reader + * @throws SourceException if the file cannot be opened + */ + public BufferedReader initReader(Path filePath) throws SourceException { + try { + return connector.initReader(filePath); + } catch (FileNotFoundException e) { + throw new SourceException("Unable to open JSON file '" + filePath + "'.", e); + } + } + + /** + * Opens an input stream for the provided file path. + * + * @param filePath relative path without ending + * @return input stream + * @throws SourceException if the file cannot be opened + */ + public InputStream initInputStream(Path filePath) throws SourceException { + try { + return connector.initInputStream(filePath); + } catch (FileNotFoundException e) { + throw new SourceException("Unable to open JSON file '" + filePath + "'.", e); + } + } + + /** + * Utility method that reads the entire JSON file into memory. + * + * @param filePath relative path without ending + * @return optional JSON string (empty if the file does not exist) + * @throws SourceException if reading fails + */ + public Optional readRaw(Path filePath) throws SourceException { + try (Reader reader = connector.initReader(filePath)) { + return Optional.of(readAll(reader)); + } catch (FileNotFoundException e) { + return Optional.empty(); + } catch (IOException e) { + throw new SourceException("Unable to read JSON file '" + filePath + "'.", e); + } + } + + /** + * Reads the JSON file using the provided consumer function. + * + * @param filePath relative path without ending + * @param readerFunction function that consumes the reader + * @param result type + * @return optional result (empty if file not found) + * @throws SourceException if reading fails + */ + public Optional readWith(Path filePath, Function readerFunction) + throws SourceException { + try (BufferedReader reader = connector.initReader(filePath)) { + return Optional.ofNullable(readerFunction.apply(reader)); + } catch (FileNotFoundException e) { + return Optional.empty(); + } catch (IOException e) { + throw new SourceException("Unable to read JSON file '" + filePath + "'.", e); + } + } + + @Override + public Optional> getSourceFields(Class entityClass) + throws SourceException { + throw unsupportedTabularAccess("getSourceFields(Class)"); + } + + @Override + public Stream> getSourceData(Class entityClass) + throws SourceException { + throw unsupportedTabularAccess("getSourceData(Class)"); + } + + @Override + public Optional> getSourceFields(Path filePath) throws SourceException { + throw unsupportedTabularAccess("getSourceFields(Path)"); + } + + @Override + public Stream> getSourceData(Path filePath) throws SourceException { + throw unsupportedTabularAccess("getSourceData(Path)"); + } + + private UnsupportedOperationException unsupportedTabularAccess(String method) { + return new UnsupportedOperationException( + "JsonDataSource does not support '" + method + "', as JSON sources are not tabular."); + } + + private String readAll(Reader reader) throws IOException { + StringBuilder builder = new StringBuilder(); + char[] buffer = new char[4096]; + int read; + while ((read = reader.read(buffer)) != -1) { + builder.append(buffer, 0, read); + } + return builder.toString(); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java new file mode 100644 index 0000000000..d3240a896f --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java @@ -0,0 +1,117 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.source.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.exceptions.FailedValidationException; +import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.exceptions.ValidationException; +import edu.ie3.datamodel.io.factory.markov.MarkovLoadModelFactory; +import edu.ie3.datamodel.io.factory.markov.MarkovModelData; +import edu.ie3.datamodel.io.file.FileType; +import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation; +import edu.ie3.datamodel.io.source.EntitySource; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + +/** Source that reads Markov-based load models from JSON files. */ +public class JsonMarkovProfileSource extends EntitySource { + + private final JsonDataSource dataSource; + private final FileLoadProfileMetaInformation metaInformation; + private final MarkovLoadModelFactory factory; + private final ObjectMapper objectMapper = new ObjectMapper(); + private MarkovLoadModel cachedModel; + + public JsonMarkovProfileSource( + JsonDataSource dataSource, FileLoadProfileMetaInformation metaInformation) { + this(dataSource, metaInformation, new MarkovLoadModelFactory()); + } + + public JsonMarkovProfileSource( + JsonDataSource dataSource, + FileLoadProfileMetaInformation metaInformation, + MarkovLoadModelFactory factory) { + this.dataSource = Objects.requireNonNull(dataSource, "dataSource"); + this.metaInformation = Objects.requireNonNull(metaInformation, "metaInformation"); + this.factory = Objects.requireNonNull(factory, "factory"); + if (metaInformation.getFileType() != FileType.JSON) { + throw new IllegalArgumentException("Markov profile source requires JSON meta information."); + } + } + + /** + * Returns the parsed Markov model, parsing the underlying file if needed. + * + * @throws SourceException if reading or parsing fails + */ + public synchronized MarkovLoadModel getModel() throws SourceException { + if (cachedModel == null) { + JsonNode root = readModelTree(); + try { + cachedModel = factory.get(new MarkovModelData(root)).getOrThrow(); + } catch (FactoryException e) { + throw new SourceException( + "Unable to build Markov load model from '" + metaInformation.getProfile() + "'.", e); + } + } + return cachedModel; + } + + @Override + public void validate() throws ValidationException { + JsonNode root; + try { + root = readModelTree(); + } catch (SourceException e) { + throw new FailedValidationException( + "Unable to read Markov model '" + metaInformation.getProfile() + "' for validation.", e); + } + Set fields = collectFieldNames(root); + factory.validate(fields, MarkovLoadModel.class).getOrThrow(); + } + + private JsonNode readModelTree() throws SourceException { + Path filePath = metaInformation.getFullFilePath(); + try (InputStream inputStream = dataSource.initInputStream(filePath)) { + return objectMapper.readTree(inputStream); + } catch (IOException e) { + throw new SourceException("Unable to read Markov model JSON from '" + filePath + "'.", e); + } + } + + private static Set collectFieldNames(JsonNode node) { + Set fields = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + collectFields("", node, fields); + return fields; + } + + private static void collectFields(String prefix, JsonNode node, Set collector) { + if (node.isArray()) { + if (!prefix.isEmpty()) { + collector.add(prefix); + } + return; + } + if (node.isObject()) { + node.fieldNames() + .forEachRemaining(name -> collectFields(join(prefix, name), node.get(name), collector)); + } else if (!prefix.isEmpty()) { + collector.add(prefix); + } + } + + private static String join(String prefix, String name) { + return prefix.isEmpty() ? name : prefix + "." + name; + } +} diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java new file mode 100644 index 0000000000..3ba8163b65 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -0,0 +1,63 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.models.profile.markov; + +import java.time.ZonedDateTime; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** Container for Markov-chain-based load models produced by simonaMarkovLoad. */ +public record MarkovLoadModel( + String schema, + ZonedDateTime generatedAt, + Generator generator, + TimeModel timeModel, + ValueModel valueModel, + Parameters parameters, + TransitionData transitionData, + Optional gmmBuckets) { + + public record Generator(String name, String version, Map config) {} + + public record TimeModel( + int bucketCount, + String bucketEncodingFormula, + int samplingIntervalMinutes, + String timezone) {} + + public record ValueModel( + String valueUnit, Normalization normalization, Discretization discretization) { + + public record Normalization(String method) {} + + public record Discretization(int states, List thresholdsRight) {} + } + + public record Parameters(TransitionParameters transitions, GmmParameters gmm) { + + public record TransitionParameters(String emptyRowStrategy) {} + + public record GmmParameters( + String valueColumn, Optional verbose, Optional heartbeatSeconds) {} + } + + public record TransitionData(String dtype, String encoding, double[][][] values) { + public int bucketCount() { + return values.length; + } + + public int stateCount() { + return values.length == 0 ? 0 : values[0].length; + } + } + + public record GmmBuckets(List buckets) { + public record GmmBucket(List> states) {} + + public record GmmState(List weights, List means, List variances) {} + } +} From 80e932dcaa134da5033bc53e0c37ce523d1f8f69 Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 10 Nov 2025 03:10:06 +0100 Subject: [PATCH 02/39] bug Fix --- .../profile/markov/MarkovLoadModel.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index 3ba8163b65..2b525a966d 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -6,8 +6,10 @@ package edu.ie3.datamodel.models.profile.markov; import java.time.ZonedDateTime; +import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; /** Container for Markov-chain-based load models produced by simonaMarkovLoad. */ @@ -53,6 +55,35 @@ public int bucketCount() { public int stateCount() { return values.length == 0 ? 0 : values[0].length; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TransitionData(String dtype1, String encoding1, double[][][] values1))) + return false; + return Objects.equals(dtype, dtype1) + && Objects.equals(encoding, encoding1) + && Arrays.deepEquals(values, values1); + } + + @Override + public int hashCode() { + return Objects.hash(dtype, encoding, Arrays.deepHashCode(values)); + } + + @Override + public String toString() { + return "TransitionData{" + + "dtype='" + + dtype + + '\'' + + ", encoding='" + + encoding + + '\'' + + ", values=" + + Arrays.deepToString(values) + + '}'; + } } public record GmmBuckets(List buckets) { From a590311d63c7064a636aa488719e69e1208d072d Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 17 Nov 2025 02:40:57 +0100 Subject: [PATCH 03/39] little fix --- .../markov/MarkovLoadModelFactory.java | 15 ++--- .../io/source/json/JsonDataSource.java | 66 ------------------- 2 files changed, 6 insertions(+), 75 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java index aeabdffe2a..6ebe5e1781 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -41,7 +41,7 @@ protected MarkovLoadModel buildModel(MarkovModelData data) { JsonNode dataNode = requireNode(root, "data"); TransitionData transitionData = parseTransitions(dataNode, timeModel.bucketCount(), valueModel.discretization().states()); - Optional gmmBuckets = parseGmmBuckets(dataNode.path("gmms")); + GmmBuckets gmmBuckets = parseGmmBuckets(requireNode(dataNode, "gmms")); return new MarkovLoadModel( schema, @@ -51,7 +51,7 @@ protected MarkovLoadModel buildModel(MarkovModelData data) { valueModel, parameters, transitionData, - gmmBuckets); + Optional.of(gmmBuckets)); } @Override @@ -234,12 +234,9 @@ private static TransitionData parseTransitions( return new TransitionData(dtype, encoding, values); } - private static Optional parseGmmBuckets(JsonNode gmmsNode) { - if (gmmsNode == null - || gmmsNode.isMissingNode() - || gmmsNode.isNull() - || !gmmsNode.has("buckets")) { - return Optional.empty(); + private static GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { + if (gmmsNode == null || gmmsNode.isMissingNode() || gmmsNode.isNull()) { + throw new FactoryException("Missing field 'gmms'"); } JsonNode bucketsNode = gmmsNode.get("buckets"); if (!bucketsNode.isArray()) { @@ -264,7 +261,7 @@ private static Optional parseGmmBuckets(JsonNode gmmsNode) { } buckets.add(new GmmBuckets.GmmBucket(List.copyOf(states))); } - return Optional.of(new GmmBuckets(List.copyOf(buckets))); + return new GmmBuckets(List.copyOf(buckets)); } private static List readDoubleArray(JsonNode node, String field) { diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java index 945a3b11a9..eb542bb4a7 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java @@ -10,16 +10,12 @@ import edu.ie3.datamodel.io.naming.FileNamingStrategy; import edu.ie3.datamodel.io.source.file.FileDataSource; import edu.ie3.datamodel.models.Entity; -import java.io.BufferedReader; import java.io.FileNotFoundException; -import java.io.IOException; import java.io.InputStream; -import java.io.Reader; import java.nio.file.Path; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.function.Function; import java.util.stream.Stream; /** Data source abstraction for JSON files. */ @@ -36,21 +32,6 @@ public JsonDataSource(JsonFileConnector connector, FileNamingStrategy fileNaming this.connector = connector; } - /** - * Opens a buffered reader for the provided file path. - * - * @param filePath relative path without ending - * @return buffered reader - * @throws SourceException if the file cannot be opened - */ - public BufferedReader initReader(Path filePath) throws SourceException { - try { - return connector.initReader(filePath); - } catch (FileNotFoundException e) { - throw new SourceException("Unable to open JSON file '" + filePath + "'.", e); - } - } - /** * Opens an input stream for the provided file path. * @@ -66,43 +47,6 @@ public InputStream initInputStream(Path filePath) throws SourceException { } } - /** - * Utility method that reads the entire JSON file into memory. - * - * @param filePath relative path without ending - * @return optional JSON string (empty if the file does not exist) - * @throws SourceException if reading fails - */ - public Optional readRaw(Path filePath) throws SourceException { - try (Reader reader = connector.initReader(filePath)) { - return Optional.of(readAll(reader)); - } catch (FileNotFoundException e) { - return Optional.empty(); - } catch (IOException e) { - throw new SourceException("Unable to read JSON file '" + filePath + "'.", e); - } - } - - /** - * Reads the JSON file using the provided consumer function. - * - * @param filePath relative path without ending - * @param readerFunction function that consumes the reader - * @param result type - * @return optional result (empty if file not found) - * @throws SourceException if reading fails - */ - public Optional readWith(Path filePath, Function readerFunction) - throws SourceException { - try (BufferedReader reader = connector.initReader(filePath)) { - return Optional.ofNullable(readerFunction.apply(reader)); - } catch (FileNotFoundException e) { - return Optional.empty(); - } catch (IOException e) { - throw new SourceException("Unable to read JSON file '" + filePath + "'.", e); - } - } - @Override public Optional> getSourceFields(Class entityClass) throws SourceException { @@ -129,14 +73,4 @@ private UnsupportedOperationException unsupportedTabularAccess(String method) { return new UnsupportedOperationException( "JsonDataSource does not support '" + method + "', as JSON sources are not tabular."); } - - private String readAll(Reader reader) throws IOException { - StringBuilder builder = new StringBuilder(); - char[] buffer = new char[4096]; - int read; - while ((read = reader.read(buffer)) != -1) { - builder.append(buffer, 0, read); - } - return builder.toString(); - } } From 26a9a5489886d0a56b032145655b581dce1dbcdc Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 26 Nov 2025 09:08:13 +0300 Subject: [PATCH 04/39] tests and regex --- .../markov/MarkovLoadModelFactory.java | 18 +-- .../EntityPersistenceNamingStrategy.java | 2 +- .../profile/markov/MarkovLoadModel.java | 9 +- .../connectors/JsonFileConnectorTest.groovy | 55 +++++++ .../markov/MarkovLoadModelFactoryTest.groovy | 111 ++++++++++++++ .../ie3/datamodel/io/file/FileTypeTest.groovy | 30 ++++ ...EntityPersistenceNamingStrategyTest.groovy | 25 ++++ .../io/naming/FileNamingStrategyTest.groovy | 4 +- .../io/source/json/JsonDataSourceTest.groovy | 60 ++++++++ .../json/JsonMarkovProfileSourceTest.groovy | 137 ++++++++++++++++++ 10 files changed, 434 insertions(+), 17 deletions(-) create mode 100644 src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy create mode 100644 src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy create mode 100644 src/test/groovy/edu/ie3/datamodel/io/file/FileTypeTest.groovy create mode 100644 src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy create mode 100644 src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java index 6ebe5e1781..55e906de93 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -59,17 +59,17 @@ protected List> getFields(Class entityClass) { Set requiredFields = newSet( "schema", - "generated_at", + "generatedAt", "generator.name", "generator.version", - "time_model.bucket_count", - "time_model.bucket_encoding.formula", - "time_model.sampling_interval_minutes", - "time_model.timezone", - "value_model.value_unit", - "value_model.normalization.method", - "value_model.discretization.states", - "value_model.discretization.thresholds_right", + "timeModel.bucketCount", + "timeModel.bucketEncoding.formula", + "timeModel.samplingIntervalMinutes", + "timeModel.timezone", + "valueModel.valueUnit", + "valueModel.normalization.method", + "valueModel.discretization.states", + "valueModel.discretization.thresholdsRight", "data.transitions.shape", "data.transitions.values", "data.gmms.buckets"); diff --git a/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java b/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java index 12a42e922a..76818163de 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java @@ -61,7 +61,7 @@ public class EntityPersistenceNamingStrategy { * profile is accessible via the named capturing group "profile", the uuid by the group "uuid" */ private static final String LOAD_PROFILE_TIME_SERIES = - "lpts_(?[a-zA-Z]{1,11}[0-9]{0,3})"; + "(?:lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})"; /** * Pattern to identify load profile time series in this instance of the naming strategy (takes diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index 2b525a966d..c791491b12 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -59,11 +59,10 @@ public int stateCount() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof TransitionData(String dtype1, String encoding1, double[][][] values1))) - return false; - return Objects.equals(dtype, dtype1) - && Objects.equals(encoding, encoding1) - && Arrays.deepEquals(values, values1); + if (!(o instanceof TransitionData other)) return false; + return Objects.equals(dtype, other.dtype) + && Objects.equals(encoding, other.encoding) + && Arrays.deepEquals(values, other.values); } @Override diff --git a/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy new file mode 100644 index 0000000000..49a3877d37 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy @@ -0,0 +1,55 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.connectors + +import spock.lang.Specification + +import java.nio.file.Files +import java.nio.file.Path + +class JsonFileConnectorTest extends Specification { + + Path tempDir + + def setup() { + tempDir = Files.createTempDirectory("jsonFileConnector") + } + + def cleanup() { + if (tempDir != null) { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .forEach { Files.deleteIfExists(it) } + } + } + + def "initInputStream resolves .json ending and reads content"() { + given: + def file = tempDir.resolve("model.json") + Files.writeString(file, """{"foo":"bar"}""") + def connector = new JsonFileConnector(tempDir) + + when: + def content = connector.initInputStream(Path.of("model")).text + + then: + content == """{"foo":"bar"}""" + } + + def "initReader returns buffered reader with UTF-8 decoding"() { + given: + def file = tempDir.resolve("data.json") + Files.writeString(file, "[1,2,3]") + def connector = new JsonFileConnector(tempDir) + + when: + def reader = connector.initReader(Path.of("data")) + def line = reader.readLine() + + then: + line == "[1,2,3]" + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy new file mode 100644 index 0000000000..b60d2864a6 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy @@ -0,0 +1,111 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.factory.markov + +import com.fasterxml.jackson.databind.ObjectMapper +import edu.ie3.datamodel.exceptions.FactoryException +import spock.lang.Specification + +class MarkovLoadModelFactoryTest extends Specification { + private final ObjectMapper objectMapper = new ObjectMapper() + private final MarkovLoadModelFactory factory = new MarkovLoadModelFactory() + + def "buildModel returns parsed Markov load model from valid JSON"() { + given: + def root = objectMapper.readTree(validModelJson()) + + when: + def model = factory.get(new MarkovModelData(root)).getOrThrow() + + then: + model.schema() == "markov.load.v1" + model.generator().name() == "simonaMarkovLoad" + model.timeModel().bucketCount() == 1 + model.valueModel().discretization().states() == 2 + model.transitionData().bucketCount() == 1 + model.transitionData().stateCount() == 2 + model.transitionData().values()[0][0][1] == 0.9d + model.gmmBuckets().isPresent() + def gmmState = model.gmmBuckets().get().buckets().first().states().first().get() + gmmState.weights() == [0.6d] + gmmState.means() == [1.0d] + gmmState.variances() == [0.2d] + } + + def "buildModel throws FactoryException on transition dimension mismatch"() { + given: + def invalidJson = objectMapper.readTree(validModelJson().replace("\"shape\": [1,2,2]", "\"shape\": [2,2,2]")) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + private static String validModelJson() { + return """ + { + "schema": "markov.load.v1", + "generated_at": "2025-01-01T00:00:00Z", + "generator": { + "name": "simonaMarkovLoad", + "version": "1.0.0", + "config": { "foo": "bar" } + }, + "time_model": { + "bucket_count": 1, + "bucket_encoding": { "formula": "hour_of_day" }, + "sampling_interval_minutes": 60, + "timezone": "UTC" + }, + "value_model": { + "value_unit": "W", + "normalization": { "method": "none" }, + "discretization": { + "states": 2, + "thresholds_right": [0.5] + } + }, + "parameters": { + "transitions": { "empty_row_strategy": "fill" }, + "gmm": { + "value_col": "p", + "verbose": 1, + "heartbeat_seconds": 5 + } + }, + "data": { + "transitions": { + "dtype": "float64", + "encoding": "dense", + "shape": [1,2,2], + "values": [ + [ + [0.1, 0.9], + [0.3, 0.7] + ] + ] + }, + "gmms": { + "buckets": [ + { + "states": [ + { + "weights": [0.6], + "means": [1.0], + "variances": [0.2] + }, + null + ] + } + ] + } + } + } + """.stripIndent() + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/file/FileTypeTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/file/FileTypeTest.groovy new file mode 100644 index 0000000000..ecfb744d2f --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/file/FileTypeTest.groovy @@ -0,0 +1,30 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.file + +import edu.ie3.datamodel.exceptions.ParsingException +import spock.lang.Specification + +class FileTypeTest extends Specification { + + def "getFileType resolves CSV and JSON endings"() { + expect: + FileType.getFileType(fileName) == expected + + where: + fileName || expected + "data.csv" || FileType.CSV + "model.json" || FileType.JSON + } + + def "getFileType throws ParsingException on unknown ending"() { + when: + FileType.getFileType("unknown.txt") + + then: + thrown(ParsingException) + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy index 8688aced04..8011e4cb7f 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy @@ -94,6 +94,19 @@ class EntityPersistenceNamingStrategyTest extends Specification { matcher.group("profile") == "g3" } + def "The pattern for a Markov load profile time series file name matches and extracts the correct profile"() { + given: + def ens = new EntityPersistenceNamingStrategy() + def validFileName = "markov_demo1" + + when: + def matcher = ens.loadProfileTimeSeriesPattern.matcher(validFileName) + + then: + matcher.matches() + matcher.group("profile") == "demo1" + } + def "Trying to extract individual time series meta information throws an Exception, if it is provided a malformed string"() { given: def ens = new EntityPersistenceNamingStrategy() @@ -120,6 +133,18 @@ class EntityPersistenceNamingStrategyTest extends Specification { ex.message == "Cannot extract meta information on load profile time series from 'foo'." } + def "loadProfileTimesSeriesMetaInformation extracts profile from Markov load profile file name"() { + given: + def ens = new EntityPersistenceNamingStrategy() + def fileName = "markov_demo2" + + when: + def meta = ens.loadProfileTimesSeriesMetaInformation(fileName) + + then: + meta.profile == "demo2" + } + def "The EntityPersistenceNamingStrategy is able to prepare the prefix properly"() { when: String actual = EntityPersistenceNamingStrategy.preparePrefix(prefix) diff --git a/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy index 3b12faef44..b68a5f166c 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy @@ -767,7 +767,7 @@ class FileNamingStrategyTest extends Specification { def actual = strategy.loadProfileTimeSeriesPattern.pattern() then: - actual == "test_grid" + escapedFileSeparator + "input" + escapedFileSeparator + "participants" + escapedFileSeparator + "time_series" + escapedFileSeparator + "lpts_(?[a-zA-Z]{1,11}[0-9]{0,3})" + actual == "test_grid" + escapedFileSeparator + "input" + escapedFileSeparator + "participants" + escapedFileSeparator + "time_series" + escapedFileSeparator + "(?:lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})" } def "A FileNamingStrategy with FlatHierarchy returns correct individual time series file name pattern"() { @@ -789,7 +789,7 @@ class FileNamingStrategyTest extends Specification { def actual = strategy.loadProfileTimeSeriesPattern.pattern() then: - actual == "lpts_(?[a-zA-Z]{1,11}[0-9]{0,3})" + actual == "(?:lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})" } def "Trying to extract time series meta information throws an Exception, if it is provided a malformed string"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy new file mode 100644 index 0000000000..423c1af23b --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy @@ -0,0 +1,60 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.json + +import edu.ie3.datamodel.exceptions.SourceException +import edu.ie3.datamodel.io.connectors.JsonFileConnector +import edu.ie3.datamodel.io.naming.FileNamingStrategy +import edu.ie3.datamodel.models.Entity +import spock.lang.Specification + +import java.nio.file.Files +import java.nio.file.Path + +class JsonDataSourceTest extends Specification { + + Path tempDir + JsonDataSource dataSource + + def setup() { + tempDir = Files.createTempDirectory("jsonDataSource") + dataSource = new JsonDataSource(tempDir, new FileNamingStrategy()) + } + + def cleanup() { + if (tempDir != null) { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .forEach { Files.deleteIfExists(it) } + } + } + + def "initInputStream opens JSON file via connector"() { + given: + def file = tempDir.resolve("sample.json") + Files.writeString(file, """{"key":42}""") + + when: + def content = dataSource.initInputStream(Path.of("sample")).text + + then: + content == """{"key":42}""" + } + + def "tabular access methods are unsupported"() { + when: + dataSource.getSourceFields(Entity) + + then: + thrown(UnsupportedOperationException) + + when: + dataSource.getSourceData(Entity) + + then: + thrown(UnsupportedOperationException) + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy new file mode 100644 index 0000000000..1b20cd92fc --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy @@ -0,0 +1,137 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.json + +import edu.ie3.datamodel.exceptions.SourceException +import edu.ie3.datamodel.io.file.FileType +import edu.ie3.datamodel.io.naming.FileNamingStrategy +import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel +import spock.lang.Specification + +import java.nio.file.Files +import java.nio.file.Path + +class JsonMarkovProfileSourceTest extends Specification { + + Path tempDir + Path jsonFile + + def setup() { + tempDir = Files.createTempDirectory("markovProfileSource") + jsonFile = tempDir.resolve("model.json") + } + + def cleanup() { + if (tempDir != null) { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .forEach { Files.deleteIfExists(it) } + } + } + + def "getModel reads and caches Markov model from JSON file"() { + given: + Files.writeString(jsonFile, validModelJson()) + def source = new JsonMarkovProfileSource( + new JsonDataSource(tempDir, new FileNamingStrategy()), + new FileLoadProfileMetaInformation("profile1", jsonFile, FileType.JSON) + ) + + when: + MarkovLoadModel modelFirst = source.getModel() + MarkovLoadModel modelSecond = source.getModel() + + then: + modelFirst.is(modelSecond) // cached instance reused + modelFirst.schema() == "markov.load.v1" + noExceptionThrown() + + when: "validation is executed on the same file" + source.validate() + + then: + noExceptionThrown() + } + + def "getModel throws SourceException on invalid JSON file"() { + given: + Files.writeString(jsonFile, "{}") + def source = new JsonMarkovProfileSource( + new JsonDataSource(tempDir, new FileNamingStrategy()), + new FileLoadProfileMetaInformation("brokenProfile", jsonFile, FileType.JSON) + ) + + when: + source.getModel() + + then: + thrown(SourceException) + } + + private static String validModelJson() { + return """ + { + "schema": "markov.load.v1", + "generated_at": "2025-01-01T00:00:00Z", + "generator": { + "name": "simonaMarkovLoad", + "version": "1.0.0", + "config": { "foo": "bar" } + }, + "time_model": { + "bucket_count": 1, + "bucket_encoding": { "formula": "hour_of_day" }, + "sampling_interval_minutes": 60, + "timezone": "UTC" + }, + "value_model": { + "value_unit": "W", + "normalization": { "method": "none" }, + "discretization": { + "states": 2, + "thresholds_right": [0.5] + } + }, + "parameters": { + "transitions": { "empty_row_strategy": "fill" }, + "gmm": { + "value_col": "p", + "verbose": 1, + "heartbeat_seconds": 5 + } + }, + "data": { + "transitions": { + "dtype": "float64", + "encoding": "dense", + "shape": [1,2,2], + "values": [ + [ + [0.1, 0.9], + [0.3, 0.7] + ] + ] + }, + "gmms": { + "buckets": [ + { + "states": [ + { + "weights": [0.6], + "means": [1.0], + "variances": [0.2] + }, + null + ] + } + ] + } + } + } + """.stripIndent() + } +} From b84ecb3b89bc5ed3ebfe09531e1079ce32f0a4e8 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 26 Nov 2025 09:32:41 +0300 Subject: [PATCH 05/39] refactor --- .../markov/MarkovLoadModelFactory.java | 112 +++++++++++------- 1 file changed, 68 insertions(+), 44 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java index 55e906de93..9461c2ffe1 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -154,13 +154,28 @@ private static TransitionData parseTransitions( String dtype = requireText(transitionsNode, "dtype"); String encoding = requireText(transitionsNode, "encoding"); + int[] shape = parseTransitionShape(transitionsNode); + int buckets = shape[0]; + int rows = shape[1]; + int columns = shape[2]; + validateTransitionShape(expectedBucketCount, stateCount, buckets, rows, columns); + + JsonNode valuesNode = requireNode(transitionsNode, "values"); + double[][][] values = parseTransitionValues(valuesNode, buckets, stateCount); + + return new TransitionData(dtype, encoding, values); + } + + private static int[] parseTransitionShape(JsonNode transitionsNode) { JsonNode shapeNode = requireNode(transitionsNode, "shape"); if (!shapeNode.isArray() || shapeNode.size() != 3) { throw new FactoryException("Transition shape must contain three dimensions"); } - int buckets = shapeNode.get(0).asInt(); - int rows = shapeNode.get(1).asInt(); - int columns = shapeNode.get(2).asInt(); + return new int[] {shapeNode.get(0).asInt(), shapeNode.get(1).asInt(), shapeNode.get(2).asInt()}; + } + + private static void validateTransitionShape( + int expectedBucketCount, int stateCount, int buckets, int rows, int columns) { if (buckets != expectedBucketCount) { throw new FactoryException( "Transition bucket count mismatch. Expected " @@ -177,61 +192,70 @@ private static TransitionData parseTransitions( + ", columns=" + columns); } + } - JsonNode valuesNode = requireNode(transitionsNode, "values"); + private static double[][][] parseTransitionValues( + JsonNode valuesNode, int buckets, int stateCount) { if (!valuesNode.isArray()) { throw new FactoryException("Transition values must be a three dimensional array"); } - double[][][] values = new double[buckets][stateCount][stateCount]; int bucketIndex = 0; for (JsonNode bucketNode : valuesNode) { - if (bucketIndex >= buckets) { - throw new FactoryException("More transition buckets present than specified in shape"); - } - int rowIndex = 0; - for (JsonNode rowNode : bucketNode) { - if (rowIndex >= stateCount) { - throw new FactoryException( - "Too many rows in transition matrix for bucket " + bucketIndex); - } - int columnIndex = 0; - for (JsonNode probNode : rowNode) { - if (columnIndex >= stateCount) { - throw new FactoryException( - "Too many columns in transition matrix for bucket " - + bucketIndex - + ", row " - + rowIndex); - } - values[bucketIndex][rowIndex][columnIndex] = probNode.asDouble(); - columnIndex++; - } - if (columnIndex != stateCount) { - throw new FactoryException( - "Row " - + rowIndex - + " in bucket " - + bucketIndex - + " had " - + columnIndex - + " columns. Expected " - + stateCount); - } - rowIndex++; - } - if (rowIndex != stateCount) { - throw new FactoryException( - "Bucket " + bucketIndex + " contained " + rowIndex + " rows. Expected " + stateCount); - } + fillBucket(values, bucketNode, bucketIndex, stateCount); bucketIndex++; } if (bucketIndex != buckets) { throw new FactoryException( "Transition values provided only " + bucketIndex + " buckets. Expected " + buckets); } + return values; + } - return new TransitionData(dtype, encoding, values); + private static void fillBucket( + double[][][] values, JsonNode bucketNode, int bucketIndex, int stateCount) { + if (bucketIndex >= values.length) { + throw new FactoryException("More transition buckets present than specified in shape"); + } + int rowIndex = 0; + for (JsonNode rowNode : bucketNode) { + fillRow(values, rowNode, bucketIndex, rowIndex, stateCount); + rowIndex++; + } + if (rowIndex != stateCount) { + throw new FactoryException( + "Bucket " + bucketIndex + " contained " + rowIndex + " rows. Expected " + stateCount); + } + } + + private static void fillRow( + double[][][] values, JsonNode rowNode, int bucketIndex, int rowIndex, int stateCount) { + if (rowIndex >= stateCount) { + throw new FactoryException("Too many rows in transition matrix for bucket " + bucketIndex); + } + int columnIndex = 0; + for (JsonNode probNode : rowNode) { + if (columnIndex >= stateCount) { + throw new FactoryException( + "Too many columns in transition matrix for bucket " + + bucketIndex + + ", row " + + rowIndex); + } + values[bucketIndex][rowIndex][columnIndex] = probNode.asDouble(); + columnIndex++; + } + if (columnIndex != stateCount) { + throw new FactoryException( + "Row " + + rowIndex + + " in bucket " + + bucketIndex + + " had " + + columnIndex + + " columns. Expected " + + stateCount); + } } private static GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { From dd38436fb7e0760b50303d269d0833c8706791f0 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 26 Nov 2025 09:40:58 +0300 Subject: [PATCH 06/39] refactor for sonar --- .../io/factory/markov/MarkovModelData.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java index e1cbdf3b0e..ca735ac2af 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java @@ -23,4 +23,18 @@ public MarkovModelData(JsonNode root) { public JsonNode getRoot() { return root; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof MarkovModelData that)) return false; + return Objects.equals(getTargetClass(), that.getTargetClass()) + && Objects.equals(getFieldsToValues(), that.getFieldsToValues()) + && Objects.equals(root, that.root); + } + + @Override + public int hashCode() { + return Objects.hash(getTargetClass(), getFieldsToValues(), root); + } } From b5a3ccd1b9443e17841f30dd5d28d20fd590cbb4 Mon Sep 17 00:00:00 2001 From: Philipp Date: Thu, 11 Dec 2025 01:53:06 +0100 Subject: [PATCH 07/39] abstraction --- .../markov/MarkovLoadModelFactory.java | 269 +--------------- .../markov/MarkovModelParsingSupport.java | 290 ++++++++++++++++++ 2 files changed, 292 insertions(+), 267 deletions(-) create mode 100644 src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java index 9461c2ffe1..7249fd3e2f 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -6,23 +6,18 @@ package edu.ie3.datamodel.io.factory.markov; import com.fasterxml.jackson.databind.JsonNode; -import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.io.factory.Factory; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.*; import java.time.ZonedDateTime; -import java.time.format.DateTimeParseException; -import java.util.ArrayList; -import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.TreeMap; /** Factory turning Markov JSON data into {@link MarkovLoadModel}s. */ public class MarkovLoadModelFactory - extends Factory { + extends Factory + implements MarkovModelParsingSupport { public MarkovLoadModelFactory() { super(MarkovLoadModel.class); @@ -75,264 +70,4 @@ protected List> getFields(Class entityClass) { "data.gmms.buckets"); return List.of(requiredFields); } - - private static Generator parseGenerator(JsonNode generatorNode) { - String name = requireText(generatorNode, "name"); - String version = requireText(generatorNode, "version"); - Map config = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - JsonNode configNode = generatorNode.path("config"); - if (configNode.isObject()) { - Iterator> fields = configNode.fields(); - while (fields.hasNext()) { - Map.Entry entry = fields.next(); - config.put(entry.getKey(), entry.getValue().asText()); - } - } - return new Generator(name, version, config); - } - - private static TimeModel parseTimeModel(JsonNode timeNode) { - int bucketCount = requireInt(timeNode, "bucket_count"); - String formula = requireNode(timeNode, "bucket_encoding").path("formula").asText(""); - if (formula.isEmpty()) { - throw new FactoryException("Missing bucket encoding formula"); - } - int samplingInterval = requireInt(timeNode, "sampling_interval_minutes"); - String timezone = requireText(timeNode, "timezone"); - return new TimeModel(bucketCount, formula, samplingInterval, timezone); - } - - private static ValueModel parseValueModel(JsonNode valueNode) { - String valueUnit = requireText(valueNode, "value_unit"); - JsonNode normalizationNode = requireNode(valueNode, "normalization"); - String normalizationMethod = requireText(normalizationNode, "method"); - ValueModel.Normalization normalization = new ValueModel.Normalization(normalizationMethod); - - JsonNode discretizationNode = requireNode(valueNode, "discretization"); - int states = requireInt(discretizationNode, "states"); - List thresholds = new ArrayList<>(); - JsonNode thresholdsNode = requireNode(discretizationNode, "thresholds_right"); - if (!thresholdsNode.isArray()) { - throw new FactoryException("thresholds_right must be an array"); - } - thresholdsNode.forEach(element -> thresholds.add(element.asDouble())); - ValueModel.Discretization discretization = - new ValueModel.Discretization(states, List.copyOf(thresholds)); - - return new ValueModel(valueUnit, normalization, discretization); - } - - private static Parameters parseParameters(JsonNode parametersNode) { - Parameters.TransitionParameters transitions = - new Parameters.TransitionParameters( - parametersNode.path("transitions").path("empty_row_strategy").asText("")); - if (transitions.emptyRowStrategy().isEmpty()) { - transitions = null; - } - - JsonNode gmmNode = parametersNode.path("gmm"); - Parameters.GmmParameters gmm = - gmmNode.isMissingNode() || gmmNode.isNull() || gmmNode.size() == 0 - ? null - : new Parameters.GmmParameters( - gmmNode.path("value_col").asText(""), - optionalInt(gmmNode, "verbose"), - optionalInt(gmmNode, "heartbeat_seconds")); - - return new Parameters(transitions, gmm); - } - - private static Optional optionalInt(JsonNode node, String field) { - JsonNode value = node.get(field); - if (value == null || value.isNull()) return Optional.empty(); - return Optional.of(value.asInt()); - } - - private static TransitionData parseTransitions( - JsonNode dataNode, int expectedBucketCount, int stateCount) { - JsonNode transitionsNode = requireNode(dataNode, "transitions"); - String dtype = requireText(transitionsNode, "dtype"); - String encoding = requireText(transitionsNode, "encoding"); - - int[] shape = parseTransitionShape(transitionsNode); - int buckets = shape[0]; - int rows = shape[1]; - int columns = shape[2]; - validateTransitionShape(expectedBucketCount, stateCount, buckets, rows, columns); - - JsonNode valuesNode = requireNode(transitionsNode, "values"); - double[][][] values = parseTransitionValues(valuesNode, buckets, stateCount); - - return new TransitionData(dtype, encoding, values); - } - - private static int[] parseTransitionShape(JsonNode transitionsNode) { - JsonNode shapeNode = requireNode(transitionsNode, "shape"); - if (!shapeNode.isArray() || shapeNode.size() != 3) { - throw new FactoryException("Transition shape must contain three dimensions"); - } - return new int[] {shapeNode.get(0).asInt(), shapeNode.get(1).asInt(), shapeNode.get(2).asInt()}; - } - - private static void validateTransitionShape( - int expectedBucketCount, int stateCount, int buckets, int rows, int columns) { - if (buckets != expectedBucketCount) { - throw new FactoryException( - "Transition bucket count mismatch. Expected " - + expectedBucketCount - + " but was " - + buckets); - } - if (rows != stateCount || columns != stateCount) { - throw new FactoryException( - "Transition state dimension mismatch. Expected " - + stateCount - + " but was rows=" - + rows - + ", columns=" - + columns); - } - } - - private static double[][][] parseTransitionValues( - JsonNode valuesNode, int buckets, int stateCount) { - if (!valuesNode.isArray()) { - throw new FactoryException("Transition values must be a three dimensional array"); - } - double[][][] values = new double[buckets][stateCount][stateCount]; - int bucketIndex = 0; - for (JsonNode bucketNode : valuesNode) { - fillBucket(values, bucketNode, bucketIndex, stateCount); - bucketIndex++; - } - if (bucketIndex != buckets) { - throw new FactoryException( - "Transition values provided only " + bucketIndex + " buckets. Expected " + buckets); - } - return values; - } - - private static void fillBucket( - double[][][] values, JsonNode bucketNode, int bucketIndex, int stateCount) { - if (bucketIndex >= values.length) { - throw new FactoryException("More transition buckets present than specified in shape"); - } - int rowIndex = 0; - for (JsonNode rowNode : bucketNode) { - fillRow(values, rowNode, bucketIndex, rowIndex, stateCount); - rowIndex++; - } - if (rowIndex != stateCount) { - throw new FactoryException( - "Bucket " + bucketIndex + " contained " + rowIndex + " rows. Expected " + stateCount); - } - } - - private static void fillRow( - double[][][] values, JsonNode rowNode, int bucketIndex, int rowIndex, int stateCount) { - if (rowIndex >= stateCount) { - throw new FactoryException("Too many rows in transition matrix for bucket " + bucketIndex); - } - int columnIndex = 0; - for (JsonNode probNode : rowNode) { - if (columnIndex >= stateCount) { - throw new FactoryException( - "Too many columns in transition matrix for bucket " - + bucketIndex - + ", row " - + rowIndex); - } - values[bucketIndex][rowIndex][columnIndex] = probNode.asDouble(); - columnIndex++; - } - if (columnIndex != stateCount) { - throw new FactoryException( - "Row " - + rowIndex - + " in bucket " - + bucketIndex - + " had " - + columnIndex - + " columns. Expected " - + stateCount); - } - } - - private static GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { - if (gmmsNode == null || gmmsNode.isMissingNode() || gmmsNode.isNull()) { - throw new FactoryException("Missing field 'gmms'"); - } - JsonNode bucketsNode = gmmsNode.get("buckets"); - if (!bucketsNode.isArray()) { - throw new FactoryException("data.gmms.buckets must be an array"); - } - List buckets = new ArrayList<>(); - for (JsonNode bucketNode : bucketsNode) { - JsonNode statesNode = bucketNode.get("states"); - if (statesNode == null || !statesNode.isArray()) { - throw new FactoryException("Each GMM bucket must contain an array 'states'"); - } - List> states = new ArrayList<>(); - for (JsonNode stateNode : statesNode) { - if (stateNode == null || stateNode.isNull()) { - states.add(Optional.empty()); - continue; - } - List weights = readDoubleArray(stateNode, "weights"); - List means = readDoubleArray(stateNode, "means"); - List variances = readDoubleArray(stateNode, "variances"); - states.add(Optional.of(new GmmBuckets.GmmState(weights, means, variances))); - } - buckets.add(new GmmBuckets.GmmBucket(List.copyOf(states))); - } - return new GmmBuckets(List.copyOf(buckets)); - } - - private static List readDoubleArray(JsonNode node, String field) { - JsonNode arrayNode = node.get(field); - if (arrayNode == null || !arrayNode.isArray()) { - throw new FactoryException("Field '" + field + "' must be an array"); - } - List values = new ArrayList<>(); - arrayNode.forEach(element -> values.add(element.asDouble())); - return List.copyOf(values); - } - - private static JsonNode requireNode(JsonNode node, String field) { - JsonNode value = node.get(field); - if (value == null || value.isMissingNode()) { - throw new FactoryException("Missing field '" + field + "'"); - } - return value; - } - - private static String requireText(JsonNode node, String field) { - JsonNode value = node.get(field); - if (value == null || value.isMissingNode() || value.isNull()) { - throw new FactoryException("Missing field '" + field + "'"); - } - if (!value.isTextual()) { - throw new FactoryException("Field '" + field + "' must be textual"); - } - return value.asText(); - } - - private static int requireInt(JsonNode node, String field) { - JsonNode value = node.get(field); - if (value == null || value.isMissingNode() || value.isNull()) { - throw new FactoryException("Missing field '" + field + "'"); - } - if (!value.canConvertToInt()) { - throw new FactoryException("Field '" + field + "' must be an integer"); - } - return value.asInt(); - } - - private static ZonedDateTime parseTimestamp(String timestamp) { - try { - return ZonedDateTime.parse(timestamp); - } catch (DateTimeParseException e) { - throw new FactoryException("Unable to parse generated_at timestamp '" + timestamp + "'", e); - } - } } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java new file mode 100644 index 0000000000..db593f0f36 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -0,0 +1,290 @@ +/* + * ЖИ 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.factory.markov; + +import com.fasterxml.jackson.databind.JsonNode; +import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.Generator; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.GmmBuckets; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.Parameters; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.TimeModel; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.TransitionData; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.ValueModel; +import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; + +/** + * Shared parsing helpers for Markov model JSON documents. This is intentionally package-private as + * it is only meant to be reused across factory implementations in this package. + */ +interface MarkovModelParsingSupport { + + default Generator parseGenerator(JsonNode generatorNode) { + String name = requireText(generatorNode, "name"); + String version = requireText(generatorNode, "version"); + Map config = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + JsonNode configNode = generatorNode.path("config"); + if (configNode.isObject()) { + Iterator> fields = configNode.fields(); + while (fields.hasNext()) { + Map.Entry entry = fields.next(); + config.put(entry.getKey(), entry.getValue().asText()); + } + } + return new Generator(name, version, config); + } + + default TimeModel parseTimeModel(JsonNode timeNode) { + int bucketCount = requireInt(timeNode, "bucket_count"); + String formula = requireNode(timeNode, "bucket_encoding").path("formula").asText(""); + if (formula.isEmpty()) { + throw new FactoryException("Missing bucket encoding formula"); + } + int samplingInterval = requireInt(timeNode, "sampling_interval_minutes"); + String timezone = requireText(timeNode, "timezone"); + return new TimeModel(bucketCount, formula, samplingInterval, timezone); + } + + default ValueModel parseValueModel(JsonNode valueNode) { + String valueUnit = requireText(valueNode, "value_unit"); + JsonNode normalizationNode = requireNode(valueNode, "normalization"); + String normalizationMethod = requireText(normalizationNode, "method"); + ValueModel.Normalization normalization = new ValueModel.Normalization(normalizationMethod); + + JsonNode discretizationNode = requireNode(valueNode, "discretization"); + int states = requireInt(discretizationNode, "states"); + List thresholds = new ArrayList<>(); + JsonNode thresholdsNode = requireNode(discretizationNode, "thresholds_right"); + if (!thresholdsNode.isArray()) { + throw new FactoryException("thresholds_right must be an array"); + } + thresholdsNode.forEach(element -> thresholds.add(element.asDouble())); + ValueModel.Discretization discretization = + new ValueModel.Discretization(states, List.copyOf(thresholds)); + + return new ValueModel(valueUnit, normalization, discretization); + } + + default Parameters parseParameters(JsonNode parametersNode) { + Parameters.TransitionParameters transitions = + new Parameters.TransitionParameters( + parametersNode.path("transitions").path("empty_row_strategy").asText("")); + if (transitions.emptyRowStrategy().isEmpty()) { + transitions = null; + } + + JsonNode gmmNode = parametersNode.path("gmm"); + Parameters.GmmParameters gmm = + gmmNode.isMissingNode() || gmmNode.isNull() || gmmNode.size() == 0 + ? null + : new Parameters.GmmParameters( + gmmNode.path("value_col").asText(""), + optionalInt(gmmNode, "verbose"), + optionalInt(gmmNode, "heartbeat_seconds")); + + return new Parameters(transitions, gmm); + } + + default TransitionData parseTransitions( + JsonNode dataNode, int expectedBucketCount, int stateCount) { + JsonNode transitionsNode = requireNode(dataNode, "transitions"); + String dtype = requireText(transitionsNode, "dtype"); + String encoding = requireText(transitionsNode, "encoding"); + + int[] shape = parseTransitionShape(transitionsNode); + int buckets = shape[0]; + int rows = shape[1]; + int columns = shape[2]; + validateTransitionShape(expectedBucketCount, stateCount, buckets, rows, columns); + + JsonNode valuesNode = requireNode(transitionsNode, "values"); + double[][][] values = parseTransitionValues(valuesNode, buckets, stateCount); + + return new TransitionData(dtype, encoding, values); + } + + default GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { + if (gmmsNode == null || gmmsNode.isMissingNode() || gmmsNode.isNull()) { + throw new FactoryException("Missing field 'gmms'"); + } + JsonNode bucketsNode = gmmsNode.get("buckets"); + if (!bucketsNode.isArray()) { + throw new FactoryException("data.gmms.buckets must be an array"); + } + List buckets = new ArrayList<>(); + for (JsonNode bucketNode : bucketsNode) { + JsonNode statesNode = bucketNode.get("states"); + if (statesNode == null || !statesNode.isArray()) { + throw new FactoryException("Each GMM bucket must contain an array 'states'"); + } + List> states = new ArrayList<>(); + for (JsonNode stateNode : statesNode) { + if (stateNode == null || stateNode.isNull()) { + states.add(Optional.empty()); + continue; + } + List weights = readDoubleArray(stateNode, "weights"); + List means = readDoubleArray(stateNode, "means"); + List variances = readDoubleArray(stateNode, "variances"); + states.add(Optional.of(new GmmBuckets.GmmState(weights, means, variances))); + } + buckets.add(new GmmBuckets.GmmBucket(List.copyOf(states))); + } + return new GmmBuckets(List.copyOf(buckets)); + } + + default JsonNode requireNode(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode()) { + throw new FactoryException("Missing field '" + field + "'"); + } + return value; + } + + default String requireText(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode() || value.isNull()) { + throw new FactoryException("Missing field '" + field + "'"); + } + if (!value.isTextual()) { + throw new FactoryException("Field '" + field + "' must be textual"); + } + return value.asText(); + } + + default int requireInt(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode() || value.isNull()) { + throw new FactoryException("Missing field '" + field + "'"); + } + if (!value.canConvertToInt()) { + throw new FactoryException("Field '" + field + "' must be an integer"); + } + return value.asInt(); + } + + default ZonedDateTime parseTimestamp(String timestamp) { + try { + return ZonedDateTime.parse(timestamp); + } catch (DateTimeParseException e) { + throw new FactoryException("Unable to parse generated_at timestamp '" + timestamp + "'", e); + } + } + + default Optional optionalInt(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isNull()) return Optional.empty(); + return Optional.of(value.asInt()); + } + + default int[] parseTransitionShape(JsonNode transitionsNode) { + JsonNode shapeNode = requireNode(transitionsNode, "shape"); + if (!shapeNode.isArray() || shapeNode.size() != 3) { + throw new FactoryException("Transition shape must contain three dimensions"); + } + return new int[] {shapeNode.get(0).asInt(), shapeNode.get(1).asInt(), shapeNode.get(2).asInt()}; + } + + default void validateTransitionShape( + int expectedBucketCount, int stateCount, int buckets, int rows, int columns) { + if (buckets != expectedBucketCount) { + throw new FactoryException( + "Transition bucket count mismatch. Expected " + + expectedBucketCount + + " but was " + + buckets); + } + if (rows != stateCount || columns != stateCount) { + throw new FactoryException( + "Transition state dimension mismatch. Expected " + + stateCount + + " but was rows=" + + rows + + ", columns=" + + columns); + } + } + + default double[][][] parseTransitionValues( + JsonNode valuesNode, int buckets, int stateCount) { + if (!valuesNode.isArray()) { + throw new FactoryException("Transition values must be a three dimensional array"); + } + double[][][] values = new double[buckets][stateCount][stateCount]; + int bucketIndex = 0; + for (JsonNode bucketNode : valuesNode) { + fillBucket(values, bucketNode, bucketIndex, stateCount); + bucketIndex++; + } + if (bucketIndex != buckets) { + throw new FactoryException( + "Transition values provided only " + bucketIndex + " buckets. Expected " + buckets); + } + return values; + } + + default void fillBucket( + double[][][] values, JsonNode bucketNode, int bucketIndex, int stateCount) { + if (bucketIndex >= values.length) { + throw new FactoryException("More transition buckets present than specified in shape"); + } + int rowIndex = 0; + for (JsonNode rowNode : bucketNode) { + fillRow(values, rowNode, bucketIndex, rowIndex, stateCount); + rowIndex++; + } + if (rowIndex != stateCount) { + throw new FactoryException( + "Bucket " + bucketIndex + " contained " + rowIndex + " rows. Expected " + stateCount); + } + } + + default void fillRow( + double[][][] values, JsonNode rowNode, int bucketIndex, int rowIndex, int stateCount) { + if (rowIndex >= stateCount) { + throw new FactoryException("Too many rows in transition matrix for bucket " + bucketIndex); + } + int columnIndex = 0; + for (JsonNode probNode : rowNode) { + if (columnIndex >= stateCount) { + throw new FactoryException( + "Too many columns in transition matrix for bucket " + + bucketIndex + + ", row " + + rowIndex); + } + values[bucketIndex][rowIndex][columnIndex] = probNode.asDouble(); + columnIndex++; + } + if (columnIndex != stateCount) { + throw new FactoryException( + "Row " + + rowIndex + + " in bucket " + + bucketIndex + + " had " + + columnIndex + + " columns. Expected " + + stateCount); + } + } + + default List readDoubleArray(JsonNode node, String field) { + JsonNode arrayNode = node.get(field); + if (arrayNode == null || !arrayNode.isArray()) { + throw new FactoryException("Field '" + field + "' must be an array"); + } + List values = new ArrayList<>(); + arrayNode.forEach(element -> values.add(element.asDouble())); + return List.copyOf(values); + } +} From fbf01ee3b07e7deda2fb1dd08c74bd3102d240e2 Mon Sep 17 00:00:00 2001 From: Philipp Date: Thu, 11 Dec 2025 02:28:31 +0100 Subject: [PATCH 08/39] spotless --- .../io/factory/markov/MarkovModelParsingSupport.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java index db593f0f36..fa360374e1 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -1,8 +1,8 @@ /* - * ЖИ 2025. TU Dortmund University, + * © 2025. TU Dortmund University, * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation - */ +*/ package edu.ie3.datamodel.io.factory.markov; import com.fasterxml.jackson.databind.JsonNode; @@ -214,8 +214,7 @@ default void validateTransitionShape( } } - default double[][][] parseTransitionValues( - JsonNode valuesNode, int buckets, int stateCount) { + default double[][][] parseTransitionValues(JsonNode valuesNode, int buckets, int stateCount) { if (!valuesNode.isArray()) { throw new FactoryException("Transition values must be a three dimensional array"); } From 59bb6d081bd300f8ef4e2901828453769644cf4d Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 5 Jan 2026 18:18:24 +0100 Subject: [PATCH 09/39] Completig PSDM Markov Logic by feeding PowerValueIdentifier and building Supplier --- .../markov/MarkovModelParsingSupport.java | 31 +- .../datamodel/io/source/PowerValueSource.java | 49 ++- .../source/json/JsonMarkovProfileSource.java | 61 +++- .../source/markov/MarkovLoadValueSource.java | 341 ++++++++++++++++++ .../profile/markov/MarkovLoadModel.java | 6 +- .../profile/markov/MarkovPowerProfile.java | 25 ++ .../connectors/JsonFileConnectorTest.groovy | 1 + .../markov/MarkovLoadModelFactoryTest.groovy | 10 +- .../json/JsonMarkovProfileSourceTest.groovy | 49 ++- .../markov/MarkovLoadValueSourceTest.groovy | 241 +++++++++++++ 10 files changed, 802 insertions(+), 12 deletions(-) create mode 100644 src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java create mode 100644 src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovPowerProfile.java create mode 100644 src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java index fa360374e1..05c24a7ff7 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -58,7 +58,11 @@ default ValueModel parseValueModel(JsonNode valueNode) { String valueUnit = requireText(valueNode, "value_unit"); JsonNode normalizationNode = requireNode(valueNode, "normalization"); String normalizationMethod = requireText(normalizationNode, "method"); - ValueModel.Normalization normalization = new ValueModel.Normalization(normalizationMethod); + ValueModel.Normalization normalization = + new ValueModel.Normalization( + normalizationMethod, + parsePowerReference(normalizationNode, "reference_power"), + parsePowerReference(normalizationNode, "min_power")); JsonNode discretizationNode = requireNode(valueNode, "discretization"); int states = requireInt(discretizationNode, "states"); @@ -161,6 +165,17 @@ default String requireText(JsonNode node, String field) { return value.asText(); } + default double requireDouble(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode() || value.isNull()) { + throw new FactoryException("Missing field '" + field + "'"); + } + if (!value.isNumber()) { + throw new FactoryException("Field '" + field + "' must be numeric"); + } + return value.asDouble(); + } + default int requireInt(JsonNode node, String field) { JsonNode value = node.get(field); if (value == null || value.isMissingNode() || value.isNull()) { @@ -286,4 +301,18 @@ default List readDoubleArray(JsonNode node, String field) { arrayNode.forEach(element -> values.add(element.asDouble())); return List.copyOf(values); } + + default Optional parsePowerReference( + JsonNode parent, String field) { + JsonNode referenceNode = parent.get(field); + if (referenceNode == null || referenceNode.isMissingNode() || referenceNode.isNull()) { + return Optional.empty(); + } + if (!referenceNode.isObject()) { + throw new FactoryException("Field '" + field + "' must be an object"); + } + double value = requireDouble(referenceNode, "value"); + String unit = requireText(referenceNode, "unit"); + return Optional.of(new ValueModel.Normalization.PowerReference(value, unit)); + } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java b/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java index aca55b78a1..ed170dd5c3 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java @@ -9,7 +9,10 @@ import edu.ie3.datamodel.models.profile.PowerProfile; import edu.ie3.datamodel.models.value.PValue; import java.time.ZonedDateTime; +import java.util.Objects; import java.util.Optional; +import java.util.OptionalDouble; +import java.util.OptionalInt; import java.util.function.Supplier; import javax.measure.quantity.Energy; import javax.measure.quantity.Power; @@ -55,7 +58,19 @@ non-sealed interface TimeSeriesBased extends PowerValueSource {} /** Interface for markov-chain-based power value sources. */ - non-sealed interface MarkovBased extends PowerValueSource {} + non-sealed interface MarkovBased + extends PowerValueSource { + + @Override + MarkovValueSupplier getValueSupplier(MarkovInputValue data); + + interface MarkovValueSupplier extends Supplier> { + /** + * Returns the next state that should be provided as {@link MarkovInputValue#previousState()}. + */ + int getNextState(); + } + } // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= // input data @@ -64,7 +79,8 @@ non-sealed interface MarkovBased extends PowerValueSource referencePower, + long randomSeed) + implements PowerValueIdentifier { + + public MarkovInputValue { + Objects.requireNonNull(time, "time"); + Objects.requireNonNull(previousState, "previousState"); + Objects.requireNonNull(initialNormalizedValue, "initialNormalizedValue"); + Objects.requireNonNull(referencePower, "referencePower"); + if (previousState.isEmpty() && initialNormalizedValue.isEmpty()) { + throw new IllegalArgumentException( + "Need either previous state or an initial normalized value to start the Markov chain."); + } + } + + @Override + public ZonedDateTime getTime() { + return time; + } + } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java index d3240a896f..b900328f53 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java @@ -16,22 +16,32 @@ import edu.ie3.datamodel.io.file.FileType; import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation; import edu.ie3.datamodel.io.source.EntitySource; +import edu.ie3.datamodel.io.source.PowerValueSource; +import edu.ie3.datamodel.io.source.markov.MarkovLoadValueSource; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; +import edu.ie3.datamodel.models.profile.markov.MarkovPowerProfile; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; +import java.time.ZonedDateTime; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.TreeSet; +import javax.measure.quantity.Energy; +import javax.measure.quantity.Power; +import tech.units.indriya.ComparableQuantity; /** Source that reads Markov-based load models from JSON files. */ -public class JsonMarkovProfileSource extends EntitySource { +public class JsonMarkovProfileSource extends EntitySource implements PowerValueSource.MarkovBased { private final JsonDataSource dataSource; private final FileLoadProfileMetaInformation metaInformation; private final MarkovLoadModelFactory factory; private final ObjectMapper objectMapper = new ObjectMapper(); + private final MarkovPowerProfile profile; private MarkovLoadModel cachedModel; + private volatile MarkovLoadValueSource delegate; public JsonMarkovProfileSource( JsonDataSource dataSource, FileLoadProfileMetaInformation metaInformation) { @@ -45,6 +55,7 @@ public JsonMarkovProfileSource( this.dataSource = Objects.requireNonNull(dataSource, "dataSource"); this.metaInformation = Objects.requireNonNull(metaInformation, "metaInformation"); this.factory = Objects.requireNonNull(factory, "factory"); + this.profile = new MarkovPowerProfile(metaInformation.getProfile()); if (metaInformation.getFileType() != FileType.JSON) { throw new IllegalArgumentException("Markov profile source requires JSON meta information."); } @@ -81,6 +92,31 @@ public void validate() throws ValidationException { factory.validate(fields, MarkovLoadModel.class).getOrThrow(); } + @Override + public MarkovPowerProfile getProfile() { + return profile; + } + + @Override + public MarkovValueSupplier getValueSupplier(PowerValueSource.MarkovInputValue data) { + return getDelegate().getValueSupplier(data); + } + + @Override + public Optional getNextTimeKey(ZonedDateTime time) { + return getDelegate().getNextTimeKey(time); + } + + @Override + public Optional> getMaxPower() { + return getDelegate().getMaxPower(); + } + + @Override + public Optional> getProfileEnergyScaling() { + return getDelegate().getProfileEnergyScaling(); + } + private JsonNode readModelTree() throws SourceException { Path filePath = metaInformation.getFullFilePath(); try (InputStream inputStream = dataSource.initInputStream(filePath)) { @@ -114,4 +150,27 @@ private static void collectFields(String prefix, JsonNode node, Set coll private static String join(String prefix, String name) { return prefix.isEmpty() ? name : prefix + "." + name; } + + private MarkovLoadValueSource getDelegate() { + MarkovLoadValueSource current = delegate; + if (current == null) { + synchronized (this) { + current = delegate; + if (current == null) { + current = new MarkovLoadValueSource(profile, loadModelUnchecked()); + delegate = current; + } + } + } + return current; + } + + private MarkovLoadModel loadModelUnchecked() { + try { + return getModel(); + } catch (SourceException e) { + throw new IllegalStateException( + "Unable to load Markov model '" + metaInformation.getProfile() + "'.", e); + } + } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java b/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java new file mode 100644 index 0000000000..6703d91332 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java @@ -0,0 +1,341 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.source.markov; + +import edu.ie3.datamodel.io.source.PowerValueSource.MarkovBased; +import edu.ie3.datamodel.io.source.PowerValueSource.MarkovInputValue; +import edu.ie3.datamodel.models.StandardUnits; +import edu.ie3.datamodel.models.profile.PowerProfile; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.GmmBuckets; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.ValueModel; +import edu.ie3.datamodel.models.value.PValue; +import java.time.DayOfWeek; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.SplittableRandom; +import javax.measure.quantity.Energy; +import javax.measure.quantity.Power; +import tech.units.indriya.ComparableQuantity; +import tech.units.indriya.quantity.Quantities; + +/** + * Implementation of a {@link MarkovBased} power value source that converts a {@link + * MarkovLoadModel} export into {@link PValue}s. + */ +public class MarkovLoadValueSource implements MarkovBased { + + private static final int QUARTER_HOURS_PER_DAY = 96; + private static final int WEEKEND_FACTOR = QUARTER_HOURS_PER_DAY; + private static final int MONTH_FACTOR = QUARTER_HOURS_PER_DAY * 2; + + private final PowerProfile profile; + private final MarkovLoadModel model; + private final ZoneId zoneId; + private final double[][][] transitions; + private final int bucketCount; + private final int stateCount; + private final int samplingIntervalMinutes; + private final double[] discretizationThresholds; + private final GmmStateData[][] gmmStates; + private final Optional> referencePowerFromModel; + + public MarkovLoadValueSource(PowerProfile profile, MarkovLoadModel model) { + this.profile = Objects.requireNonNull(profile, "profile"); + this.model = Objects.requireNonNull(model, "model"); + this.zoneId = ZoneId.of(model.timeModel().timezone()); + this.transitions = model.transitionData().values(); + this.bucketCount = model.timeModel().bucketCount(); + this.stateCount = model.valueModel().discretization().states(); + this.samplingIntervalMinutes = model.timeModel().samplingIntervalMinutes(); + this.discretizationThresholds = + model.valueModel().discretization().thresholdsRight().stream() + .mapToDouble(Double::doubleValue) + .toArray(); + this.gmmStates = + buildGmmStates( + model + .gmmBuckets() + .orElseThrow(() -> new IllegalArgumentException("Markov model lacks GMM data."))); + this.referencePowerFromModel = + model.valueModel().normalization().referencePower().map(this::convertReferencePower); + } + + @Override + public PowerProfile getProfile() { + return profile; + } + + @Override + public MarkovValueSupplier getValueSupplier(MarkovInputValue data) { + Objects.requireNonNull(data, "data"); + return new MarkovSupplier(data); + } + + @Override + public Optional getNextTimeKey(ZonedDateTime time) { + return Optional.ofNullable(time).map(t -> t.plusMinutes(samplingIntervalMinutes)); + } + + @Override + public Optional> getMaxPower() { + return referencePowerFromModel; + } + + @Override + public Optional> getProfileEnergyScaling() { + return Optional.empty(); + } + + private GmmStateData[][] buildGmmStates(GmmBuckets buckets) { + List bucketList = buckets.buckets(); + if (bucketList.size() != bucketCount) { + throw new IllegalArgumentException( + "GMM bucket count mismatch. Expected " + bucketCount + " but was " + bucketList.size()); + } + GmmStateData[][] lookup = new GmmStateData[bucketCount][stateCount]; + for (int bucket = 0; bucket < bucketCount; bucket++) { + List> states = bucketList.get(bucket).states(); + if (states.size() != stateCount) { + throw new IllegalArgumentException( + "State count mismatch in bucket " + bucket + ". Expected " + stateCount); + } + for (int state = 0; state < stateCount; state++) { + lookup[bucket][state] = states.get(state).map(GmmStateData::from).orElse(null); + } + } + return lookup; + } + + private int bucketId(ZonedDateTime time) { + ZonedDateTime zoned = time.withZoneSameInstant(zoneId); + int month = zoned.getMonthValue() - 1; + int isWeekend = isWeekend(zoned) ? 1 : 0; + int quarterHour = zoned.getHour() * 4 + zoned.getMinute() / 15; + return Math.floorMod( + month * MONTH_FACTOR + isWeekend * WEEKEND_FACTOR + quarterHour, bucketCount); + } + + private boolean isWeekend(ZonedDateTime time) { + DayOfWeek day = time.getDayOfWeek(); + return day == DayOfWeek.SATURDAY || day == DayOfWeek.SUNDAY; + } + + private int resolveState(MarkovInputValue input) { + if (input.previousState().isPresent()) { + int state = input.previousState().getAsInt(); + if (state < 0 || state >= stateCount) { + throw new IllegalArgumentException("Previous state out of bounds: " + state); + } + return state; + } + double normalized = input.initialNormalizedValue().orElseThrow(); + return discretize(normalized); + } + + private int discretize(double normalized) { + double value = clamp01(normalized); + for (int i = 0; i < discretizationThresholds.length; i++) { + if (value <= discretizationThresholds[i]) { + return i; + } + } + return discretizationThresholds.length; + } + + private StepResult simulateStep(int bucket, int currentState, SplittableRandom rng) { + double[] row = transitions[bucket][currentState]; + double[] distribution = sanitizeDistribution(bucket, row); + if (distribution == null) { + return new StepResult(currentState, 0d); + } + int nextState = drawState(distribution, rng); + double normalized = sampleNormalizedValue(bucket, nextState, rng); + return new StepResult(nextState, normalized); + } + + private double[] sanitizeDistribution(int bucket, double[] row) { + double[] sanitized = new double[stateCount]; + double sum = 0d; + for (int state = 0; state < stateCount; state++) { + double value = state < row.length ? row[state] : 0d; + if (value <= 0d || Double.isNaN(value)) { + sanitized[state] = 0d; + continue; + } + if (gmmStates[bucket][state] == null) { + sanitized[state] = 0d; + continue; + } + sanitized[state] = value; + sum += value; + } + if (sum <= 0d) { + return null; + } + for (int i = 0; i < sanitized.length; i++) { + sanitized[i] /= sum; + } + return sanitized; + } + + private int drawState(double[] distribution, SplittableRandom rng) { + double sample = rng.nextDouble(); + double cumulative = 0d; + for (int i = 0; i < distribution.length; i++) { + cumulative += distribution[i]; + if (sample <= cumulative) { + return i; + } + } + return distribution.length - 1; + } + + private double sampleNormalizedValue(int bucket, int state, SplittableRandom rng) { + GmmStateData gmm = gmmStates[bucket][state]; + if (gmm == null) { + return 0d; + } + return clamp01(gmm.sample(rng)); + } + + private long deriveSeed(MarkovInputValue input, int bucket, int state) { + long seed = input.randomSeed(); + seed = 31 * seed + bucket; + seed = 31 * seed + state; + long slot = + input.time().withZoneSameInstant(zoneId).toInstant().toEpochMilli() + / (samplingIntervalMinutes * 60_000L); + return 31 * seed + slot; + } + + private ComparableQuantity scale( + ComparableQuantity referencePower, double normalizedValue) { + double clamped = clamp01(normalizedValue); + @SuppressWarnings("unchecked") + ComparableQuantity scaled = + (ComparableQuantity) referencePower.multiply(clamped).asType(Power.class); + return scaled; + } + + private ComparableQuantity convertReferencePower( + ValueModel.Normalization.PowerReference reference) { + if (!"kW".equalsIgnoreCase(reference.unit())) { + throw new IllegalArgumentException( + "Unsupported reference power unit '" + reference.unit() + "'. Only kW is supported."); + } + return Quantities.getQuantity(reference.value(), StandardUnits.ACTIVE_POWER_IN); + } + + private static double clamp01(double value) { + if (value < 0d) return 0d; + if (value > 1d) return 1d; + return value; + } + + private StepResult calculate(MarkovInputValue input) { + int bucket = bucketId(input.time()); + int currentState = resolveState(input); + SplittableRandom rng = new SplittableRandom(deriveSeed(input, bucket, currentState)); + StepResult step = simulateStep(bucket, currentState, rng); + ComparableQuantity power = scale(input.referencePower(), step.normalizedValue()); + return new StepResult(step.nextState(), step.normalizedValue(), Optional.of(new PValue(power))); + } + + private record StepResult(int nextState, double normalizedValue, Optional value) { + private StepResult(int nextState, double normalizedValue) { + this(nextState, normalizedValue, Optional.empty()); + } + } + + private static final class GmmStateData { + private final double[] weights; + private final double[] means; + private final double[] variances; + + private GmmStateData(double[] weights, double[] means, double[] variances) { + this.weights = weights; + this.means = means; + this.variances = variances; + } + + private static GmmStateData from(GmmBuckets.GmmState state) { + return new GmmStateData( + toArray(state.weights()), toArray(state.means()), toArray(state.variances())); + } + + private static double[] toArray(List values) { + double[] array = new double[values.size()]; + for (int i = 0; i < values.size(); i++) { + array[i] = values.get(i); + } + return array; + } + + private double sample(SplittableRandom rng) { + int component = drawComponent(rng.nextDouble()); + double mean = means[component]; + double variance = Math.max(0d, variances[component]); + if (variance == 0d) { + return mean; + } + return mean + Math.sqrt(variance) * nextGaussian(rng); + } + + private int drawComponent(double sample) { + double cumulative = 0d; + for (int i = 0; i < weights.length; i++) { + cumulative += weights[i]; + if (sample <= cumulative) { + return i; + } + } + return weights.length - 1; + } + + private double nextGaussian(SplittableRandom rng) { + double u1 = Math.max(Double.MIN_VALUE, rng.nextDouble()); + double u2 = rng.nextDouble(); + return Math.sqrt(-2.0d * Math.log(u1)) * Math.cos(2.0d * Math.PI * u2); + } + } + + private final class MarkovSupplier implements MarkovValueSupplier { + private final MarkovInputValue input; + private Optional cached = Optional.empty(); + private Integer nextState; + private boolean evaluated; + + private MarkovSupplier(MarkovInputValue input) { + this.input = input; + } + + @Override + public Optional get() { + evaluate(); + return cached; + } + + @Override + public int getNextState() { + evaluate(); + return nextState; + } + + private void evaluate() { + if (evaluated) { + return; + } + evaluated = true; + StepResult result = calculate(input); + this.cached = result.value(); + this.nextState = result.nextState(); + } + } +} diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index c791491b12..ae4c2978ca 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -34,7 +34,11 @@ public record TimeModel( public record ValueModel( String valueUnit, Normalization normalization, Discretization discretization) { - public record Normalization(String method) {} + public record Normalization( + String method, Optional referencePower, Optional minPower) { + + public record PowerReference(double value, String unit) {} + } public record Discretization(int states, List thresholdsRight) {} } diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovPowerProfile.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovPowerProfile.java new file mode 100644 index 0000000000..bd6d10a963 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovPowerProfile.java @@ -0,0 +1,25 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.models.profile.markov; + +import edu.ie3.datamodel.models.profile.PowerProfile; +import java.util.Objects; + +/** Simple {@link PowerProfile} implementation for Markov-based load models. */ +public record MarkovPowerProfile(String key) implements PowerProfile { + + public MarkovPowerProfile { + Objects.requireNonNull(key, "key"); + if (key.isBlank()) { + throw new IllegalArgumentException("Profile key must not be blank."); + } + } + + @Override + public String getKey() { + return key; + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy index 49a3877d37..ff6509d158 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy @@ -48,6 +48,7 @@ class JsonFileConnectorTest extends Specification { when: def reader = connector.initReader(Path.of("data")) def line = reader.readLine() + reader.close() then: line == "[1,2,3]" diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy index b60d2864a6..a4cb7db3e6 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy @@ -33,6 +33,11 @@ class MarkovLoadModelFactoryTest extends Specification { gmmState.weights() == [0.6d] gmmState.means() == [1.0d] gmmState.variances() == [0.2d] + model.valueModel().normalization().referencePower().isPresent() + with(model.valueModel().normalization().referencePower().get()) { + value() == 1.5d + unit() == "kW" + } } def "buildModel throws FactoryException on transition dimension mismatch"() { @@ -64,7 +69,10 @@ class MarkovLoadModelFactoryTest extends Specification { }, "value_model": { "value_unit": "W", - "normalization": { "method": "none" }, + "normalization": { + "method": "none", + "reference_power": { "value": 1.5, "unit": "kW" } + }, "discretization": { "states": 2, "thresholds_right": [0.5] diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy index 1b20cd92fc..681d9590cb 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy @@ -9,11 +9,17 @@ import edu.ie3.datamodel.exceptions.SourceException import edu.ie3.datamodel.io.file.FileType import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation +import edu.ie3.datamodel.io.source.PowerValueSource +import edu.ie3.datamodel.models.StandardUnits import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel import spock.lang.Specification +import tech.units.indriya.quantity.Quantities import java.nio.file.Files import java.nio.file.Path +import java.time.ZonedDateTime +import java.util.OptionalDouble +import java.util.OptionalInt class JsonMarkovProfileSourceTest extends Specification { @@ -72,6 +78,34 @@ class JsonMarkovProfileSourceTest extends Specification { thrown(SourceException) } + def "source exposes Markov-based power value supplier"() { + given: + Files.writeString(jsonFile, validModelJson()) + def source = new JsonMarkovProfileSource( + new JsonDataSource(tempDir, new FileNamingStrategy()), + new FileLoadProfileMetaInformation("profile1", jsonFile, FileType.JSON) + ) + def referencePower = Quantities.getQuantity(10d, StandardUnits.ACTIVE_POWER_IN) + def input = new PowerValueSource.MarkovInputValue( + ZonedDateTime.parse("2025-01-01T00:00:00Z"), + OptionalInt.of(0), + OptionalDouble.empty(), + referencePower, + 42L) + + when: + def supplier = source.getValueSupplier(input) + def value = supplier.get() + + then: + source.getProfile().key() == "profile1" + source.getMaxPower().isPresent() + source.getMaxPower().get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 10d + value.isPresent() + supplier.getNextState() == 0 + value.get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 10d + } + private static String validModelJson() { return """ { @@ -90,7 +124,10 @@ class JsonMarkovProfileSourceTest extends Specification { }, "value_model": { "value_unit": "W", - "normalization": { "method": "none" }, + "normalization": { + "method": "none", + "reference_power": { "value": 10.0, "unit": "kW" } + }, "discretization": { "states": 2, "thresholds_right": [0.5] @@ -120,11 +157,11 @@ class JsonMarkovProfileSourceTest extends Specification { "buckets": [ { "states": [ - { - "weights": [0.6], - "means": [1.0], - "variances": [0.2] - }, + { + "weights": [0.6], + "means": [1.0], + "variances": [0.0] + }, null ] } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy new file mode 100644 index 0000000000..6f50acfc81 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy @@ -0,0 +1,241 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.markov + +import com.fasterxml.jackson.databind.ObjectMapper +import edu.ie3.datamodel.io.factory.markov.MarkovLoadModelFactory +import edu.ie3.datamodel.io.factory.markov.MarkovModelData +import edu.ie3.datamodel.io.source.PowerValueSource +import edu.ie3.datamodel.models.StandardUnits +import edu.ie3.datamodel.models.profile.markov.MarkovPowerProfile +import spock.lang.Specification +import tech.units.indriya.quantity.Quantities + +import java.time.ZonedDateTime +import java.util.OptionalDouble +import java.util.OptionalInt + +class MarkovLoadValueSourceTest extends Specification { + + private final ObjectMapper objectMapper = new ObjectMapper() + private final MarkovLoadModelFactory factory = new MarkovLoadModelFactory() + private final MarkovPowerProfile profile = new MarkovPowerProfile("profile1") + + def "supplier scales deterministic normalized values and exposes next state"() { + given: + def model = loadModel(deterministicTransitions(), deterministicStates()) + def source = new MarkovLoadValueSource(profile, model) + def reference = Quantities.getQuantity(5d, StandardUnits.ACTIVE_POWER_IN) + def input = new PowerValueSource.MarkovInputValue( + ZonedDateTime.parse("2025-01-01T00:00:00Z"), + OptionalInt.of(0), + OptionalDouble.empty(), + reference, + 99L + ) + + when: + def supplier = source.getValueSupplier(input) + def value = supplier.get() + + then: + value.isPresent() + supplier.getNextState() == 1 + value.get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 4d + supplier.get().is(value) // cached result reused + source.getMaxPower().isPresent() + source.getMaxPower().get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 5d + } + + def "supplier uses initial normalized value when no previous state is present"() { + given: + def model = loadModel(selfLoopTransitions(), deterministicStates()) + def source = new MarkovLoadValueSource(profile, model) + def reference = Quantities.getQuantity(5d, StandardUnits.ACTIVE_POWER_IN) + def input = new PowerValueSource.MarkovInputValue( + ZonedDateTime.parse("2025-01-01T00:00:00Z"), + OptionalInt.empty(), + OptionalDouble.of(0.25d), + reference, + 13L + ) + + when: + def supplier = source.getValueSupplier(input) + + then: + supplier.getNextState() == 0 // discretized from initial normalized value + supplier.get().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 2d + } + + def "supplier returns zero power when transitions row has no usable probabilities"() { + given: + def model = loadModel(emptyTransitions(), missingStateGmms()) + def source = new MarkovLoadValueSource(profile, model) + def reference = Quantities.getQuantity(3d, StandardUnits.ACTIVE_POWER_IN) + def input = new PowerValueSource.MarkovInputValue( + ZonedDateTime.parse("2025-01-01T00:00:00Z"), + OptionalInt.of(0), + OptionalDouble.empty(), + reference, + 7L + ) + + when: + def supplier = source.getValueSupplier(input) + def value = supplier.get() + + then: + supplier.getNextState() == 0 // stays in current state + value.isPresent() + value.get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 0d + } + + def "reference power metadata optional"() { + given: + def model = loadModel(deterministicTransitions(), deterministicStates(), false) + def source = new MarkovLoadValueSource(profile, model) + + expect: + source.getMaxPower().isEmpty() + } + + private loadModel(String transitions, String states) { + return loadModel(transitions, states, true) + } + + private loadModel(String transitions, String states, boolean includeReference) { + def json = modelJson(transitions, states, includeReference) + def root = objectMapper.readTree(json) + factory.get(new MarkovModelData(root)).getOrThrow() + } + + private static String deterministicTransitions() { + return """ + [ + [ + [0.0, 1.0], + [0.0, 1.0] + ] + ] + """.stripIndent() + } + + private static String deterministicStates() { + return """ + [ + { + "weights": [1.0], + "means": [0.4], + "variances": [0.0] + }, + { + "weights": [1.0], + "means": [0.8], + "variances": [0.0] + } + ] + """.stripIndent() + } + + private static String selfLoopTransitions() { + return """ + [ + [ + [1.0, 0.0], + [0.0, 1.0] + ] + ] + """.stripIndent() + } + + private static String emptyTransitions() { + return """ + [ + [ + [0.0, 0.0], + [0.0, 0.0] + ] + ] + """.stripIndent() + } + + private static String missingStateGmms() { + return """ + [ + { + "weights": [1.0], + "means": [0.4], + "variances": [0.0] + }, + null + ] + """.stripIndent() + } + + private static String modelJson(String transitions, String states, boolean includeReference) { + def normalization = includeReference ? + """ + { + "method": "none", + "reference_power": { "value": 5.0, "unit": "kW" } + } + """ : + """ + { + "method": "none" + } + """ + return """ + { + "schema": "markov.load.v1", + "generated_at": "2025-01-01T00:00:00Z", + "generator": { + "name": "simonaMarkovLoad", + "version": "1.0.0", + "config": { "foo": "bar" } + }, + "time_model": { + "bucket_count": 1, + "bucket_encoding": { "formula": "hour_of_day" }, + "sampling_interval_minutes": 60, + "timezone": "UTC" + }, + "value_model": { + "value_unit": "W", + "normalization": $normalization, + "discretization": { + "states": 2, + "thresholds_right": [0.5] + } + }, + "parameters": { + "transitions": { "empty_row_strategy": "fill" }, + "gmm": { + "value_col": "p", + "verbose": 1, + "heartbeat_seconds": 5 + } + }, + "data": { + "transitions": { + "dtype": "float64", + "encoding": "dense", + "shape": [1,2,2], + "values": $transitions + }, + "gmms": { + "buckets": [ + { + "states": $states + } + ] + } + } + } + """.stripIndent() + } +} From cda6f4216ca536a46aa315b0dd63c83f0e9b8715 Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 5 Jan 2026 18:31:26 +0100 Subject: [PATCH 10/39] Addressed Sonar improvements --- .../source/json/JsonMarkovProfileSource.java | 21 +++--- .../source/markov/MarkovLoadValueSource.java | 68 +++++++++---------- 2 files changed, 41 insertions(+), 48 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java index b900328f53..2b7d74ef10 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java @@ -28,6 +28,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicReference; import javax.measure.quantity.Energy; import javax.measure.quantity.Power; import tech.units.indriya.ComparableQuantity; @@ -41,7 +42,7 @@ public class JsonMarkovProfileSource extends EntitySource implements PowerValueS private final ObjectMapper objectMapper = new ObjectMapper(); private final MarkovPowerProfile profile; private MarkovLoadModel cachedModel; - private volatile MarkovLoadValueSource delegate; + private final AtomicReference delegate = new AtomicReference<>(); public JsonMarkovProfileSource( JsonDataSource dataSource, FileLoadProfileMetaInformation metaInformation) { @@ -152,17 +153,15 @@ private static String join(String prefix, String name) { } private MarkovLoadValueSource getDelegate() { - MarkovLoadValueSource current = delegate; - if (current == null) { - synchronized (this) { - current = delegate; - if (current == null) { - current = new MarkovLoadValueSource(profile, loadModelUnchecked()); - delegate = current; - } - } + MarkovLoadValueSource current = delegate.get(); + if (current != null) { + return current; + } + MarkovLoadValueSource created = new MarkovLoadValueSource(profile, loadModelUnchecked()); + if (delegate.compareAndSet(null, created)) { + return created; } - return current; + return delegate.get(); } private MarkovLoadModel loadModelUnchecked() { diff --git a/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java b/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java index 6703d91332..ef4c36beea 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java @@ -6,7 +6,6 @@ package edu.ie3.datamodel.io.source.markov; import edu.ie3.datamodel.io.source.PowerValueSource.MarkovBased; -import edu.ie3.datamodel.io.source.PowerValueSource.MarkovInputValue; import edu.ie3.datamodel.models.StandardUnits; import edu.ie3.datamodel.models.profile.PowerProfile; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; @@ -36,7 +35,6 @@ public class MarkovLoadValueSource implements MarkovBased { private static final int MONTH_FACTOR = QUARTER_HOURS_PER_DAY * 2; private final PowerProfile profile; - private final MarkovLoadModel model; private final ZoneId zoneId; private final double[][][] transitions; private final int bucketCount; @@ -48,23 +46,23 @@ public class MarkovLoadValueSource implements MarkovBased { public MarkovLoadValueSource(PowerProfile profile, MarkovLoadModel model) { this.profile = Objects.requireNonNull(profile, "profile"); - this.model = Objects.requireNonNull(model, "model"); - this.zoneId = ZoneId.of(model.timeModel().timezone()); - this.transitions = model.transitionData().values(); - this.bucketCount = model.timeModel().bucketCount(); - this.stateCount = model.valueModel().discretization().states(); - this.samplingIntervalMinutes = model.timeModel().samplingIntervalMinutes(); + MarkovLoadModel nonNullModel = Objects.requireNonNull(model, "model"); + this.zoneId = ZoneId.of(nonNullModel.timeModel().timezone()); + this.transitions = nonNullModel.transitionData().values(); + this.bucketCount = nonNullModel.timeModel().bucketCount(); + this.stateCount = nonNullModel.valueModel().discretization().states(); + this.samplingIntervalMinutes = nonNullModel.timeModel().samplingIntervalMinutes(); this.discretizationThresholds = - model.valueModel().discretization().thresholdsRight().stream() + nonNullModel.valueModel().discretization().thresholdsRight().stream() .mapToDouble(Double::doubleValue) .toArray(); this.gmmStates = buildGmmStates( - model + nonNullModel .gmmBuckets() .orElseThrow(() -> new IllegalArgumentException("Markov model lacks GMM data."))); this.referencePowerFromModel = - model.valueModel().normalization().referencePower().map(this::convertReferencePower); + nonNullModel.valueModel().normalization().referencePower().map(this::convertReferencePower); } @Override @@ -152,7 +150,7 @@ private int discretize(double normalized) { private StepResult simulateStep(int bucket, int currentState, SplittableRandom rng) { double[] row = transitions[bucket][currentState]; double[] distribution = sanitizeDistribution(bucket, row); - if (distribution == null) { + if (distribution.length == 0) { return new StepResult(currentState, 0d); } int nextState = drawState(distribution, rng); @@ -164,20 +162,18 @@ private double[] sanitizeDistribution(int bucket, double[] row) { double[] sanitized = new double[stateCount]; double sum = 0d; for (int state = 0; state < stateCount; state++) { - double value = state < row.length ? row[state] : 0d; - if (value <= 0d || Double.isNaN(value)) { - sanitized[state] = 0d; - continue; - } - if (gmmStates[bucket][state] == null) { - sanitized[state] = 0d; - continue; + double sanitizedValue = 0d; + if (state < row.length) { + double value = row[state]; + if (value > 0d && !Double.isNaN(value) && gmmStates[bucket][state] != null) { + sanitizedValue = value; + sum += value; + } } - sanitized[state] = value; - sum += value; + sanitized[state] = sanitizedValue; } if (sum <= 0d) { - return null; + return new double[0]; } for (int i = 0; i < sanitized.length; i++) { sanitized[i] /= sum; @@ -218,10 +214,7 @@ private long deriveSeed(MarkovInputValue input, int bucket, int state) { private ComparableQuantity scale( ComparableQuantity referencePower, double normalizedValue) { double clamped = clamp01(normalizedValue); - @SuppressWarnings("unchecked") - ComparableQuantity scaled = - (ComparableQuantity) referencePower.multiply(clamped).asType(Power.class); - return scaled; + return referencePower.multiply(clamped).asType(Power.class); } private ComparableQuantity convertReferencePower( @@ -239,15 +232,6 @@ private static double clamp01(double value) { return value; } - private StepResult calculate(MarkovInputValue input) { - int bucket = bucketId(input.time()); - int currentState = resolveState(input); - SplittableRandom rng = new SplittableRandom(deriveSeed(input, bucket, currentState)); - StepResult step = simulateStep(bucket, currentState, rng); - ComparableQuantity power = scale(input.referencePower(), step.normalizedValue()); - return new StepResult(step.nextState(), step.normalizedValue(), Optional.of(new PValue(power))); - } - private record StepResult(int nextState, double normalizedValue, Optional value) { private StepResult(int nextState, double normalizedValue) { this(nextState, normalizedValue, Optional.empty()); @@ -333,9 +317,19 @@ private void evaluate() { return; } evaluated = true; - StepResult result = calculate(input); + StepResult result = calculate(); this.cached = result.value(); this.nextState = result.nextState(); } + + private StepResult calculate() { + int bucket = bucketId(input.time()); + int currentState = resolveState(input); + SplittableRandom rng = new SplittableRandom(deriveSeed(input, bucket, currentState)); + StepResult step = simulateStep(bucket, currentState, rng); + ComparableQuantity power = scale(input.referencePower(), step.normalizedValue()); + return new StepResult( + step.nextState(), step.normalizedValue(), Optional.of(new PValue(power))); + } } } From 9e9c0175a63e8f9eb342f4ce4c9336a73242b11b Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 5 Jan 2026 18:43:51 +0100 Subject: [PATCH 11/39] some more sonar --- .../source/markov/MarkovLoadValueSource.java | 224 +++++++++--------- 1 file changed, 112 insertions(+), 112 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java b/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java index ef4c36beea..aa0a8b3a96 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java @@ -111,112 +111,6 @@ private GmmStateData[][] buildGmmStates(GmmBuckets buckets) { return lookup; } - private int bucketId(ZonedDateTime time) { - ZonedDateTime zoned = time.withZoneSameInstant(zoneId); - int month = zoned.getMonthValue() - 1; - int isWeekend = isWeekend(zoned) ? 1 : 0; - int quarterHour = zoned.getHour() * 4 + zoned.getMinute() / 15; - return Math.floorMod( - month * MONTH_FACTOR + isWeekend * WEEKEND_FACTOR + quarterHour, bucketCount); - } - - private boolean isWeekend(ZonedDateTime time) { - DayOfWeek day = time.getDayOfWeek(); - return day == DayOfWeek.SATURDAY || day == DayOfWeek.SUNDAY; - } - - private int resolveState(MarkovInputValue input) { - if (input.previousState().isPresent()) { - int state = input.previousState().getAsInt(); - if (state < 0 || state >= stateCount) { - throw new IllegalArgumentException("Previous state out of bounds: " + state); - } - return state; - } - double normalized = input.initialNormalizedValue().orElseThrow(); - return discretize(normalized); - } - - private int discretize(double normalized) { - double value = clamp01(normalized); - for (int i = 0; i < discretizationThresholds.length; i++) { - if (value <= discretizationThresholds[i]) { - return i; - } - } - return discretizationThresholds.length; - } - - private StepResult simulateStep(int bucket, int currentState, SplittableRandom rng) { - double[] row = transitions[bucket][currentState]; - double[] distribution = sanitizeDistribution(bucket, row); - if (distribution.length == 0) { - return new StepResult(currentState, 0d); - } - int nextState = drawState(distribution, rng); - double normalized = sampleNormalizedValue(bucket, nextState, rng); - return new StepResult(nextState, normalized); - } - - private double[] sanitizeDistribution(int bucket, double[] row) { - double[] sanitized = new double[stateCount]; - double sum = 0d; - for (int state = 0; state < stateCount; state++) { - double sanitizedValue = 0d; - if (state < row.length) { - double value = row[state]; - if (value > 0d && !Double.isNaN(value) && gmmStates[bucket][state] != null) { - sanitizedValue = value; - sum += value; - } - } - sanitized[state] = sanitizedValue; - } - if (sum <= 0d) { - return new double[0]; - } - for (int i = 0; i < sanitized.length; i++) { - sanitized[i] /= sum; - } - return sanitized; - } - - private int drawState(double[] distribution, SplittableRandom rng) { - double sample = rng.nextDouble(); - double cumulative = 0d; - for (int i = 0; i < distribution.length; i++) { - cumulative += distribution[i]; - if (sample <= cumulative) { - return i; - } - } - return distribution.length - 1; - } - - private double sampleNormalizedValue(int bucket, int state, SplittableRandom rng) { - GmmStateData gmm = gmmStates[bucket][state]; - if (gmm == null) { - return 0d; - } - return clamp01(gmm.sample(rng)); - } - - private long deriveSeed(MarkovInputValue input, int bucket, int state) { - long seed = input.randomSeed(); - seed = 31 * seed + bucket; - seed = 31 * seed + state; - long slot = - input.time().withZoneSameInstant(zoneId).toInstant().toEpochMilli() - / (samplingIntervalMinutes * 60_000L); - return 31 * seed + slot; - } - - private ComparableQuantity scale( - ComparableQuantity referencePower, double normalizedValue) { - double clamped = clamp01(normalizedValue); - return referencePower.multiply(clamped).asType(Power.class); - } - private ComparableQuantity convertReferencePower( ValueModel.Normalization.PowerReference reference) { if (!"kW".equalsIgnoreCase(reference.unit())) { @@ -226,12 +120,6 @@ private ComparableQuantity convertReferencePower( return Quantities.getQuantity(reference.value(), StandardUnits.ACTIVE_POWER_IN); } - private static double clamp01(double value) { - if (value < 0d) return 0d; - if (value > 1d) return 1d; - return value; - } - private record StepResult(int nextState, double normalizedValue, Optional value) { private StepResult(int nextState, double normalizedValue) { this(nextState, normalizedValue, Optional.empty()); @@ -331,5 +219,117 @@ private StepResult calculate() { return new StepResult( step.nextState(), step.normalizedValue(), Optional.of(new PValue(power))); } + + private int bucketId(ZonedDateTime time) { + ZonedDateTime zoned = time.withZoneSameInstant(zoneId); + int month = zoned.getMonthValue() - 1; + int weekendFlag = isWeekend(zoned) ? 1 : 0; + int quarterHour = zoned.getHour() * 4 + zoned.getMinute() / 15; + return Math.floorMod( + month * MONTH_FACTOR + weekendFlag * WEEKEND_FACTOR + quarterHour, bucketCount); + } + + private boolean isWeekend(ZonedDateTime time) { + DayOfWeek day = time.getDayOfWeek(); + return day == DayOfWeek.SATURDAY || day == DayOfWeek.SUNDAY; + } + + private int resolveState(MarkovInputValue input) { + if (input.previousState().isPresent()) { + int state = input.previousState().getAsInt(); + if (state < 0 || state >= stateCount) { + throw new IllegalArgumentException("Previous state out of bounds: " + state); + } + return state; + } + double normalized = input.initialNormalizedValue().orElseThrow(); + return discretize(normalized); + } + + private int discretize(double normalized) { + double value = clamp01(normalized); + for (int i = 0; i < discretizationThresholds.length; i++) { + if (value <= discretizationThresholds[i]) { + return i; + } + } + return discretizationThresholds.length; + } + + private StepResult simulateStep(int bucket, int currentState, SplittableRandom rng) { + double[] row = transitions[bucket][currentState]; + double[] distribution = sanitizeDistribution(bucket, row); + if (distribution.length == 0) { + return new StepResult(currentState, 0d); + } + int nextState = drawState(distribution, rng); + double normalized = sampleNormalizedValue(bucket, nextState, rng); + return new StepResult(nextState, normalized); + } + + private double[] sanitizeDistribution(int bucket, double[] row) { + double[] sanitized = new double[stateCount]; + double sum = 0d; + for (int state = 0; state < stateCount; state++) { + double sanitizedValue = 0d; + if (state < row.length) { + double value = row[state]; + if (value > 0d && !Double.isNaN(value) && gmmStates[bucket][state] != null) { + sanitizedValue = value; + sum += value; + } + } + sanitized[state] = sanitizedValue; + } + if (sum <= 0d) { + return new double[0]; + } + for (int i = 0; i < sanitized.length; i++) { + sanitized[i] /= sum; + } + return sanitized; + } + + private int drawState(double[] distribution, SplittableRandom rng) { + double sample = rng.nextDouble(); + double cumulative = 0d; + for (int i = 0; i < distribution.length; i++) { + cumulative += distribution[i]; + if (sample <= cumulative) { + return i; + } + } + return distribution.length - 1; + } + + private double sampleNormalizedValue(int bucket, int state, SplittableRandom rng) { + GmmStateData gmm = gmmStates[bucket][state]; + if (gmm == null) { + return 0d; + } + return clamp01(gmm.sample(rng)); + } + + private long deriveSeed(MarkovInputValue input, int bucket, int state) { + long seed = input.randomSeed(); + seed = 31 * seed + bucket; + seed = 31 * seed + state; + long slot = + input.time().withZoneSameInstant(zoneId).toInstant().toEpochMilli() + / (samplingIntervalMinutes * 60_000L); + return 31 * seed + slot; + } + + private ComparableQuantity scale( + ComparableQuantity referencePower, double normalizedValue) { + double clamped = clamp01(normalizedValue); + return referencePower.multiply(clamped).asType(Power.class); + } + + private double clamp01(double value) { + if (value < 0d) return 0d; + if (value > 1d) return 1d; + return value; + } } } From b8bea2939c52d084821759ad8ee4eabfad80a94f Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 5 Jan 2026 18:50:36 +0100 Subject: [PATCH 12/39] sonar.. --- .../datamodel/io/source/markov/MarkovLoadValueSource.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java b/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java index aa0a8b3a96..9eb2ac7eb1 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java @@ -262,9 +262,9 @@ private StepResult simulateStep(int bucket, int currentState, SplittableRandom r if (distribution.length == 0) { return new StepResult(currentState, 0d); } - int nextState = drawState(distribution, rng); - double normalized = sampleNormalizedValue(bucket, nextState, rng); - return new StepResult(nextState, normalized); + int nextStateIndex = drawState(distribution, rng); + double normalized = sampleNormalizedValue(bucket, nextStateIndex, rng); + return new StepResult(nextStateIndex, normalized); } private double[] sanitizeDistribution(int bucket, double[] row) { From 6b9852d11a0478a5efd780327107fcfc4c77c9f7 Mon Sep 17 00:00:00 2001 From: Philipp Date: Wed, 7 Jan 2026 17:44:05 +0100 Subject: [PATCH 13/39] fixed denormalization issue --- .../markov/MarkovModelParsingSupport.java | 2 +- .../source/markov/MarkovLoadValueSource.java | 33 +++++++++-- .../profile/markov/MarkovLoadModel.java | 2 +- .../io/connectors/CsvFileConnectorTest.groovy | 9 ++- .../markov/MarkovLoadModelFactoryTest.groovy | 12 +++- .../json/JsonMarkovProfileSourceTest.groovy | 3 +- .../markov/MarkovLoadValueSourceTest.groovy | 56 ++++++++++--------- 7 files changed, 78 insertions(+), 39 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java index 05c24a7ff7..cadbe7a310 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -61,7 +61,7 @@ default ValueModel parseValueModel(JsonNode valueNode) { ValueModel.Normalization normalization = new ValueModel.Normalization( normalizationMethod, - parsePowerReference(normalizationNode, "reference_power"), + parsePowerReference(normalizationNode, "max_power"), parsePowerReference(normalizationNode, "min_power")); JsonNode discretizationNode = requireNode(valueNode, "discretization"); diff --git a/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java b/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java index 9eb2ac7eb1..42a6aff4ea 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java @@ -42,7 +42,8 @@ public class MarkovLoadValueSource implements MarkovBased { private final int samplingIntervalMinutes; private final double[] discretizationThresholds; private final GmmStateData[][] gmmStates; - private final Optional> referencePowerFromModel; + private final ComparableQuantity maxPowerFromModel; + private final ComparableQuantity minPowerFromModel; public MarkovLoadValueSource(PowerProfile profile, MarkovLoadModel model) { this.profile = Objects.requireNonNull(profile, "profile"); @@ -61,8 +62,22 @@ public MarkovLoadValueSource(PowerProfile profile, MarkovLoadModel model) { nonNullModel .gmmBuckets() .orElseThrow(() -> new IllegalArgumentException("Markov model lacks GMM data."))); - this.referencePowerFromModel = - nonNullModel.valueModel().normalization().referencePower().map(this::convertReferencePower); + this.maxPowerFromModel = + nonNullModel + .valueModel() + .normalization() + .maxPower() + .map(this::convertPowerReference) + .orElseThrow( + () -> new IllegalArgumentException("Markov model lacks normalization.max_power")); + this.minPowerFromModel = + nonNullModel + .valueModel() + .normalization() + .minPower() + .map(this::convertPowerReference) + .orElseThrow( + () -> new IllegalArgumentException("Markov model lacks normalization.min_power")); } @Override @@ -83,7 +98,7 @@ public Optional getNextTimeKey(ZonedDateTime time) { @Override public Optional> getMaxPower() { - return referencePowerFromModel; + return Optional.of(maxPowerFromModel); } @Override @@ -111,7 +126,7 @@ private GmmStateData[][] buildGmmStates(GmmBuckets buckets) { return lookup; } - private ComparableQuantity convertReferencePower( + private ComparableQuantity convertPowerReference( ValueModel.Normalization.PowerReference reference) { if (!"kW".equalsIgnoreCase(reference.unit())) { throw new IllegalArgumentException( @@ -322,8 +337,14 @@ private long deriveSeed(MarkovInputValue input, int bucket, int state) { private ComparableQuantity scale( ComparableQuantity referencePower, double normalizedValue) { + Objects.requireNonNull(referencePower, "referencePower"); + if (!maxPowerFromModel.isGreaterThan(minPowerFromModel)) { + throw new IllegalStateException( + "Markov model normalization has non-positive range (max <= min)."); + } double clamped = clamp01(normalizedValue); - return referencePower.multiply(clamped).asType(Power.class); + ComparableQuantity range = maxPowerFromModel.subtract(minPowerFromModel); + return minPowerFromModel.add(range.multiply(clamped)).asType(Power.class); } private double clamp01(double value) { diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index ae4c2978ca..7e5888c1b5 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -35,7 +35,7 @@ public record ValueModel( String valueUnit, Normalization normalization, Discretization discretization) { public record Normalization( - String method, Optional referencePower, Optional minPower) { + String method, Optional maxPower, Optional minPower) { public record PowerReference(double value, String unit) {} } diff --git a/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy index c5b2735d00..d11d78e98a 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy @@ -15,6 +15,7 @@ import edu.ie3.util.io.FileIOUtils import spock.lang.Shared import spock.lang.Specification +import java.io.Reader import java.nio.file.Files import java.nio.file.Path @@ -50,12 +51,18 @@ class CsvFileConnectorTest extends Specification { } def "The csv file connector initializes a reader without Exception, if the foreseen file is apparent"() { + given: + Reader reader = null + when: def filePath = fileNamingStrategy.getFilePath(NodeInput).orElseThrow() - cfc.initReader(filePath) + reader = cfc.initReader(filePath) then: noExceptionThrown() + + cleanup: + reader?.close() } def "The csv file connector is able to init writers utilizing a directory hierarchy"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy index a4cb7db3e6..3907073759 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy @@ -33,11 +33,16 @@ class MarkovLoadModelFactoryTest extends Specification { gmmState.weights() == [0.6d] gmmState.means() == [1.0d] gmmState.variances() == [0.2d] - model.valueModel().normalization().referencePower().isPresent() - with(model.valueModel().normalization().referencePower().get()) { + model.valueModel().normalization().maxPower().isPresent() + with(model.valueModel().normalization().maxPower().get()) { value() == 1.5d unit() == "kW" } + model.valueModel().normalization().minPower().isPresent() + with(model.valueModel().normalization().minPower().get()) { + value() == 0.1d + unit() == "kW" + } } def "buildModel throws FactoryException on transition dimension mismatch"() { @@ -71,7 +76,8 @@ class MarkovLoadModelFactoryTest extends Specification { "value_unit": "W", "normalization": { "method": "none", - "reference_power": { "value": 1.5, "unit": "kW" } + "max_power": { "value": 1.5, "unit": "kW" }, + "min_power": { "value": 0.1, "unit": "kW" } }, "discretization": { "states": 2, diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy index 681d9590cb..90599b5e29 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy @@ -126,7 +126,8 @@ class JsonMarkovProfileSourceTest extends Specification { "value_unit": "W", "normalization": { "method": "none", - "reference_power": { "value": 10.0, "unit": "kW" } + "max_power": { "value": 10.0, "unit": "kW" }, + "min_power": { "value": 0.5, "unit": "kW" } }, "discretization": { "states": 2, diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy index 6f50acfc81..3950a61896 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy @@ -44,12 +44,34 @@ class MarkovLoadValueSourceTest extends Specification { then: value.isPresent() supplier.getNextState() == 1 - value.get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 4d + value.get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 4.2d supplier.get().is(value) // cached result reused source.getMaxPower().isPresent() source.getMaxPower().get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 5d } + def "supplier denormalizes using model min and max power"() { + given: + def model = loadModel(deterministicTransitions(), deterministicStates()) + def source = new MarkovLoadValueSource(profile, model) + def reference = Quantities.getQuantity(5d, StandardUnits.ACTIVE_POWER_IN) + def input = new PowerValueSource.MarkovInputValue( + ZonedDateTime.parse("2025-01-01T00:00:00Z"), + OptionalInt.of(0), + OptionalDouble.empty(), + reference, + 17L + ) + + when: + def supplier = source.getValueSupplier(input) + def value = supplier.get() + + then: + supplier.getNextState() == 1 + value.get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 4.2d + } + def "supplier uses initial normalized value when no previous state is present"() { given: def model = loadModel(selfLoopTransitions(), deterministicStates()) @@ -68,7 +90,7 @@ class MarkovLoadValueSourceTest extends Specification { then: supplier.getNextState() == 0 // discretized from initial normalized value - supplier.get().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 2d + supplier.get().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 2.6d } def "supplier returns zero power when transitions row has no usable probabilities"() { @@ -91,24 +113,11 @@ class MarkovLoadValueSourceTest extends Specification { then: supplier.getNextState() == 0 // stays in current state value.isPresent() - value.get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 0d - } - - def "reference power metadata optional"() { - given: - def model = loadModel(deterministicTransitions(), deterministicStates(), false) - def source = new MarkovLoadValueSource(profile, model) - - expect: - source.getMaxPower().isEmpty() + value.get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 1d } private loadModel(String transitions, String states) { - return loadModel(transitions, states, true) - } - - private loadModel(String transitions, String states, boolean includeReference) { - def json = modelJson(transitions, states, includeReference) + def json = modelJson(transitions, states) def root = objectMapper.readTree(json) factory.get(new MarkovModelData(root)).getOrThrow() } @@ -176,17 +185,12 @@ class MarkovLoadValueSourceTest extends Specification { """.stripIndent() } - private static String modelJson(String transitions, String states, boolean includeReference) { - def normalization = includeReference ? - """ + private static String modelJson(String transitions, String states) { + def normalization = """ { "method": "none", - "reference_power": { "value": 5.0, "unit": "kW" } - } - """ : - """ - { - "method": "none" + "max_power": { "value": 5.0, "unit": "kW" }, + "min_power": { "value": 1.0, "unit": "kW" } } """ return """ From 2f1a7b4fdfb235a702ed9904eaa1a045fd799f5c Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Thu, 8 Jan 2026 12:05:50 +0100 Subject: [PATCH 14/39] Improving PowerValueSource.getValueSupplier() --- .../markov/MarkovModelParsingSupport.java | 2 +- .../datamodel/io/source/PowerValueSource.java | 61 ++++++++++--------- .../io/source/csv/CsvLoadProfileSource.java | 5 +- .../source/json/JsonMarkovProfileSource.java | 5 +- .../source/markov/MarkovLoadValueSource.java | 18 +++--- .../io/source/sql/SqlLoadProfileSource.java | 5 +- .../profile/markov/MarkovLoadModel.java | 6 ++ .../json/JsonMarkovProfileSourceTest.groovy | 4 +- .../markov/MarkovLoadValueSourceTest.groovy | 10 ++- .../source/sql/SqlLoadProfileSourceIT.groovy | 4 +- 10 files changed, 62 insertions(+), 58 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java index cadbe7a310..e82622cf8e 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -88,7 +88,7 @@ default Parameters parseParameters(JsonNode parametersNode) { JsonNode gmmNode = parametersNode.path("gmm"); Parameters.GmmParameters gmm = - gmmNode.isMissingNode() || gmmNode.isNull() || gmmNode.size() == 0 + gmmNode.isMissingNode() || gmmNode.isNull() || gmmNode.isEmpty() ? null : new Parameters.GmmParameters( gmmNode.path("value_col").asText(""), diff --git a/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java b/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java index ed170dd5c3..80c3cb3002 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java @@ -20,7 +20,9 @@ /** Interface defining base functionality for power value sources. */ public sealed interface PowerValueSource< - P extends PowerProfile, I extends PowerValueSource.PowerValueIdentifier> + P extends PowerProfile, + I extends PowerValueSource.PowerValueIdentifier, + O extends PowerValueSource.PowerOutputValue> permits PowerValueSource.MarkovBased, PowerValueSource.TimeSeriesBased { /** Returns the profile of this source. */ @@ -34,7 +36,7 @@ public sealed interface PowerValueSource< * @param data input data that is used to calculate the next power value. * @return A supplier for an option on the value at the given time step. */ - Supplier> getValueSupplier(I data); + Supplier getValueSupplier(I data); /** * Method to determine the next timestamp for which data is present. @@ -55,22 +57,11 @@ public sealed interface PowerValueSource< /** Interface for time-series-based power value sources. */ non-sealed interface TimeSeriesBased - extends PowerValueSource {} + extends PowerValueSource {} /** Interface for markov-chain-based power value sources. */ non-sealed interface MarkovBased - extends PowerValueSource { - - @Override - MarkovValueSupplier getValueSupplier(MarkovInputValue data); - - interface MarkovValueSupplier extends Supplier> { - /** - * Returns the next state that should be provided as {@link MarkovInputValue#previousState()}. - */ - int getNextState(); - } - } + extends PowerValueSource {} // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= // input data @@ -79,10 +70,9 @@ interface MarkovValueSupplier extends Supplier> { * Interface for the input data of {@link #getValueSupplier(PowerValueIdentifier)}. The data is * used to determine the next power. */ - sealed interface PowerValueIdentifier - permits PowerValueSource.TimeSeriesInputValue, PowerValueSource.MarkovInputValue { + sealed interface PowerValueIdentifier permits TimeSeriesIdentifier, MarkovIdentifier { /** Returns the timestamp for which a power value is needed. */ - ZonedDateTime getTime(); + ZonedDateTime time(); } /** @@ -90,18 +80,13 @@ sealed interface PowerValueIdentifier * * @param time */ - record TimeSeriesInputValue(ZonedDateTime time) implements PowerValueIdentifier { - @Override - public ZonedDateTime getTime() { - return time; - } - } + record TimeSeriesIdentifier(ZonedDateTime time) implements PowerValueIdentifier {} /** * Input data for Markov-based power value sources, containing everything needed for a single * simonaMarkovLoad step. */ - record MarkovInputValue( + record MarkovIdentifier( ZonedDateTime time, OptionalInt previousState, OptionalDouble initialNormalizedValue, @@ -109,7 +94,7 @@ record MarkovInputValue( long randomSeed) implements PowerValueIdentifier { - public MarkovInputValue { + public MarkovIdentifier { Objects.requireNonNull(time, "time"); Objects.requireNonNull(previousState, "previousState"); Objects.requireNonNull(initialNormalizedValue, "initialNormalizedValue"); @@ -119,10 +104,28 @@ record MarkovInputValue( "Need either previous state or an initial normalized value to start the Markov chain."); } } + } - @Override - public ZonedDateTime getTime() { - return time; + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= + // output data + + /** Interface for the output data of {@link #getValueSupplier(PowerValueIdentifier)}. */ + sealed interface PowerOutputValue + permits PowerValueSource.TimeSeriesOutputValue, PowerValueSource.MarkovOutputValue { + Optional value(); + } + + /** + * Interface for time-series-based power values. + * + * @param value + */ + record TimeSeriesOutputValue(Optional value) implements PowerOutputValue { + public static Supplier from(Supplier> supplier) { + return () -> new TimeSeriesOutputValue(supplier.get()); } } + + /** Input data for Markov-based power values. */ + record MarkovOutputValue(Optional value, int nextState) implements PowerOutputValue {} } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvLoadProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvLoadProfileSource.java index 544b0e3187..52b72dfefd 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvLoadProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvLoadProfileSource.java @@ -14,7 +14,6 @@ import edu.ie3.datamodel.models.profile.LoadProfile; import edu.ie3.datamodel.models.timeseries.repetitive.LoadProfileEntry; import edu.ie3.datamodel.models.timeseries.repetitive.LoadProfileTimeSeries; -import edu.ie3.datamodel.models.value.PValue; import edu.ie3.datamodel.models.value.load.LoadValues; import edu.ie3.datamodel.utils.Try; import java.nio.file.Path; @@ -71,8 +70,8 @@ public Set> getEntries() { } @Override - public Supplier> getValueSupplier(TimeSeriesInputValue data) { - return loadProfileTimeSeries.supplyValue(data.time()); + public Supplier getValueSupplier(TimeSeriesIdentifier data) { + return TimeSeriesOutputValue.from(loadProfileTimeSeries.supplyValue(data.time())); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java index 2b7d74ef10..1090f11179 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java @@ -29,6 +29,7 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import javax.measure.quantity.Energy; import javax.measure.quantity.Power; import tech.units.indriya.ComparableQuantity; @@ -99,8 +100,8 @@ public MarkovPowerProfile getProfile() { } @Override - public MarkovValueSupplier getValueSupplier(PowerValueSource.MarkovInputValue data) { - return getDelegate().getValueSupplier(data); + public Supplier getValueSupplier(MarkovIdentifier data) { + return () -> cachedModel.getPower(data); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java b/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java index 42a6aff4ea..1f2dbe8d48 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java @@ -19,6 +19,7 @@ import java.util.Objects; import java.util.Optional; import java.util.SplittableRandom; +import java.util.function.Supplier; import javax.measure.quantity.Energy; import javax.measure.quantity.Power; import tech.units.indriya.ComparableQuantity; @@ -86,9 +87,10 @@ public PowerProfile getProfile() { } @Override - public MarkovValueSupplier getValueSupplier(MarkovInputValue data) { + public Supplier getValueSupplier(MarkovIdentifier data) { Objects.requireNonNull(data, "data"); - return new MarkovSupplier(data); + MarkovSupplier s = new MarkovSupplier(data); + return () -> new MarkovOutputValue(s.get(), s.nextState); } @Override @@ -193,23 +195,21 @@ private double nextGaussian(SplittableRandom rng) { } } - private final class MarkovSupplier implements MarkovValueSupplier { - private final MarkovInputValue input; + private final class MarkovSupplier { + private final MarkovIdentifier input; private Optional cached = Optional.empty(); private Integer nextState; private boolean evaluated; - private MarkovSupplier(MarkovInputValue input) { + private MarkovSupplier(MarkovIdentifier input) { this.input = input; } - @Override public Optional get() { evaluate(); return cached; } - @Override public int getNextState() { evaluate(); return nextState; @@ -249,7 +249,7 @@ private boolean isWeekend(ZonedDateTime time) { return day == DayOfWeek.SATURDAY || day == DayOfWeek.SUNDAY; } - private int resolveState(MarkovInputValue input) { + private int resolveState(MarkovIdentifier input) { if (input.previousState().isPresent()) { int state = input.previousState().getAsInt(); if (state < 0 || state >= stateCount) { @@ -325,7 +325,7 @@ private double sampleNormalizedValue(int bucket, int state, SplittableRandom rng return clamp01(gmm.sample(rng)); } - private long deriveSeed(MarkovInputValue input, int bucket, int state) { + private long deriveSeed(MarkovIdentifier input, int bucket, int state) { long seed = input.randomSeed(); seed = 31 * seed + bucket; seed = 31 * seed + state; diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSource.java index 2af0e70356..16f1c4c865 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSource.java @@ -16,7 +16,6 @@ import edu.ie3.datamodel.models.profile.LoadProfile; import edu.ie3.datamodel.models.timeseries.repetitive.LoadProfileEntry; import edu.ie3.datamodel.models.timeseries.repetitive.LoadProfileTimeSeries; -import edu.ie3.datamodel.models.value.PValue; import edu.ie3.datamodel.models.value.Value; import edu.ie3.datamodel.models.value.load.LoadValues; import edu.ie3.datamodel.utils.TimeSeriesUtils; @@ -100,10 +99,10 @@ public Set> getEntries() { } @Override - public Supplier> getValueSupplier(TimeSeriesInputValue data) { + public Supplier getValueSupplier(TimeSeriesIdentifier data) { ZonedDateTime time = data.time(); Optional> loadValueOption = queryForValue(time); - return () -> loadValueOption.map(v -> v.getValue(time, profile)); + return TimeSeriesOutputValue.from(() -> loadValueOption.map(v -> v.getValue(time, profile))); } @Override diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index 7e5888c1b5..ce56ae2f55 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.models.profile.markov; +import edu.ie3.datamodel.io.source.PowerValueSource; import java.time.ZonedDateTime; import java.util.Arrays; import java.util.List; @@ -23,6 +24,11 @@ public record MarkovLoadModel( TransitionData transitionData, Optional gmmBuckets) { + public PowerValueSource.MarkovOutputValue getPower(PowerValueSource.MarkovIdentifier data) { + // TODO: Implement + return null; + } + public record Generator(String name, String version, Map config) {} public record TimeModel( diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy index 90599b5e29..a042eaadfd 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy @@ -18,8 +18,6 @@ import tech.units.indriya.quantity.Quantities import java.nio.file.Files import java.nio.file.Path import java.time.ZonedDateTime -import java.util.OptionalDouble -import java.util.OptionalInt class JsonMarkovProfileSourceTest extends Specification { @@ -86,7 +84,7 @@ class JsonMarkovProfileSourceTest extends Specification { new FileLoadProfileMetaInformation("profile1", jsonFile, FileType.JSON) ) def referencePower = Quantities.getQuantity(10d, StandardUnits.ACTIVE_POWER_IN) - def input = new PowerValueSource.MarkovInputValue( + def input = new PowerValueSource.MarkovIdentifier( ZonedDateTime.parse("2025-01-01T00:00:00Z"), OptionalInt.of(0), OptionalDouble.empty(), diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy index 3950a61896..f842dfca8d 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy @@ -15,8 +15,6 @@ import spock.lang.Specification import tech.units.indriya.quantity.Quantities import java.time.ZonedDateTime -import java.util.OptionalDouble -import java.util.OptionalInt class MarkovLoadValueSourceTest extends Specification { @@ -29,7 +27,7 @@ class MarkovLoadValueSourceTest extends Specification { def model = loadModel(deterministicTransitions(), deterministicStates()) def source = new MarkovLoadValueSource(profile, model) def reference = Quantities.getQuantity(5d, StandardUnits.ACTIVE_POWER_IN) - def input = new PowerValueSource.MarkovInputValue( + def input = new PowerValueSource.MarkovIdentifier( ZonedDateTime.parse("2025-01-01T00:00:00Z"), OptionalInt.of(0), OptionalDouble.empty(), @@ -55,7 +53,7 @@ class MarkovLoadValueSourceTest extends Specification { def model = loadModel(deterministicTransitions(), deterministicStates()) def source = new MarkovLoadValueSource(profile, model) def reference = Quantities.getQuantity(5d, StandardUnits.ACTIVE_POWER_IN) - def input = new PowerValueSource.MarkovInputValue( + def input = new PowerValueSource.MarkovIdentifier( ZonedDateTime.parse("2025-01-01T00:00:00Z"), OptionalInt.of(0), OptionalDouble.empty(), @@ -77,7 +75,7 @@ class MarkovLoadValueSourceTest extends Specification { def model = loadModel(selfLoopTransitions(), deterministicStates()) def source = new MarkovLoadValueSource(profile, model) def reference = Quantities.getQuantity(5d, StandardUnits.ACTIVE_POWER_IN) - def input = new PowerValueSource.MarkovInputValue( + def input = new PowerValueSource.MarkovIdentifier( ZonedDateTime.parse("2025-01-01T00:00:00Z"), OptionalInt.empty(), OptionalDouble.of(0.25d), @@ -98,7 +96,7 @@ class MarkovLoadValueSourceTest extends Specification { def model = loadModel(emptyTransitions(), missingStateGmms()) def source = new MarkovLoadValueSource(profile, model) def reference = Quantities.getQuantity(3d, StandardUnits.ACTIVE_POWER_IN) - def input = new PowerValueSource.MarkovInputValue( + def input = new PowerValueSource.MarkovIdentifier( ZonedDateTime.parse("2025-01-01T00:00:00Z"), OptionalInt.of(0), OptionalDouble.empty(), diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSourceIT.groovy index 4e486db9bc..f563a70f09 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSourceIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSourceIT.groovy @@ -63,8 +63,8 @@ class SqlLoadProfileSourceIT extends Specification implements TestContainerHelpe def "A SqlTimeSeriesSource can read and correctly parse a single value for a specific date"() { when: - def supplier = loadSource.getValueSupplier(new PowerValueSource.TimeSeriesInputValue(TIME_00MIN)) - def value = supplier.get() + def supplier = loadSource.getValueSupplier(new PowerValueSource.TimeSeriesIdentifier(TIME_00MIN)) + def value = supplier.get().value() then: value.present From 90dd28f0564c85a460a1eb2836cc852f244dadf9 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Thu, 8 Jan 2026 12:23:35 +0100 Subject: [PATCH 15/39] Updating to jackson-databind 3.0.3 --- build.gradle | 2 +- .../markov/MarkovLoadModelFactory.java | 2 +- .../io/factory/markov/MarkovModelData.java | 2 +- .../markov/MarkovModelParsingSupport.java | 35 +++++++------------ .../source/json/JsonMarkovProfileSource.java | 7 ++-- .../markov/MarkovLoadModelFactoryTest.groovy | 2 +- .../markov/MarkovLoadValueSourceTest.groovy | 2 +- 7 files changed, 20 insertions(+), 32 deletions(-) diff --git a/build.gradle b/build.gradle index d502a713e5..b43e15bb13 100644 --- a/build.gradle +++ b/build.gradle @@ -106,7 +106,7 @@ dependencies { implementation 'commons-codec:commons-codec:1.20.0' // needed by commons-compress implementation 'org.apache.commons:commons-compress:1.28.0' // I/O functionalities - implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.2' + implementation 'tools.jackson.core:jackson-databind:3.0.3' } tasks.withType(JavaCompile) { diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java index 7249fd3e2f..7dc8f69a3f 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -5,7 +5,7 @@ */ package edu.ie3.datamodel.io.factory.markov; -import com.fasterxml.jackson.databind.JsonNode; +import tools.jackson.databind.JsonNode; import edu.ie3.datamodel.io.factory.Factory; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.*; diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java index ca735ac2af..4296ffbd22 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java @@ -5,7 +5,7 @@ */ package edu.ie3.datamodel.io.factory.markov; -import com.fasterxml.jackson.databind.JsonNode; +import tools.jackson.databind.JsonNode; import edu.ie3.datamodel.io.factory.FactoryData; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; import java.util.Collections; diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java index e82622cf8e..3ec8d456fa 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -5,22 +5,13 @@ */ package edu.ie3.datamodel.io.factory.markov; -import com.fasterxml.jackson.databind.JsonNode; import edu.ie3.datamodel.exceptions.FactoryException; -import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.Generator; -import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.GmmBuckets; -import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.Parameters; -import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.TimeModel; -import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.TransitionData; -import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.ValueModel; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.*; +import tools.jackson.databind.JsonNode; + import java.time.ZonedDateTime; import java.time.format.DateTimeParseException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; +import java.util.*; /** * Shared parsing helpers for Markov model JSON documents. This is intentionally package-private as @@ -34,18 +25,16 @@ default Generator parseGenerator(JsonNode generatorNode) { Map config = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); JsonNode configNode = generatorNode.path("config"); if (configNode.isObject()) { - Iterator> fields = configNode.fields(); - while (fields.hasNext()) { - Map.Entry entry = fields.next(); - config.put(entry.getKey(), entry.getValue().asText()); - } + for (Map.Entry entry : configNode.properties()) { + config.put(entry.getKey(), entry.getValue().asString()); + } } return new Generator(name, version, config); } default TimeModel parseTimeModel(JsonNode timeNode) { int bucketCount = requireInt(timeNode, "bucket_count"); - String formula = requireNode(timeNode, "bucket_encoding").path("formula").asText(""); + String formula = requireNode(timeNode, "bucket_encoding").path("formula").asString(""); if (formula.isEmpty()) { throw new FactoryException("Missing bucket encoding formula"); } @@ -81,7 +70,7 @@ default ValueModel parseValueModel(JsonNode valueNode) { default Parameters parseParameters(JsonNode parametersNode) { Parameters.TransitionParameters transitions = new Parameters.TransitionParameters( - parametersNode.path("transitions").path("empty_row_strategy").asText("")); + parametersNode.path("transitions").path("empty_row_strategy").asString("")); if (transitions.emptyRowStrategy().isEmpty()) { transitions = null; } @@ -91,7 +80,7 @@ default Parameters parseParameters(JsonNode parametersNode) { gmmNode.isMissingNode() || gmmNode.isNull() || gmmNode.isEmpty() ? null : new Parameters.GmmParameters( - gmmNode.path("value_col").asText(""), + gmmNode.path("value_col").asString(""), optionalInt(gmmNode, "verbose"), optionalInt(gmmNode, "heartbeat_seconds")); @@ -159,10 +148,10 @@ default String requireText(JsonNode node, String field) { if (value == null || value.isMissingNode() || value.isNull()) { throw new FactoryException("Missing field '" + field + "'"); } - if (!value.isTextual()) { + if (!value.isString()) { throw new FactoryException("Field '" + field + "' must be textual"); } - return value.asText(); + return value.asString(); } default double requireDouble(JsonNode node, String field) { diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java index 1090f11179..e5912e773c 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java @@ -5,8 +5,8 @@ */ package edu.ie3.datamodel.io.source.json; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.FailedValidationException; import edu.ie3.datamodel.exceptions.SourceException; @@ -142,8 +142,7 @@ private static void collectFields(String prefix, JsonNode node, Set coll return; } if (node.isObject()) { - node.fieldNames() - .forEachRemaining(name -> collectFields(join(prefix, name), node.get(name), collector)); + node.propertyNames().forEach(name -> collectFields(join(prefix, name), node.get(name), collector)); } else if (!prefix.isEmpty()) { collector.add(prefix); } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy index 3907073759..b19e237204 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy @@ -5,7 +5,7 @@ */ package edu.ie3.datamodel.io.factory.markov -import com.fasterxml.jackson.databind.ObjectMapper +import tools.jackson.databind.ObjectMapper import edu.ie3.datamodel.exceptions.FactoryException import spock.lang.Specification diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy index f842dfca8d..31b109e957 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.source.markov -import com.fasterxml.jackson.databind.ObjectMapper import edu.ie3.datamodel.io.factory.markov.MarkovLoadModelFactory import edu.ie3.datamodel.io.factory.markov.MarkovModelData import edu.ie3.datamodel.io.source.PowerValueSource @@ -13,6 +12,7 @@ import edu.ie3.datamodel.models.StandardUnits import edu.ie3.datamodel.models.profile.markov.MarkovPowerProfile import spock.lang.Specification import tech.units.indriya.quantity.Quantities +import tools.jackson.databind.ObjectMapper import java.time.ZonedDateTime From 67a806c3f65ab4cb52700405b248d7fc8d448814 Mon Sep 17 00:00:00 2001 From: Philipp Date: Tue, 10 Feb 2026 07:18:44 +0100 Subject: [PATCH 16/39] refactored to new supplier wrapper approach --- .../io/connectors/JsonFileConnector.java | 14 - .../markov/MarkovLoadModelFactory.java | 14 +- .../io/factory/markov/MarkovModelData.java | 2 +- .../markov/MarkovModelParsingSupport.java | 18 +- .../datamodel/io/source/PowerValueSource.java | 6 +- .../source/json/JsonMarkovProfileSource.java | 40 +- .../source/markov/MarkovLoadValueSource.java | 356 -------------- .../profile/markov/MarkovLoadModel.java | 459 +++++++++++++++++- .../connectors/JsonFileConnectorTest.groovy | 15 - .../markov/MarkovLoadModelFactoryTest.groovy | 2 +- .../json/JsonMarkovProfileSourceTest.groovy | 8 +- .../markov/MarkovLoadModelTest.groovy} | 53 +- 12 files changed, 524 insertions(+), 463 deletions(-) delete mode 100644 src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java rename src/test/groovy/edu/ie3/datamodel/{io/source/markov/MarkovLoadValueSourceTest.groovy => models/profile/markov/MarkovLoadModelTest.groovy} (77%) diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java index 40764d5511..1eaea3188b 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java @@ -5,11 +5,8 @@ */ package edu.ie3.datamodel.io.connectors; -import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.function.Function; @@ -25,17 +22,6 @@ public JsonFileConnector(Path baseDirectory, Function custo super(baseDirectory, customInputStream); } - /** - * Opens a buffered reader for the given JSON file, using UTF-8 decoding. - * - * @param filePath relative path without ending - * @return buffered reader referencing the JSON file - */ - public BufferedReader initReader(Path filePath) throws FileNotFoundException { - InputStream inputStream = openInputStream(filePath); - return new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8), 16384); - } - /** * Opens an input stream for the given JSON file. * diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java index 7dc8f69a3f..79b5cecc21 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.factory.markov; -import tools.jackson.databind.JsonNode; import edu.ie3.datamodel.io.factory.Factory; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.*; @@ -13,8 +12,14 @@ import java.util.List; import java.util.Optional; import java.util.Set; +import tools.jackson.databind.JsonNode; -/** Factory turning Markov JSON data into {@link MarkovLoadModel}s. */ +/** + * Factory turning Markov JSON data into {@link MarkovLoadModel}s. + * + *

The JSON fields follow the simonaMarkovLoad schema (snake_case), which is mapped to the model + * records used within PSDM. + */ public class MarkovLoadModelFactory extends Factory implements MarkovModelParsingSupport { @@ -23,6 +28,11 @@ public MarkovLoadModelFactory() { super(MarkovLoadModel.class); } + /** + * Build a {@link MarkovLoadModel} from a parsed JSON tree. + * + *

This method validates the transition shape and requires GMM buckets to be present. + */ @Override protected MarkovLoadModel buildModel(MarkovModelData data) { JsonNode root = data.getRoot(); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java index 4296ffbd22..de649ead19 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java @@ -5,11 +5,11 @@ */ package edu.ie3.datamodel.io.factory.markov; -import tools.jackson.databind.JsonNode; import edu.ie3.datamodel.io.factory.FactoryData; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; import java.util.Collections; import java.util.Objects; +import tools.jackson.databind.JsonNode; /** Factory data wrapper around a parsed Markov-load JSON tree. */ public class MarkovModelData extends FactoryData { diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java index 3ec8d456fa..2cb038d62c 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -7,11 +7,10 @@ import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.*; -import tools.jackson.databind.JsonNode; - import java.time.ZonedDateTime; import java.time.format.DateTimeParseException; import java.util.*; +import tools.jackson.databind.JsonNode; /** * Shared parsing helpers for Markov model JSON documents. This is intentionally package-private as @@ -25,13 +24,14 @@ default Generator parseGenerator(JsonNode generatorNode) { Map config = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); JsonNode configNode = generatorNode.path("config"); if (configNode.isObject()) { - for (Map.Entry entry : configNode.properties()) { - config.put(entry.getKey(), entry.getValue().asString()); - } + for (Map.Entry entry : configNode.properties()) { + config.put(entry.getKey(), entry.getValue().asString()); + } } return new Generator(name, version, config); } + /** Parses the time model block, including bucket count and sampling interval. */ default TimeModel parseTimeModel(JsonNode timeNode) { int bucketCount = requireInt(timeNode, "bucket_count"); String formula = requireNode(timeNode, "bucket_encoding").path("formula").asString(""); @@ -43,6 +43,7 @@ default TimeModel parseTimeModel(JsonNode timeNode) { return new TimeModel(bucketCount, formula, samplingInterval, timezone); } + /** Parses value model settings (unit, normalization, discretization thresholds). */ default ValueModel parseValueModel(JsonNode valueNode) { String valueUnit = requireText(valueNode, "value_unit"); JsonNode normalizationNode = requireNode(valueNode, "normalization"); @@ -67,6 +68,7 @@ default ValueModel parseValueModel(JsonNode valueNode) { return new ValueModel(valueUnit, normalization, discretization); } + /** Parses optional parameter blocks (transitions and GMM). */ default Parameters parseParameters(JsonNode parametersNode) { Parameters.TransitionParameters transitions = new Parameters.TransitionParameters( @@ -87,6 +89,11 @@ default Parameters parseParameters(JsonNode parametersNode) { return new Parameters(transitions, gmm); } + /** + * Parses the transition matrix section. + * + *

The expected shape is [bucket, state, state]. + */ default TransitionData parseTransitions( JsonNode dataNode, int expectedBucketCount, int stateCount) { JsonNode transitionsNode = requireNode(dataNode, "transitions"); @@ -105,6 +112,7 @@ default TransitionData parseTransitions( return new TransitionData(dtype, encoding, values); } + /** Parses GMM buckets. Individual states may be null, which disables sampling for that state. */ default GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { if (gmmsNode == null || gmmsNode.isMissingNode() || gmmsNode.isNull()) { throw new FactoryException("Missing field 'gmms'"); diff --git a/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java b/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java index 80c3cb3002..eb9215c620 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java @@ -85,6 +85,10 @@ record TimeSeriesIdentifier(ZonedDateTime time) implements PowerValueIdentifier /** * Input data for Markov-based power value sources, containing everything needed for a single * simonaMarkovLoad step. + * + *

Provide either {@code previousState} (typical for subsequent steps) or an {@code + * initialNormalizedValue} (for the first step). The {@code referencePower} is used to scale + * normalized values to real power, while {@code randomSeed} enables reproducible sampling. */ record MarkovIdentifier( ZonedDateTime time, @@ -126,6 +130,6 @@ public static Supplier from(Supplier> su } } - /** Input data for Markov-based power values. */ + /** Output data for Markov-based power values, including the next state. */ record MarkovOutputValue(Optional value, int nextState) implements PowerOutputValue {} } diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java index e5912e773c..ea1992fb5a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java @@ -5,8 +5,6 @@ */ package edu.ie3.datamodel.io.source.json; -import tools.jackson.databind.JsonNode; -import tools.jackson.databind.ObjectMapper; import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.FailedValidationException; import edu.ie3.datamodel.exceptions.SourceException; @@ -17,7 +15,6 @@ import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation; import edu.ie3.datamodel.io.source.EntitySource; import edu.ie3.datamodel.io.source.PowerValueSource; -import edu.ie3.datamodel.io.source.markov.MarkovLoadValueSource; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; import edu.ie3.datamodel.models.profile.markov.MarkovPowerProfile; import java.io.IOException; @@ -28,13 +25,19 @@ import java.util.Optional; import java.util.Set; import java.util.TreeSet; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import javax.measure.quantity.Energy; import javax.measure.quantity.Power; import tech.units.indriya.ComparableQuantity; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; -/** Source that reads Markov-based load models from JSON files. */ +/** + * Source that reads Markov-based load models from JSON files. + * + *

The JSON file is parsed lazily and cached. All power value requests are delegated to the + * parsed {@link MarkovLoadModel}. + */ public class JsonMarkovProfileSource extends EntitySource implements PowerValueSource.MarkovBased { private final JsonDataSource dataSource; @@ -43,7 +46,6 @@ public class JsonMarkovProfileSource extends EntitySource implements PowerValueS private final ObjectMapper objectMapper = new ObjectMapper(); private final MarkovPowerProfile profile; private MarkovLoadModel cachedModel; - private final AtomicReference delegate = new AtomicReference<>(); public JsonMarkovProfileSource( JsonDataSource dataSource, FileLoadProfileMetaInformation metaInformation) { @@ -99,24 +101,25 @@ public MarkovPowerProfile getProfile() { return profile; } + /** Delegates to the cached {@link MarkovLoadModel} for a single simulation step. */ @Override public Supplier getValueSupplier(MarkovIdentifier data) { - return () -> cachedModel.getPower(data); + return getModelUnchecked().getValueSupplier(data); } @Override public Optional getNextTimeKey(ZonedDateTime time) { - return getDelegate().getNextTimeKey(time); + return getModelUnchecked().getNextTimeKey(time); } @Override public Optional> getMaxPower() { - return getDelegate().getMaxPower(); + return getModelUnchecked().getMaxPower(); } @Override public Optional> getProfileEnergyScaling() { - return getDelegate().getProfileEnergyScaling(); + return getModelUnchecked().getProfileEnergyScaling(); } private JsonNode readModelTree() throws SourceException { @@ -142,7 +145,8 @@ private static void collectFields(String prefix, JsonNode node, Set coll return; } if (node.isObject()) { - node.propertyNames().forEach(name -> collectFields(join(prefix, name), node.get(name), collector)); + node.propertyNames() + .forEach(name -> collectFields(join(prefix, name), node.get(name), collector)); } else if (!prefix.isEmpty()) { collector.add(prefix); } @@ -152,19 +156,7 @@ private static String join(String prefix, String name) { return prefix.isEmpty() ? name : prefix + "." + name; } - private MarkovLoadValueSource getDelegate() { - MarkovLoadValueSource current = delegate.get(); - if (current != null) { - return current; - } - MarkovLoadValueSource created = new MarkovLoadValueSource(profile, loadModelUnchecked()); - if (delegate.compareAndSet(null, created)) { - return created; - } - return delegate.get(); - } - - private MarkovLoadModel loadModelUnchecked() { + private MarkovLoadModel getModelUnchecked() { try { return getModel(); } catch (SourceException e) { diff --git a/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java b/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java deleted file mode 100644 index 1f2dbe8d48..0000000000 --- a/src/main/java/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSource.java +++ /dev/null @@ -1,356 +0,0 @@ -/* - * © 2025. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation -*/ -package edu.ie3.datamodel.io.source.markov; - -import edu.ie3.datamodel.io.source.PowerValueSource.MarkovBased; -import edu.ie3.datamodel.models.StandardUnits; -import edu.ie3.datamodel.models.profile.PowerProfile; -import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; -import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.GmmBuckets; -import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.ValueModel; -import edu.ie3.datamodel.models.value.PValue; -import java.time.DayOfWeek; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.SplittableRandom; -import java.util.function.Supplier; -import javax.measure.quantity.Energy; -import javax.measure.quantity.Power; -import tech.units.indriya.ComparableQuantity; -import tech.units.indriya.quantity.Quantities; - -/** - * Implementation of a {@link MarkovBased} power value source that converts a {@link - * MarkovLoadModel} export into {@link PValue}s. - */ -public class MarkovLoadValueSource implements MarkovBased { - - private static final int QUARTER_HOURS_PER_DAY = 96; - private static final int WEEKEND_FACTOR = QUARTER_HOURS_PER_DAY; - private static final int MONTH_FACTOR = QUARTER_HOURS_PER_DAY * 2; - - private final PowerProfile profile; - private final ZoneId zoneId; - private final double[][][] transitions; - private final int bucketCount; - private final int stateCount; - private final int samplingIntervalMinutes; - private final double[] discretizationThresholds; - private final GmmStateData[][] gmmStates; - private final ComparableQuantity maxPowerFromModel; - private final ComparableQuantity minPowerFromModel; - - public MarkovLoadValueSource(PowerProfile profile, MarkovLoadModel model) { - this.profile = Objects.requireNonNull(profile, "profile"); - MarkovLoadModel nonNullModel = Objects.requireNonNull(model, "model"); - this.zoneId = ZoneId.of(nonNullModel.timeModel().timezone()); - this.transitions = nonNullModel.transitionData().values(); - this.bucketCount = nonNullModel.timeModel().bucketCount(); - this.stateCount = nonNullModel.valueModel().discretization().states(); - this.samplingIntervalMinutes = nonNullModel.timeModel().samplingIntervalMinutes(); - this.discretizationThresholds = - nonNullModel.valueModel().discretization().thresholdsRight().stream() - .mapToDouble(Double::doubleValue) - .toArray(); - this.gmmStates = - buildGmmStates( - nonNullModel - .gmmBuckets() - .orElseThrow(() -> new IllegalArgumentException("Markov model lacks GMM data."))); - this.maxPowerFromModel = - nonNullModel - .valueModel() - .normalization() - .maxPower() - .map(this::convertPowerReference) - .orElseThrow( - () -> new IllegalArgumentException("Markov model lacks normalization.max_power")); - this.minPowerFromModel = - nonNullModel - .valueModel() - .normalization() - .minPower() - .map(this::convertPowerReference) - .orElseThrow( - () -> new IllegalArgumentException("Markov model lacks normalization.min_power")); - } - - @Override - public PowerProfile getProfile() { - return profile; - } - - @Override - public Supplier getValueSupplier(MarkovIdentifier data) { - Objects.requireNonNull(data, "data"); - MarkovSupplier s = new MarkovSupplier(data); - return () -> new MarkovOutputValue(s.get(), s.nextState); - } - - @Override - public Optional getNextTimeKey(ZonedDateTime time) { - return Optional.ofNullable(time).map(t -> t.plusMinutes(samplingIntervalMinutes)); - } - - @Override - public Optional> getMaxPower() { - return Optional.of(maxPowerFromModel); - } - - @Override - public Optional> getProfileEnergyScaling() { - return Optional.empty(); - } - - private GmmStateData[][] buildGmmStates(GmmBuckets buckets) { - List bucketList = buckets.buckets(); - if (bucketList.size() != bucketCount) { - throw new IllegalArgumentException( - "GMM bucket count mismatch. Expected " + bucketCount + " but was " + bucketList.size()); - } - GmmStateData[][] lookup = new GmmStateData[bucketCount][stateCount]; - for (int bucket = 0; bucket < bucketCount; bucket++) { - List> states = bucketList.get(bucket).states(); - if (states.size() != stateCount) { - throw new IllegalArgumentException( - "State count mismatch in bucket " + bucket + ". Expected " + stateCount); - } - for (int state = 0; state < stateCount; state++) { - lookup[bucket][state] = states.get(state).map(GmmStateData::from).orElse(null); - } - } - return lookup; - } - - private ComparableQuantity convertPowerReference( - ValueModel.Normalization.PowerReference reference) { - if (!"kW".equalsIgnoreCase(reference.unit())) { - throw new IllegalArgumentException( - "Unsupported reference power unit '" + reference.unit() + "'. Only kW is supported."); - } - return Quantities.getQuantity(reference.value(), StandardUnits.ACTIVE_POWER_IN); - } - - private record StepResult(int nextState, double normalizedValue, Optional value) { - private StepResult(int nextState, double normalizedValue) { - this(nextState, normalizedValue, Optional.empty()); - } - } - - private static final class GmmStateData { - private final double[] weights; - private final double[] means; - private final double[] variances; - - private GmmStateData(double[] weights, double[] means, double[] variances) { - this.weights = weights; - this.means = means; - this.variances = variances; - } - - private static GmmStateData from(GmmBuckets.GmmState state) { - return new GmmStateData( - toArray(state.weights()), toArray(state.means()), toArray(state.variances())); - } - - private static double[] toArray(List values) { - double[] array = new double[values.size()]; - for (int i = 0; i < values.size(); i++) { - array[i] = values.get(i); - } - return array; - } - - private double sample(SplittableRandom rng) { - int component = drawComponent(rng.nextDouble()); - double mean = means[component]; - double variance = Math.max(0d, variances[component]); - if (variance == 0d) { - return mean; - } - return mean + Math.sqrt(variance) * nextGaussian(rng); - } - - private int drawComponent(double sample) { - double cumulative = 0d; - for (int i = 0; i < weights.length; i++) { - cumulative += weights[i]; - if (sample <= cumulative) { - return i; - } - } - return weights.length - 1; - } - - private double nextGaussian(SplittableRandom rng) { - double u1 = Math.max(Double.MIN_VALUE, rng.nextDouble()); - double u2 = rng.nextDouble(); - return Math.sqrt(-2.0d * Math.log(u1)) * Math.cos(2.0d * Math.PI * u2); - } - } - - private final class MarkovSupplier { - private final MarkovIdentifier input; - private Optional cached = Optional.empty(); - private Integer nextState; - private boolean evaluated; - - private MarkovSupplier(MarkovIdentifier input) { - this.input = input; - } - - public Optional get() { - evaluate(); - return cached; - } - - public int getNextState() { - evaluate(); - return nextState; - } - - private void evaluate() { - if (evaluated) { - return; - } - evaluated = true; - StepResult result = calculate(); - this.cached = result.value(); - this.nextState = result.nextState(); - } - - private StepResult calculate() { - int bucket = bucketId(input.time()); - int currentState = resolveState(input); - SplittableRandom rng = new SplittableRandom(deriveSeed(input, bucket, currentState)); - StepResult step = simulateStep(bucket, currentState, rng); - ComparableQuantity power = scale(input.referencePower(), step.normalizedValue()); - return new StepResult( - step.nextState(), step.normalizedValue(), Optional.of(new PValue(power))); - } - - private int bucketId(ZonedDateTime time) { - ZonedDateTime zoned = time.withZoneSameInstant(zoneId); - int month = zoned.getMonthValue() - 1; - int weekendFlag = isWeekend(zoned) ? 1 : 0; - int quarterHour = zoned.getHour() * 4 + zoned.getMinute() / 15; - return Math.floorMod( - month * MONTH_FACTOR + weekendFlag * WEEKEND_FACTOR + quarterHour, bucketCount); - } - - private boolean isWeekend(ZonedDateTime time) { - DayOfWeek day = time.getDayOfWeek(); - return day == DayOfWeek.SATURDAY || day == DayOfWeek.SUNDAY; - } - - private int resolveState(MarkovIdentifier input) { - if (input.previousState().isPresent()) { - int state = input.previousState().getAsInt(); - if (state < 0 || state >= stateCount) { - throw new IllegalArgumentException("Previous state out of bounds: " + state); - } - return state; - } - double normalized = input.initialNormalizedValue().orElseThrow(); - return discretize(normalized); - } - - private int discretize(double normalized) { - double value = clamp01(normalized); - for (int i = 0; i < discretizationThresholds.length; i++) { - if (value <= discretizationThresholds[i]) { - return i; - } - } - return discretizationThresholds.length; - } - - private StepResult simulateStep(int bucket, int currentState, SplittableRandom rng) { - double[] row = transitions[bucket][currentState]; - double[] distribution = sanitizeDistribution(bucket, row); - if (distribution.length == 0) { - return new StepResult(currentState, 0d); - } - int nextStateIndex = drawState(distribution, rng); - double normalized = sampleNormalizedValue(bucket, nextStateIndex, rng); - return new StepResult(nextStateIndex, normalized); - } - - private double[] sanitizeDistribution(int bucket, double[] row) { - double[] sanitized = new double[stateCount]; - double sum = 0d; - for (int state = 0; state < stateCount; state++) { - double sanitizedValue = 0d; - if (state < row.length) { - double value = row[state]; - if (value > 0d && !Double.isNaN(value) && gmmStates[bucket][state] != null) { - sanitizedValue = value; - sum += value; - } - } - sanitized[state] = sanitizedValue; - } - if (sum <= 0d) { - return new double[0]; - } - for (int i = 0; i < sanitized.length; i++) { - sanitized[i] /= sum; - } - return sanitized; - } - - private int drawState(double[] distribution, SplittableRandom rng) { - double sample = rng.nextDouble(); - double cumulative = 0d; - for (int i = 0; i < distribution.length; i++) { - cumulative += distribution[i]; - if (sample <= cumulative) { - return i; - } - } - return distribution.length - 1; - } - - private double sampleNormalizedValue(int bucket, int state, SplittableRandom rng) { - GmmStateData gmm = gmmStates[bucket][state]; - if (gmm == null) { - return 0d; - } - return clamp01(gmm.sample(rng)); - } - - private long deriveSeed(MarkovIdentifier input, int bucket, int state) { - long seed = input.randomSeed(); - seed = 31 * seed + bucket; - seed = 31 * seed + state; - long slot = - input.time().withZoneSameInstant(zoneId).toInstant().toEpochMilli() - / (samplingIntervalMinutes * 60_000L); - return 31 * seed + slot; - } - - private ComparableQuantity scale( - ComparableQuantity referencePower, double normalizedValue) { - Objects.requireNonNull(referencePower, "referencePower"); - if (!maxPowerFromModel.isGreaterThan(minPowerFromModel)) { - throw new IllegalStateException( - "Markov model normalization has non-positive range (max <= min)."); - } - double clamped = clamp01(normalizedValue); - ComparableQuantity range = maxPowerFromModel.subtract(minPowerFromModel); - return minPowerFromModel.add(range.multiply(clamped)).asType(Power.class); - } - - private double clamp01(double value) { - if (value < 0d) return 0d; - if (value > 1d) return 1d; - return value; - } - } -} diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index ce56ae2f55..b24370d1c6 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -6,27 +6,413 @@ package edu.ie3.datamodel.models.profile.markov; import edu.ie3.datamodel.io.source.PowerValueSource; +import edu.ie3.datamodel.models.StandardUnits; +import edu.ie3.datamodel.models.value.PValue; +import java.time.DayOfWeek; +import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.SplittableRandom; +import java.util.function.Supplier; +import javax.measure.quantity.Energy; +import javax.measure.quantity.Power; +import tech.units.indriya.ComparableQuantity; +import tech.units.indriya.quantity.Quantities; -/** Container for Markov-chain-based load models produced by simonaMarkovLoad. */ -public record MarkovLoadModel( - String schema, - ZonedDateTime generatedAt, - Generator generator, - TimeModel timeModel, - ValueModel valueModel, - Parameters parameters, - TransitionData transitionData, - Optional gmmBuckets) { +/** + * Container for Markov-chain-based load models produced by simonaMarkovLoad. + * + *

The model bundles the static data (transition matrices, GMM parameters, normalization) with + * the simulation helpers needed to generate stepwise power values. Each simulation step should use + * a fresh supplier obtained via {@link #getValueSupplier(PowerValueSource.MarkovIdentifier)}. + */ +public class MarkovLoadModel { + private static final int QUARTER_HOURS_PER_DAY = 96; + private static final int WEEKEND_FACTOR = QUARTER_HOURS_PER_DAY; + private static final int MONTH_FACTOR = QUARTER_HOURS_PER_DAY * 2; + + private final String schema; + private final ZonedDateTime generatedAt; + private final Generator generator; + private final TimeModel timeModel; + private final ValueModel valueModel; + private final Parameters parameters; + private final TransitionData transitionData; + private final Optional gmmBuckets; + + private final ZoneId zoneId; + private final double[][][] transitions; + private final int bucketCount; + private final int stateCount; + private final int samplingIntervalMinutes; + private final double[] discretizationThresholds; + private final GmmStateData[][] gmmStates; + private final ComparableQuantity maxPowerFromModel; + private final ComparableQuantity minPowerFromModel; + + public MarkovLoadModel( + String schema, + ZonedDateTime generatedAt, + Generator generator, + TimeModel timeModel, + ValueModel valueModel, + Parameters parameters, + TransitionData transitionData, + Optional gmmBuckets) { + this.schema = Objects.requireNonNull(schema, "schema"); + this.generatedAt = Objects.requireNonNull(generatedAt, "generatedAt"); + this.generator = Objects.requireNonNull(generator, "generator"); + this.timeModel = Objects.requireNonNull(timeModel, "timeModel"); + this.valueModel = Objects.requireNonNull(valueModel, "valueModel"); + this.parameters = Objects.requireNonNull(parameters, "parameters"); + this.transitionData = Objects.requireNonNull(transitionData, "transitionData"); + this.gmmBuckets = Objects.requireNonNull(gmmBuckets, "gmmBuckets"); + + this.zoneId = ZoneId.of(timeModel.timezone()); + this.transitions = transitionData.values(); + this.bucketCount = timeModel.bucketCount(); + this.stateCount = valueModel.discretization().states(); + this.samplingIntervalMinutes = timeModel.samplingIntervalMinutes(); + this.discretizationThresholds = + valueModel.discretization().thresholdsRight().stream() + .mapToDouble(Double::doubleValue) + .toArray(); + this.gmmStates = + buildGmmStates( + gmmBuckets.orElseThrow( + () -> new IllegalArgumentException("Markov model lacks GMM data."))); + this.maxPowerFromModel = + valueModel + .normalization() + .maxPower() + .map(this::convertPowerReference) + .orElseThrow( + () -> new IllegalArgumentException("Markov model lacks normalization.max_power")); + this.minPowerFromModel = + valueModel + .normalization() + .minPower() + .map(this::convertPowerReference) + .orElseThrow( + () -> new IllegalArgumentException("Markov model lacks normalization.min_power")); + } + + public String schema() { + return schema; + } + + public ZonedDateTime generatedAt() { + return generatedAt; + } + + public Generator generator() { + return generator; + } + + public TimeModel timeModel() { + return timeModel; + } + + public ValueModel valueModel() { + return valueModel; + } + + public Parameters parameters() { + return parameters; + } + + public TransitionData transitionData() { + return transitionData; + } + + public Optional gmmBuckets() { + return gmmBuckets; + } + + /** + * Returns a supplier for a single Markov step. The supplier computes the {@link + * PowerValueSource.MarkovOutputValue} once (value + next state) and caches the result for + * subsequent {@link Supplier#get()} calls. + * + *

Callers are expected to create a new supplier for each time step. + */ + public Supplier getValueSupplier( + PowerValueSource.MarkovIdentifier data) { + Objects.requireNonNull(data, "data"); + MarkovSupplier supplier = new MarkovSupplier(data); + return () -> new PowerValueSource.MarkovOutputValue(supplier.get(), supplier.getNextState()); + } + + /** Convenience helper to compute a single step immediately. */ public PowerValueSource.MarkovOutputValue getPower(PowerValueSource.MarkovIdentifier data) { - // TODO: Implement - return null; + return getValueSupplier(data).get(); + } + + /** Returns the next timestamp by advancing the model's sampling interval. */ + public Optional getNextTimeKey(ZonedDateTime time) { + return Optional.ofNullable(time).map(t -> t.plusMinutes(samplingIntervalMinutes)); + } + + /** Returns the maximum power from the model's normalization configuration. */ + public Optional> getMaxPower() { + return Optional.of(maxPowerFromModel); + } + + /** Markov models do not define an energy scaling factor. */ + public Optional> getProfileEnergyScaling() { + return Optional.empty(); + } + + private GmmStateData[][] buildGmmStates(GmmBuckets buckets) { + List bucketList = buckets.buckets(); + if (bucketList.size() != bucketCount) { + throw new IllegalArgumentException( + "GMM bucket count mismatch. Expected " + bucketCount + " but was " + bucketList.size()); + } + GmmStateData[][] lookup = new GmmStateData[bucketCount][stateCount]; + for (int bucket = 0; bucket < bucketCount; bucket++) { + List> states = bucketList.get(bucket).states(); + if (states.size() != stateCount) { + throw new IllegalArgumentException( + "State count mismatch in bucket " + bucket + ". Expected " + stateCount); + } + for (int state = 0; state < stateCount; state++) { + lookup[bucket][state] = states.get(state).map(GmmStateData::from).orElse(null); + } + } + return lookup; + } + + private ComparableQuantity convertPowerReference( + ValueModel.Normalization.PowerReference reference) { + if (!"kW".equalsIgnoreCase(reference.unit())) { + throw new IllegalArgumentException( + "Unsupported reference power unit '" + reference.unit() + "'. Only kW is supported."); + } + return Quantities.getQuantity(reference.value(), StandardUnits.ACTIVE_POWER_IN); + } + + private record StepResult(int nextState, double normalizedValue, Optional value) { + private StepResult(int nextState, double normalizedValue) { + this(nextState, normalizedValue, Optional.empty()); + } + } + + private static final class GmmStateData { + private final double[] weights; + private final double[] means; + private final double[] variances; + + private GmmStateData(double[] weights, double[] means, double[] variances) { + this.weights = weights; + this.means = means; + this.variances = variances; + } + + private static GmmStateData from(GmmBuckets.GmmState state) { + return new GmmStateData( + toArray(state.weights()), toArray(state.means()), toArray(state.variances())); + } + + private static double[] toArray(List values) { + double[] array = new double[values.size()]; + for (int i = 0; i < values.size(); i++) { + array[i] = values.get(i); + } + return array; + } + + private double sample(SplittableRandom rng) { + int component = drawComponent(rng.nextDouble()); + double mean = means[component]; + double variance = Math.max(0d, variances[component]); + if (variance == 0d) { + return mean; + } + return mean + Math.sqrt(variance) * nextGaussian(rng); + } + + private int drawComponent(double sample) { + double cumulative = 0d; + for (int i = 0; i < weights.length; i++) { + cumulative += weights[i]; + if (sample <= cumulative) { + return i; + } + } + return weights.length - 1; + } + + private double nextGaussian(SplittableRandom rng) { + double u1 = Math.max(Double.MIN_VALUE, rng.nextDouble()); + double u2 = rng.nextDouble(); + return Math.sqrt(-2.0d * Math.log(u1)) * Math.cos(2.0d * Math.PI * u2); + } + } + + private final class MarkovSupplier { + private final PowerValueSource.MarkovIdentifier input; + private Optional cached = Optional.empty(); + private Integer nextState; + private boolean evaluated; + + private MarkovSupplier(PowerValueSource.MarkovIdentifier input) { + this.input = input; + } + + public Optional get() { + evaluate(); + return cached; + } + + public int getNextState() { + evaluate(); + return nextState; + } + + private void evaluate() { + if (evaluated) { + return; + } + evaluated = true; + StepResult result = calculate(); + this.cached = result.value(); + this.nextState = result.nextState(); + } + + private StepResult calculate() { + int bucket = bucketId(input.time()); + int currentState = resolveState(input); + SplittableRandom rng = new SplittableRandom(deriveSeed(input, bucket, currentState)); + StepResult step = simulateStep(bucket, currentState, rng); + ComparableQuantity power = scale(input.referencePower(), step.normalizedValue()); + return new StepResult( + step.nextState(), step.normalizedValue(), Optional.of(new PValue(power))); + } + + private int bucketId(ZonedDateTime time) { + ZonedDateTime zoned = time.withZoneSameInstant(zoneId); + int month = zoned.getMonthValue() - 1; + int weekendFlag = isWeekend(zoned) ? 1 : 0; + int quarterHour = zoned.getHour() * 4 + zoned.getMinute() / 15; + return Math.floorMod( + month * MONTH_FACTOR + weekendFlag * WEEKEND_FACTOR + quarterHour, bucketCount); + } + + private boolean isWeekend(ZonedDateTime time) { + DayOfWeek day = time.getDayOfWeek(); + return day == DayOfWeek.SATURDAY || day == DayOfWeek.SUNDAY; + } + + private int resolveState(PowerValueSource.MarkovIdentifier input) { + if (input.previousState().isPresent()) { + int state = input.previousState().getAsInt(); + if (state < 0 || state >= stateCount) { + throw new IllegalArgumentException("Previous state out of bounds: " + state); + } + return state; + } + double normalized = input.initialNormalizedValue().orElseThrow(); + return discretize(normalized); + } + + private int discretize(double normalized) { + double value = clamp01(normalized); + for (int i = 0; i < discretizationThresholds.length; i++) { + if (value <= discretizationThresholds[i]) { + return i; + } + } + return discretizationThresholds.length; + } + + private StepResult simulateStep(int bucket, int currentState, SplittableRandom rng) { + double[] row = transitions[bucket][currentState]; + double[] distribution = sanitizeDistribution(bucket, row); + if (distribution.length == 0) { + return new StepResult(currentState, 0d); + } + int nextStateIndex = drawState(distribution, rng); + double normalized = sampleNormalizedValue(bucket, nextStateIndex, rng); + return new StepResult(nextStateIndex, normalized); + } + + private double[] sanitizeDistribution(int bucket, double[] row) { + // Ignore invalid probabilities and states without GMM data, then renormalize. + double[] sanitized = new double[stateCount]; + double sum = 0d; + for (int state = 0; state < stateCount; state++) { + double sanitizedValue = 0d; + if (state < row.length) { + double value = row[state]; + if (value > 0d && !Double.isNaN(value) && gmmStates[bucket][state] != null) { + sanitizedValue = value; + sum += value; + } + } + sanitized[state] = sanitizedValue; + } + if (sum <= 0d) { + return new double[0]; + } + for (int i = 0; i < sanitized.length; i++) { + sanitized[i] /= sum; + } + return sanitized; + } + + private int drawState(double[] distribution, SplittableRandom rng) { + double sample = rng.nextDouble(); + double cumulative = 0d; + for (int i = 0; i < distribution.length; i++) { + cumulative += distribution[i]; + if (sample <= cumulative) { + return i; + } + } + return distribution.length - 1; + } + + private double sampleNormalizedValue(int bucket, int state, SplittableRandom rng) { + GmmStateData gmm = gmmStates[bucket][state]; + if (gmm == null) { + return 0d; + } + return clamp01(gmm.sample(rng)); + } + + private long deriveSeed(PowerValueSource.MarkovIdentifier input, int bucket, int state) { + long seed = input.randomSeed(); + seed = 31 * seed + bucket; + seed = 31 * seed + state; + long slot = + input.time().withZoneSameInstant(zoneId).toInstant().toEpochMilli() + / (samplingIntervalMinutes * 60_000L); + return 31 * seed + slot; + } + + private ComparableQuantity scale( + ComparableQuantity referencePower, double normalizedValue) { + Objects.requireNonNull(referencePower, "referencePower"); + if (!maxPowerFromModel.isGreaterThan(minPowerFromModel)) { + throw new IllegalStateException( + "Markov model normalization has non-positive range (max <= min)."); + } + double clamped = clamp01(normalizedValue); + ComparableQuantity range = maxPowerFromModel.subtract(minPowerFromModel); + return minPowerFromModel.add(range.multiply(clamped)).asType(Power.class); + } + + private double clamp01(double value) { + if (value < 0d) return 0d; + if (value > 1d) return 1d; + return value; + } } public record Generator(String name, String version, Map config) {} @@ -100,4 +486,53 @@ public record GmmBucket(List> states) {} public record GmmState(List weights, List means, List variances) {} } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof MarkovLoadModel that)) return false; + return Objects.equals(schema, that.schema) + && Objects.equals(generatedAt, that.generatedAt) + && Objects.equals(generator, that.generator) + && Objects.equals(timeModel, that.timeModel) + && Objects.equals(valueModel, that.valueModel) + && Objects.equals(parameters, that.parameters) + && Objects.equals(transitionData, that.transitionData) + && Objects.equals(gmmBuckets, that.gmmBuckets); + } + + @Override + public int hashCode() { + return Objects.hash( + schema, + generatedAt, + generator, + timeModel, + valueModel, + parameters, + transitionData, + gmmBuckets); + } + + @Override + public String toString() { + return "MarkovLoadModel[" + + "schema=" + + schema + + ", generatedAt=" + + generatedAt + + ", generator=" + + generator + + ", timeModel=" + + timeModel + + ", valueModel=" + + valueModel + + ", parameters=" + + parameters + + ", transitionData=" + + transitionData + + ", gmmBuckets=" + + gmmBuckets + + ']'; + } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy index ff6509d158..c204662768 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy @@ -38,19 +38,4 @@ class JsonFileConnectorTest extends Specification { then: content == """{"foo":"bar"}""" } - - def "initReader returns buffered reader with UTF-8 decoding"() { - given: - def file = tempDir.resolve("data.json") - Files.writeString(file, "[1,2,3]") - def connector = new JsonFileConnector(tempDir) - - when: - def reader = connector.initReader(Path.of("data")) - def line = reader.readLine() - reader.close() - - then: - line == "[1,2,3]" - } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy index b19e237204..cd873c33ed 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy @@ -5,9 +5,9 @@ */ package edu.ie3.datamodel.io.factory.markov -import tools.jackson.databind.ObjectMapper import edu.ie3.datamodel.exceptions.FactoryException import spock.lang.Specification +import tools.jackson.databind.ObjectMapper class MarkovLoadModelFactoryTest extends Specification { private final ObjectMapper objectMapper = new ObjectMapper() diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy index a042eaadfd..4db82316f5 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy @@ -93,15 +93,15 @@ class JsonMarkovProfileSourceTest extends Specification { when: def supplier = source.getValueSupplier(input) - def value = supplier.get() + def output = supplier.get() then: source.getProfile().key() == "profile1" source.getMaxPower().isPresent() source.getMaxPower().get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 10d - value.isPresent() - supplier.getNextState() == 0 - value.get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 10d + output.value().isPresent() + output.nextState() == 0 + output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 10d } private static String validModelJson() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy similarity index 77% rename from src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy rename to src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy index 31b109e957..2c2df04112 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/markov/MarkovLoadValueSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy @@ -3,29 +3,26 @@ * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation */ -package edu.ie3.datamodel.io.source.markov +package edu.ie3.datamodel.models.profile.markov import edu.ie3.datamodel.io.factory.markov.MarkovLoadModelFactory import edu.ie3.datamodel.io.factory.markov.MarkovModelData import edu.ie3.datamodel.io.source.PowerValueSource import edu.ie3.datamodel.models.StandardUnits -import edu.ie3.datamodel.models.profile.markov.MarkovPowerProfile import spock.lang.Specification import tech.units.indriya.quantity.Quantities import tools.jackson.databind.ObjectMapper import java.time.ZonedDateTime -class MarkovLoadValueSourceTest extends Specification { +class MarkovLoadModelTest extends Specification { private final ObjectMapper objectMapper = new ObjectMapper() private final MarkovLoadModelFactory factory = new MarkovLoadModelFactory() - private final MarkovPowerProfile profile = new MarkovPowerProfile("profile1") def "supplier scales deterministic normalized values and exposes next state"() { given: def model = loadModel(deterministicTransitions(), deterministicStates()) - def source = new MarkovLoadValueSource(profile, model) def reference = Quantities.getQuantity(5d, StandardUnits.ACTIVE_POWER_IN) def input = new PowerValueSource.MarkovIdentifier( ZonedDateTime.parse("2025-01-01T00:00:00Z"), @@ -36,22 +33,23 @@ class MarkovLoadValueSourceTest extends Specification { ) when: - def supplier = source.getValueSupplier(input) - def value = supplier.get() + def supplier = model.getValueSupplier(input) + def output = supplier.get() + def outputAgain = supplier.get() then: - value.isPresent() - supplier.getNextState() == 1 - value.get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 4.2d - supplier.get().is(value) // cached result reused - source.getMaxPower().isPresent() - source.getMaxPower().get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 5d + output.value().isPresent() + output.nextState() == 1 + output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 4.2d + outputAgain.value() == output.value() + outputAgain.nextState() == output.nextState() + model.getMaxPower().isPresent() + model.getMaxPower().get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 5d } def "supplier denormalizes using model min and max power"() { given: def model = loadModel(deterministicTransitions(), deterministicStates()) - def source = new MarkovLoadValueSource(profile, model) def reference = Quantities.getQuantity(5d, StandardUnits.ACTIVE_POWER_IN) def input = new PowerValueSource.MarkovIdentifier( ZonedDateTime.parse("2025-01-01T00:00:00Z"), @@ -62,18 +60,17 @@ class MarkovLoadValueSourceTest extends Specification { ) when: - def supplier = source.getValueSupplier(input) - def value = supplier.get() + def supplier = model.getValueSupplier(input) + def output = supplier.get() then: - supplier.getNextState() == 1 - value.get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 4.2d + output.nextState() == 1 + output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 4.2d } def "supplier uses initial normalized value when no previous state is present"() { given: def model = loadModel(selfLoopTransitions(), deterministicStates()) - def source = new MarkovLoadValueSource(profile, model) def reference = Quantities.getQuantity(5d, StandardUnits.ACTIVE_POWER_IN) def input = new PowerValueSource.MarkovIdentifier( ZonedDateTime.parse("2025-01-01T00:00:00Z"), @@ -84,17 +81,17 @@ class MarkovLoadValueSourceTest extends Specification { ) when: - def supplier = source.getValueSupplier(input) + def supplier = model.getValueSupplier(input) + def output = supplier.get() then: - supplier.getNextState() == 0 // discretized from initial normalized value - supplier.get().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 2.6d + output.nextState() == 0 // discretized from initial normalized value + output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 2.6d } def "supplier returns zero power when transitions row has no usable probabilities"() { given: def model = loadModel(emptyTransitions(), missingStateGmms()) - def source = new MarkovLoadValueSource(profile, model) def reference = Quantities.getQuantity(3d, StandardUnits.ACTIVE_POWER_IN) def input = new PowerValueSource.MarkovIdentifier( ZonedDateTime.parse("2025-01-01T00:00:00Z"), @@ -105,13 +102,13 @@ class MarkovLoadValueSourceTest extends Specification { ) when: - def supplier = source.getValueSupplier(input) - def value = supplier.get() + def supplier = model.getValueSupplier(input) + def output = supplier.get() then: - supplier.getNextState() == 0 // stays in current state - value.isPresent() - value.get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 1d + output.nextState() == 0 // stays in current state + output.value().isPresent() + output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 1d } private loadModel(String transitions, String states) { From 3f656efd1a37bb37e7fb88c4706c78460be12f6e Mon Sep 17 00:00:00 2001 From: Philipp Date: Tue, 10 Feb 2026 07:51:12 +0100 Subject: [PATCH 17/39] some fixes --- .../io/factory/markov/MarkovModelParsingSupport.java | 9 +++++++++ .../edu/ie3/datamodel/io/source/PowerValueSource.java | 6 ++---- .../datamodel/models/profile/markov/MarkovLoadModel.java | 6 ++---- .../io/source/json/JsonMarkovProfileSourceTest.groovy | 3 --- .../models/profile/markov/MarkovLoadModelTest.groovy | 9 --------- 5 files changed, 13 insertions(+), 20 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java index 2cb038d62c..396acebffc 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -62,6 +62,15 @@ default ValueModel parseValueModel(JsonNode valueNode) { throw new FactoryException("thresholds_right must be an array"); } thresholdsNode.forEach(element -> thresholds.add(element.asDouble())); + if (thresholds.size() != Math.max(0, states - 1)) { + throw new FactoryException( + "Discretization thresholds_right must contain " + + Math.max(0, states - 1) + + " entries for " + + states + + " states, but found " + + thresholds.size()); + } ValueModel.Discretization discretization = new ValueModel.Discretization(states, List.copyOf(thresholds)); diff --git a/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java b/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java index eb9215c620..033fe13faa 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java @@ -87,14 +87,13 @@ record TimeSeriesIdentifier(ZonedDateTime time) implements PowerValueIdentifier * simonaMarkovLoad step. * *

Provide either {@code previousState} (typical for subsequent steps) or an {@code - * initialNormalizedValue} (for the first step). The {@code referencePower} is used to scale - * normalized values to real power, while {@code randomSeed} enables reproducible sampling. + * initialNormalizedValue} (for the first step). The {@code randomSeed} enables reproducible + * sampling. */ record MarkovIdentifier( ZonedDateTime time, OptionalInt previousState, OptionalDouble initialNormalizedValue, - ComparableQuantity referencePower, long randomSeed) implements PowerValueIdentifier { @@ -102,7 +101,6 @@ record MarkovIdentifier( Objects.requireNonNull(time, "time"); Objects.requireNonNull(previousState, "previousState"); Objects.requireNonNull(initialNormalizedValue, "initialNormalizedValue"); - Objects.requireNonNull(referencePower, "referencePower"); if (previousState.isEmpty() && initialNormalizedValue.isEmpty()) { throw new IllegalArgumentException( "Need either previous state or an initial normalized value to start the Markov chain."); diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index b24370d1c6..496332d57b 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -290,7 +290,7 @@ private StepResult calculate() { int currentState = resolveState(input); SplittableRandom rng = new SplittableRandom(deriveSeed(input, bucket, currentState)); StepResult step = simulateStep(bucket, currentState, rng); - ComparableQuantity power = scale(input.referencePower(), step.normalizedValue()); + ComparableQuantity power = scale(step.normalizedValue()); return new StepResult( step.nextState(), step.normalizedValue(), Optional.of(new PValue(power))); } @@ -396,9 +396,7 @@ private long deriveSeed(PowerValueSource.MarkovIdentifier input, int bucket, int return 31 * seed + slot; } - private ComparableQuantity scale( - ComparableQuantity referencePower, double normalizedValue) { - Objects.requireNonNull(referencePower, "referencePower"); + private ComparableQuantity scale(double normalizedValue) { if (!maxPowerFromModel.isGreaterThan(minPowerFromModel)) { throw new IllegalStateException( "Markov model normalization has non-positive range (max <= min)."); diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy index 4db82316f5..5e26e2d259 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy @@ -13,7 +13,6 @@ import edu.ie3.datamodel.io.source.PowerValueSource import edu.ie3.datamodel.models.StandardUnits import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel import spock.lang.Specification -import tech.units.indriya.quantity.Quantities import java.nio.file.Files import java.nio.file.Path @@ -83,12 +82,10 @@ class JsonMarkovProfileSourceTest extends Specification { new JsonDataSource(tempDir, new FileNamingStrategy()), new FileLoadProfileMetaInformation("profile1", jsonFile, FileType.JSON) ) - def referencePower = Quantities.getQuantity(10d, StandardUnits.ACTIVE_POWER_IN) def input = new PowerValueSource.MarkovIdentifier( ZonedDateTime.parse("2025-01-01T00:00:00Z"), OptionalInt.of(0), OptionalDouble.empty(), - referencePower, 42L) when: diff --git a/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy b/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy index 2c2df04112..6d310d1757 100644 --- a/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy @@ -10,7 +10,6 @@ import edu.ie3.datamodel.io.factory.markov.MarkovModelData import edu.ie3.datamodel.io.source.PowerValueSource import edu.ie3.datamodel.models.StandardUnits import spock.lang.Specification -import tech.units.indriya.quantity.Quantities import tools.jackson.databind.ObjectMapper import java.time.ZonedDateTime @@ -23,12 +22,10 @@ class MarkovLoadModelTest extends Specification { def "supplier scales deterministic normalized values and exposes next state"() { given: def model = loadModel(deterministicTransitions(), deterministicStates()) - def reference = Quantities.getQuantity(5d, StandardUnits.ACTIVE_POWER_IN) def input = new PowerValueSource.MarkovIdentifier( ZonedDateTime.parse("2025-01-01T00:00:00Z"), OptionalInt.of(0), OptionalDouble.empty(), - reference, 99L ) @@ -50,12 +47,10 @@ class MarkovLoadModelTest extends Specification { def "supplier denormalizes using model min and max power"() { given: def model = loadModel(deterministicTransitions(), deterministicStates()) - def reference = Quantities.getQuantity(5d, StandardUnits.ACTIVE_POWER_IN) def input = new PowerValueSource.MarkovIdentifier( ZonedDateTime.parse("2025-01-01T00:00:00Z"), OptionalInt.of(0), OptionalDouble.empty(), - reference, 17L ) @@ -71,12 +66,10 @@ class MarkovLoadModelTest extends Specification { def "supplier uses initial normalized value when no previous state is present"() { given: def model = loadModel(selfLoopTransitions(), deterministicStates()) - def reference = Quantities.getQuantity(5d, StandardUnits.ACTIVE_POWER_IN) def input = new PowerValueSource.MarkovIdentifier( ZonedDateTime.parse("2025-01-01T00:00:00Z"), OptionalInt.empty(), OptionalDouble.of(0.25d), - reference, 13L ) @@ -92,12 +85,10 @@ class MarkovLoadModelTest extends Specification { def "supplier returns zero power when transitions row has no usable probabilities"() { given: def model = loadModel(emptyTransitions(), missingStateGmms()) - def reference = Quantities.getQuantity(3d, StandardUnits.ACTIVE_POWER_IN) def input = new PowerValueSource.MarkovIdentifier( ZonedDateTime.parse("2025-01-01T00:00:00Z"), OptionalInt.of(0), OptionalDouble.empty(), - reference, 7L ) From 85d8365acdc8b8ff5de23cfd6679997396ec7800 Mon Sep 17 00:00:00 2001 From: Philipp Date: Tue, 10 Feb 2026 08:13:44 +0100 Subject: [PATCH 18/39] conflicts --- .../datamodel/io/source/json/JsonMarkovProfileSource.java | 5 +---- .../ie3/datamodel/io/source/sql/SqlLoadProfileSource.java | 3 ++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java index 5c07486c50..b18889bf98 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java @@ -172,10 +172,7 @@ private MarkovLoadModel getModelUnchecked() { return getModel(); } catch (SourceException e) { throw new IllegalStateException( - "Unable to load Markov model '" - + metaInformation.getProfileKey().getValue() - + "'.", - e); + "Unable to load Markov model '" + metaInformation.getProfileKey().getValue() + "'.", e); } } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSource.java index b535698188..09f46fb52c 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSource.java @@ -99,7 +99,8 @@ public Set> getEntries() { public Supplier getValueSupplier(TimeSeriesInputValue data) { ZonedDateTime time = data.time(); Optional loadValueOption = queryForValue(time); - return TimeSeriesOutputValue.from(() -> loadValueOption.map(v -> v.getValue(time, powerProfileKey))); + return TimeSeriesOutputValue.from( + () -> loadValueOption.map(v -> v.getValue(time, powerProfileKey))); } @Override From cb0edea4a14875cba79aeac55bfd014dca6d3c06 Mon Sep 17 00:00:00 2001 From: Philipp Date: Tue, 10 Feb 2026 08:16:40 +0100 Subject: [PATCH 19/39] fix --- .../datamodel/models/profile/markov/MarkovPowerProfile.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovPowerProfile.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovPowerProfile.java index 171781d012..343bcbb56e 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovPowerProfile.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovPowerProfile.java @@ -29,4 +29,9 @@ private static PowerProfileKey buildKey(String key) { } return new PowerProfileKey(key); } + + @Override + public PowerProfileKey getKey() { + return key; + } } From 9153fb3c1469205876e420ba75e3bf18fd1e1531 Mon Sep 17 00:00:00 2001 From: Philipp Date: Tue, 10 Feb 2026 08:23:33 +0100 Subject: [PATCH 20/39] fix --- .../io/naming/EntityPersistenceNamingStrategyTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy index 2562c2e3dd..3b8b418e64 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy @@ -142,7 +142,7 @@ class EntityPersistenceNamingStrategyTest extends Specification { def meta = ens.loadProfileTimesSeriesMetaInformation(fileName) then: - meta.profile == "demo2" + meta.profileKey.getValue() == "demo2" } def "The EntityPersistenceNamingStrategy is able to prepare the prefix properly"() { From 413f707c511827a035b11d30ced4b17577c3367b Mon Sep 17 00:00:00 2001 From: Philipp Date: Tue, 10 Feb 2026 09:25:44 +0100 Subject: [PATCH 21/39] rtd and rename --- docs/readthedocs/io/csvfiles.md | 8 +++++++- .../io/factory/markov/MarkovLoadModelFactory.java | 2 +- .../io/factory/markov/MarkovModelParsingSupport.java | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/readthedocs/io/csvfiles.md b/docs/readthedocs/io/csvfiles.md index 3e455edeac..aa50a1e86e 100644 --- a/docs/readthedocs/io/csvfiles.md +++ b/docs/readthedocs/io/csvfiles.md @@ -160,6 +160,12 @@ The following keys are supported until now: ##### Load Profile Time Series The following profiles are supported until now: + +Note: +Load profile keys must be unique across all load-profile sources. Do not use the same profile key +for both CSV (lpts_*) and Markov JSON (markov_*), e.g. avoid having both lpts_h0.csv and +markov_h0.json. + ```{list-table} :widths: auto :class: wrapping @@ -335,4 +341,4 @@ occur. We consider either regular directories or compressed [tarball archives](https://en.wikipedia.org/wiki/Tar_(computing)) (`*.tar.gz`) as source of input files. -The class `TarballUtils` offers some helpful functions to compress or extract input data files for easier shipping. \ No newline at end of file +The class `TarballUtils` offers some helpful functions to compress or extract input data files for easier shipping. diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java index 79b5cecc21..0272a5c84a 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -39,7 +39,7 @@ protected MarkovLoadModel buildModel(MarkovModelData data) { String schema = requireText(root, "schema"); ZonedDateTime generatedAt = parseTimestamp(requireText(root, "generated_at")); Generator generator = parseGenerator(requireNode(root, "generator")); - TimeModel timeModel = parseTimeModel(requireNode(root, "time_model")); + TimeModel timeModel = extractTimeModel(requireNode(root, "time_model")); ValueModel valueModel = parseValueModel(requireNode(root, "value_model")); Parameters parameters = parseParameters(root.path("parameters")); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java index 396acebffc..f08b61d2e5 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -31,8 +31,8 @@ default Generator parseGenerator(JsonNode generatorNode) { return new Generator(name, version, config); } - /** Parses the time model block, including bucket count and sampling interval. */ - default TimeModel parseTimeModel(JsonNode timeNode) { + /** Extracts the time model block, including bucket count and sampling interval. */ + default TimeModel extractTimeModel(JsonNode timeNode) { int bucketCount = requireInt(timeNode, "bucket_count"); String formula = requireNode(timeNode, "bucket_encoding").path("formula").asString(""); if (formula.isEmpty()) { From b6c81b505a49a8dd29b8709a5a1e1e8d53686b92 Mon Sep 17 00:00:00 2001 From: Philipp Date: Wed, 25 Feb 2026 15:58:26 +0100 Subject: [PATCH 22/39] refactoring after conversation --- docs/readthedocs/io/csvfiles.md | 3 + .../markov/MarkovLoadModelFactory.java | 14 +- .../markov/MarkovModelParsingSupport.java | 77 +++--- .../io/source/json/JsonDataSource.java | 55 +++- .../source/json/JsonMarkovProfileSource.java | 58 +--- .../profile/markov/MarkovLoadModel.java | 254 ++++++++---------- .../profile/markov/MarkovPowerProfile.java | 37 --- .../markov/MarkovLoadModelFactoryTest.groovy | 2 +- .../io/source/json/JsonDataSourceTest.groovy | 34 +++ 9 files changed, 246 insertions(+), 288 deletions(-) delete mode 100644 src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovPowerProfile.java diff --git a/docs/readthedocs/io/csvfiles.md b/docs/readthedocs/io/csvfiles.md index aa50a1e86e..cf80316992 100644 --- a/docs/readthedocs/io/csvfiles.md +++ b/docs/readthedocs/io/csvfiles.md @@ -166,6 +166,9 @@ Load profile keys must be unique across all load-profile sources. Do not use the for both CSV (lpts_*) and Markov JSON (markov_*), e.g. avoid having both lpts_h0.csv and markov_h0.json. +Markov-based load models (`markov_*`) do not support energy scaling. Calls to +`getProfileEnergyScaling()` will always return `Optional.empty()`. + ```{list-table} :widths: auto :class: wrapping diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java index 0272a5c84a..4441a8a4b6 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -36,17 +36,17 @@ public MarkovLoadModelFactory() { @Override protected MarkovLoadModel buildModel(MarkovModelData data) { JsonNode root = data.getRoot(); - String schema = requireText(root, "schema"); - ZonedDateTime generatedAt = parseTimestamp(requireText(root, "generated_at")); - Generator generator = parseGenerator(requireNode(root, "generator")); - TimeModel timeModel = extractTimeModel(requireNode(root, "time_model")); - ValueModel valueModel = parseValueModel(requireNode(root, "value_model")); + String schema = extractText(root, "schema"); + ZonedDateTime generatedAt = parseTimestamp(extractText(root, "generated_at")); + Generator generator = parseGenerator(extractNode(root, "generator")); + TimeModel timeModel = extractTimeModel(extractNode(root, "time_model")); + ValueModel valueModel = parseValueModel(extractNode(root, "value_model")); Parameters parameters = parseParameters(root.path("parameters")); - JsonNode dataNode = requireNode(root, "data"); + JsonNode dataNode = extractNode(root, "data"); TransitionData transitionData = parseTransitions(dataNode, timeModel.bucketCount(), valueModel.discretization().states()); - GmmBuckets gmmBuckets = parseGmmBuckets(requireNode(dataNode, "gmms")); + GmmBuckets gmmBuckets = parseGmmBuckets(extractNode(dataNode, "gmms")); return new MarkovLoadModel( schema, diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java index f08b61d2e5..16a3f5fa52 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -19,8 +19,8 @@ interface MarkovModelParsingSupport { default Generator parseGenerator(JsonNode generatorNode) { - String name = requireText(generatorNode, "name"); - String version = requireText(generatorNode, "version"); + String name = extractText(generatorNode, "name"); + String version = extractText(generatorNode, "version"); Map config = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); JsonNode configNode = generatorNode.path("config"); if (configNode.isObject()) { @@ -33,35 +33,30 @@ default Generator parseGenerator(JsonNode generatorNode) { /** Extracts the time model block, including bucket count and sampling interval. */ default TimeModel extractTimeModel(JsonNode timeNode) { - int bucketCount = requireInt(timeNode, "bucket_count"); - String formula = requireNode(timeNode, "bucket_encoding").path("formula").asString(""); + int bucketCount = extractInt(timeNode, "bucket_count"); + String formula = extractNode(timeNode, "bucket_encoding").path("formula").asString(""); if (formula.isEmpty()) { throw new FactoryException("Missing bucket encoding formula"); } - int samplingInterval = requireInt(timeNode, "sampling_interval_minutes"); - String timezone = requireText(timeNode, "timezone"); + int samplingInterval = extractInt(timeNode, "sampling_interval_minutes"); + String timezone = extractText(timeNode, "timezone"); return new TimeModel(bucketCount, formula, samplingInterval, timezone); } /** Parses value model settings (unit, normalization, discretization thresholds). */ default ValueModel parseValueModel(JsonNode valueNode) { - String valueUnit = requireText(valueNode, "value_unit"); - JsonNode normalizationNode = requireNode(valueNode, "normalization"); - String normalizationMethod = requireText(normalizationNode, "method"); + String valueUnit = extractText(valueNode, "value_unit"); + JsonNode normalizationNode = extractNode(valueNode, "normalization"); + String normalizationMethod = extractText(normalizationNode, "method"); ValueModel.Normalization normalization = new ValueModel.Normalization( normalizationMethod, parsePowerReference(normalizationNode, "max_power"), parsePowerReference(normalizationNode, "min_power")); - JsonNode discretizationNode = requireNode(valueNode, "discretization"); - int states = requireInt(discretizationNode, "states"); - List thresholds = new ArrayList<>(); - JsonNode thresholdsNode = requireNode(discretizationNode, "thresholds_right"); - if (!thresholdsNode.isArray()) { - throw new FactoryException("thresholds_right must be an array"); - } - thresholdsNode.forEach(element -> thresholds.add(element.asDouble())); + JsonNode discretizationNode = extractNode(valueNode, "discretization"); + int states = extractInt(discretizationNode, "states"); + List thresholds = readDoubleArray(discretizationNode, "thresholds_right"); if (thresholds.size() != Math.max(0, states - 1)) { throw new FactoryException( "Discretization thresholds_right must contain " @@ -71,8 +66,7 @@ default ValueModel parseValueModel(JsonNode valueNode) { + " states, but found " + thresholds.size()); } - ValueModel.Discretization discretization = - new ValueModel.Discretization(states, List.copyOf(thresholds)); + ValueModel.Discretization discretization = new ValueModel.Discretization(states, thresholds); return new ValueModel(valueUnit, normalization, discretization); } @@ -105,9 +99,9 @@ default Parameters parseParameters(JsonNode parametersNode) { */ default TransitionData parseTransitions( JsonNode dataNode, int expectedBucketCount, int stateCount) { - JsonNode transitionsNode = requireNode(dataNode, "transitions"); - String dtype = requireText(transitionsNode, "dtype"); - String encoding = requireText(transitionsNode, "encoding"); + JsonNode transitionsNode = extractNode(dataNode, "transitions"); + String dtype = extractText(transitionsNode, "dtype"); + String encoding = extractText(transitionsNode, "encoding"); int[] shape = parseTransitionShape(transitionsNode); int buckets = shape[0]; @@ -115,7 +109,7 @@ default TransitionData parseTransitions( int columns = shape[2]; validateTransitionShape(expectedBucketCount, stateCount, buckets, rows, columns); - JsonNode valuesNode = requireNode(transitionsNode, "values"); + JsonNode valuesNode = extractNode(transitionsNode, "values"); double[][][] values = parseTransitionValues(valuesNode, buckets, stateCount); return new TransitionData(dtype, encoding, values); @@ -127,7 +121,7 @@ default GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { throw new FactoryException("Missing field 'gmms'"); } JsonNode bucketsNode = gmmsNode.get("buckets"); - if (!bucketsNode.isArray()) { + if (bucketsNode == null || !bucketsNode.isArray()) { throw new FactoryException("data.gmms.buckets must be an array"); } List buckets = new ArrayList<>(); @@ -136,23 +130,23 @@ default GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { if (statesNode == null || !statesNode.isArray()) { throw new FactoryException("Each GMM bucket must contain an array 'states'"); } - List> states = new ArrayList<>(); + List states = new ArrayList<>(); for (JsonNode stateNode : statesNode) { if (stateNode == null || stateNode.isNull()) { - states.add(Optional.empty()); + states.add(null); continue; } List weights = readDoubleArray(stateNode, "weights"); List means = readDoubleArray(stateNode, "means"); List variances = readDoubleArray(stateNode, "variances"); - states.add(Optional.of(new GmmBuckets.GmmState(weights, means, variances))); + states.add(new GmmBuckets.GmmState(weights, means, variances)); } - buckets.add(new GmmBuckets.GmmBucket(List.copyOf(states))); + buckets.add(new GmmBuckets.GmmBucket(Collections.unmodifiableList(states))); } return new GmmBuckets(List.copyOf(buckets)); } - default JsonNode requireNode(JsonNode node, String field) { + default JsonNode extractNode(JsonNode node, String field) { JsonNode value = node.get(field); if (value == null || value.isMissingNode()) { throw new FactoryException("Missing field '" + field + "'"); @@ -160,7 +154,7 @@ default JsonNode requireNode(JsonNode node, String field) { return value; } - default String requireText(JsonNode node, String field) { + default String extractText(JsonNode node, String field) { JsonNode value = node.get(field); if (value == null || value.isMissingNode() || value.isNull()) { throw new FactoryException("Missing field '" + field + "'"); @@ -171,18 +165,18 @@ default String requireText(JsonNode node, String field) { return value.asString(); } - default double requireDouble(JsonNode node, String field) { + default double extractDouble(JsonNode node, String field) { JsonNode value = node.get(field); if (value == null || value.isMissingNode() || value.isNull()) { throw new FactoryException("Missing field '" + field + "'"); } if (!value.isNumber()) { - throw new FactoryException("Field '" + field + "' must be numeric"); + throw new FactoryException("Field '" + field + "' must be a double"); } return value.asDouble(); } - default int requireInt(JsonNode node, String field) { + default int extractInt(JsonNode node, String field) { JsonNode value = node.get(field); if (value == null || value.isMissingNode() || value.isNull()) { throw new FactoryException("Missing field '" + field + "'"); @@ -208,7 +202,7 @@ default Optional optionalInt(JsonNode node, String field) { } default int[] parseTransitionShape(JsonNode transitionsNode) { - JsonNode shapeNode = requireNode(transitionsNode, "shape"); + JsonNode shapeNode = extractNode(transitionsNode, "shape"); if (!shapeNode.isArray() || shapeNode.size() != 3) { throw new FactoryException("Transition shape must contain three dimensions"); } @@ -239,24 +233,21 @@ default double[][][] parseTransitionValues(JsonNode valuesNode, int buckets, int if (!valuesNode.isArray()) { throw new FactoryException("Transition values must be a three dimensional array"); } + if (valuesNode.size() != buckets) { + throw new FactoryException( + "Transition values provided " + valuesNode.size() + " buckets. Expected " + buckets); + } double[][][] values = new double[buckets][stateCount][stateCount]; int bucketIndex = 0; for (JsonNode bucketNode : valuesNode) { fillBucket(values, bucketNode, bucketIndex, stateCount); bucketIndex++; } - if (bucketIndex != buckets) { - throw new FactoryException( - "Transition values provided only " + bucketIndex + " buckets. Expected " + buckets); - } return values; } default void fillBucket( double[][][] values, JsonNode bucketNode, int bucketIndex, int stateCount) { - if (bucketIndex >= values.length) { - throw new FactoryException("More transition buckets present than specified in shape"); - } int rowIndex = 0; for (JsonNode rowNode : bucketNode) { fillRow(values, rowNode, bucketIndex, rowIndex, stateCount); @@ -317,8 +308,8 @@ default Optional parsePowerReference( if (!referenceNode.isObject()) { throw new FactoryException("Field '" + field + "' must be an object"); } - double value = requireDouble(referenceNode, "value"); - String unit = requireText(referenceNode, "unit"); + double value = extractDouble(referenceNode, "value"); + String unit = extractText(referenceNode, "unit"); return Optional.of(new ValueModel.Normalization.PowerReference(value, unit)); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java index eb542bb4a7..f82ad74caf 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java @@ -11,17 +11,22 @@ import edu.ie3.datamodel.io.source.file.FileDataSource; import edu.ie3.datamodel.models.Entity; import java.io.FileNotFoundException; +import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeSet; import java.util.stream.Stream; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; /** Data source abstraction for JSON files. */ public class JsonDataSource extends FileDataSource { private final JsonFileConnector connector; + private final ObjectMapper objectMapper = new ObjectMapper(); public JsonDataSource(Path directoryPath, FileNamingStrategy fileNamingStrategy) { this(new JsonFileConnector(directoryPath), fileNamingStrategy); @@ -47,6 +52,21 @@ public InputStream initInputStream(Path filePath) throws SourceException { } } + /** + * Reads and parses a JSON file into a {@link JsonNode} tree. + * + * @param filePath path to the JSON file + * @return the parsed JSON tree + * @throws SourceException if the file cannot be read or parsed + */ + public JsonNode readTree(Path filePath) throws SourceException { + try (InputStream inputStream = initInputStream(filePath)) { + return objectMapper.readTree(inputStream); + } catch (IOException e) { + throw new SourceException("Unable to read JSON from '" + filePath + "'.", e); + } + } + @Override public Optional> getSourceFields(Class entityClass) throws SourceException { @@ -59,9 +79,17 @@ public Stream> getSourceData(Class entityC throw unsupportedTabularAccess("getSourceData(Class)"); } + /** + * Returns the set of field names present in the JSON file at the given path. + * + * @param filePath path to the JSON file + * @return an optional containing the field names, or empty if none found + * @throws SourceException if the file cannot be read + */ @Override public Optional> getSourceFields(Path filePath) throws SourceException { - throw unsupportedTabularAccess("getSourceFields(Path)"); + JsonNode root = readTree(filePath); + return Optional.of(collectFieldNames(root)); } @Override @@ -69,6 +97,31 @@ public Stream> getSourceData(Path filePath) throws SourceExc throw unsupportedTabularAccess("getSourceData(Path)"); } + private static Set collectFieldNames(JsonNode node) { + Set fields = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + collectFields("", node, fields); + return fields; + } + + private static void collectFields(String prefix, JsonNode node, Set collector) { + if (node.isArray()) { + if (!prefix.isEmpty()) { + collector.add(prefix); + } + return; + } + if (node.isObject()) { + node.propertyNames() + .forEach(name -> collectFields(join(prefix, name), node.get(name), collector)); + } else if (!prefix.isEmpty()) { + collector.add(prefix); + } + } + + private static String join(String prefix, String name) { + return prefix.isEmpty() ? name : prefix + "." + name; + } + private UnsupportedOperationException unsupportedTabularAccess(String method) { return new UnsupportedOperationException( "JsonDataSource does not support '" + method + "', as JSON sources are not tabular."); diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java index b18889bf98..a532df00a7 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java @@ -17,21 +17,15 @@ import edu.ie3.datamodel.io.source.PowerValueSource; import edu.ie3.datamodel.models.profile.PowerProfileKey; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; -import edu.ie3.datamodel.models.profile.markov.MarkovPowerProfile; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Path; import java.time.ZonedDateTime; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.TreeSet; import java.util.function.Supplier; import javax.measure.quantity.Energy; import javax.measure.quantity.Power; import tech.units.indriya.ComparableQuantity; import tools.jackson.databind.JsonNode; -import tools.jackson.databind.ObjectMapper; /** * Source that reads Markov-based load models from JSON files. @@ -44,8 +38,6 @@ public class JsonMarkovProfileSource extends EntitySource implements PowerValueS private final JsonDataSource dataSource; private final FileLoadProfileMetaInformation metaInformation; private final MarkovLoadModelFactory factory; - private final ObjectMapper objectMapper = new ObjectMapper(); - private final MarkovPowerProfile profile; private MarkovLoadModel cachedModel; public JsonMarkovProfileSource( @@ -60,7 +52,6 @@ public JsonMarkovProfileSource( this.dataSource = Objects.requireNonNull(dataSource, "dataSource"); this.metaInformation = Objects.requireNonNull(metaInformation, "metaInformation"); this.factory = Objects.requireNonNull(factory, "factory"); - this.profile = new MarkovPowerProfile(metaInformation.getProfileKey()); if (metaInformation.getFileType() != FileType.JSON) { throw new IllegalArgumentException("Markov profile source requires JSON meta information."); } @@ -73,7 +64,7 @@ public JsonMarkovProfileSource( */ public synchronized MarkovLoadModel getModel() throws SourceException { if (cachedModel == null) { - JsonNode root = readModelTree(); + JsonNode root = dataSource.readTree(metaInformation.getFullFilePath()); try { cachedModel = factory.get(new MarkovModelData(root)).getOrThrow(); } catch (FactoryException e) { @@ -89,9 +80,10 @@ public synchronized MarkovLoadModel getModel() throws SourceException { @Override public void validate() throws ValidationException { - JsonNode root; try { - root = readModelTree(); + Set fields = + dataSource.getSourceFields(metaInformation.getFullFilePath()).orElse(Set.of()); + factory.validate(fields, MarkovLoadModel.class).getOrThrow(); } catch (SourceException e) { throw new FailedValidationException( "Unable to read Markov model '" @@ -99,17 +91,11 @@ public void validate() throws ValidationException { + "' for validation.", e); } - Set fields = collectFieldNames(root); - factory.validate(fields, MarkovLoadModel.class).getOrThrow(); } @Override public PowerProfileKey getProfileKey() { - return profile.key(); - } - - public MarkovPowerProfile getProfile() { - return profile; + return metaInformation.getProfileKey(); } /** Delegates to the cached {@link MarkovLoadModel} for a single simulation step. */ @@ -133,40 +119,6 @@ public Optional> getProfileEnergyScaling() { return getModelUnchecked().getProfileEnergyScaling(); } - private JsonNode readModelTree() throws SourceException { - Path filePath = metaInformation.getFullFilePath(); - try (InputStream inputStream = dataSource.initInputStream(filePath)) { - return objectMapper.readTree(inputStream); - } catch (IOException e) { - throw new SourceException("Unable to read Markov model JSON from '" + filePath + "'.", e); - } - } - - private static Set collectFieldNames(JsonNode node) { - Set fields = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - collectFields("", node, fields); - return fields; - } - - private static void collectFields(String prefix, JsonNode node, Set collector) { - if (node.isArray()) { - if (!prefix.isEmpty()) { - collector.add(prefix); - } - return; - } - if (node.isObject()) { - node.propertyNames() - .forEach(name -> collectFields(join(prefix, name), node.get(name), collector)); - } else if (!prefix.isEmpty()) { - collector.add(prefix); - } - } - - private static String join(String prefix, String name) { - return prefix.isEmpty() ? name : prefix + "." + name; - } - private MarkovLoadModel getModelUnchecked() { try { return getModel(); diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index 496332d57b..07d366c0f4 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -135,17 +135,23 @@ public Optional gmmBuckets() { } /** - * Returns a supplier for a single Markov step. The supplier computes the {@link - * PowerValueSource.MarkovOutputValue} once (value + next state) and caches the result for - * subsequent {@link Supplier#get()} calls. + * Returns a supplier for a single Markov step. * *

Callers are expected to create a new supplier for each time step. */ public Supplier getValueSupplier( PowerValueSource.MarkovIdentifier data) { Objects.requireNonNull(data, "data"); - MarkovSupplier supplier = new MarkovSupplier(data); - return () -> new PowerValueSource.MarkovOutputValue(supplier.get(), supplier.getNextState()); + return () -> computeStep(data); + } + + private PowerValueSource.MarkovOutputValue computeStep(PowerValueSource.MarkovIdentifier data) { + int bucket = bucketId(data.time()); + int currentState = resolveState(data); + SplittableRandom rng = new SplittableRandom(deriveSeed(data, bucket, currentState)); + StepResult step = simulateStep(bucket, currentState, rng); + ComparableQuantity power = scale(step.normalizedValue()); + return new PowerValueSource.MarkovOutputValue(Optional.of(new PValue(power)), step.nextState()); } /** Convenience helper to compute a single step immediately. */ @@ -176,13 +182,14 @@ private GmmStateData[][] buildGmmStates(GmmBuckets buckets) { } GmmStateData[][] lookup = new GmmStateData[bucketCount][stateCount]; for (int bucket = 0; bucket < bucketCount; bucket++) { - List> states = bucketList.get(bucket).states(); + List states = bucketList.get(bucket).states(); if (states.size() != stateCount) { throw new IllegalArgumentException( "State count mismatch in bucket " + bucket + ". Expected " + stateCount); } for (int state = 0; state < stateCount; state++) { - lookup[bucket][state] = states.get(state).map(GmmStateData::from).orElse(null); + GmmBuckets.GmmState s = states.get(state); + lookup[bucket][state] = s != null ? GmmStateData.from(s) : null; } } return lookup; @@ -197,11 +204,7 @@ private ComparableQuantity convertPowerReference( return Quantities.getQuantity(reference.value(), StandardUnits.ACTIVE_POWER_IN); } - private record StepResult(int nextState, double normalizedValue, Optional value) { - private StepResult(int nextState, double normalizedValue) { - this(nextState, normalizedValue, Optional.empty()); - } - } + private record StepResult(int nextState, double normalizedValue) {} private static final class GmmStateData { private final double[] weights; @@ -255,162 +258,121 @@ private double nextGaussian(SplittableRandom rng) { } } - private final class MarkovSupplier { - private final PowerValueSource.MarkovIdentifier input; - private Optional cached = Optional.empty(); - private Integer nextState; - private boolean evaluated; - - private MarkovSupplier(PowerValueSource.MarkovIdentifier input) { - this.input = input; - } - - public Optional get() { - evaluate(); - return cached; - } + private int bucketId(ZonedDateTime time) { + ZonedDateTime zoned = time.withZoneSameInstant(zoneId); + int month = zoned.getMonthValue() - 1; + int weekendFlag = isWeekend(zoned) ? 1 : 0; + int quarterHour = zoned.getHour() * 4 + zoned.getMinute() / 15; + return Math.floorMod( + month * MONTH_FACTOR + weekendFlag * WEEKEND_FACTOR + quarterHour, bucketCount); + } - public int getNextState() { - evaluate(); - return nextState; - } + private boolean isWeekend(ZonedDateTime time) { + DayOfWeek day = time.getDayOfWeek(); + return day == DayOfWeek.SATURDAY || day == DayOfWeek.SUNDAY; + } - private void evaluate() { - if (evaluated) { - return; + private int resolveState(PowerValueSource.MarkovIdentifier input) { + if (input.previousState().isPresent()) { + int state = input.previousState().getAsInt(); + if (state < 0 || state >= stateCount) { + throw new IllegalArgumentException("Previous state out of bounds: " + state); } - evaluated = true; - StepResult result = calculate(); - this.cached = result.value(); - this.nextState = result.nextState(); - } - - private StepResult calculate() { - int bucket = bucketId(input.time()); - int currentState = resolveState(input); - SplittableRandom rng = new SplittableRandom(deriveSeed(input, bucket, currentState)); - StepResult step = simulateStep(bucket, currentState, rng); - ComparableQuantity power = scale(step.normalizedValue()); - return new StepResult( - step.nextState(), step.normalizedValue(), Optional.of(new PValue(power))); + return state; } + double normalized = input.initialNormalizedValue().orElseThrow(); + return discretize(normalized); + } - private int bucketId(ZonedDateTime time) { - ZonedDateTime zoned = time.withZoneSameInstant(zoneId); - int month = zoned.getMonthValue() - 1; - int weekendFlag = isWeekend(zoned) ? 1 : 0; - int quarterHour = zoned.getHour() * 4 + zoned.getMinute() / 15; - return Math.floorMod( - month * MONTH_FACTOR + weekendFlag * WEEKEND_FACTOR + quarterHour, bucketCount); + private int discretize(double normalized) { + double value = clamp01(normalized); + for (int i = 0; i < discretizationThresholds.length; i++) { + if (value <= discretizationThresholds[i]) { + return i; + } } + return discretizationThresholds.length; + } - private boolean isWeekend(ZonedDateTime time) { - DayOfWeek day = time.getDayOfWeek(); - return day == DayOfWeek.SATURDAY || day == DayOfWeek.SUNDAY; + private StepResult simulateStep(int bucket, int currentState, SplittableRandom rng) { + double[] row = transitions[bucket][currentState]; + double[] distribution = sanitizeDistribution(bucket, row); + if (distribution.length == 0) { + return new StepResult(currentState, 0d); } + int nextStateIndex = drawState(distribution, rng); + double normalized = sampleNormalizedValue(bucket, nextStateIndex, rng); + return new StepResult(nextStateIndex, normalized); + } - private int resolveState(PowerValueSource.MarkovIdentifier input) { - if (input.previousState().isPresent()) { - int state = input.previousState().getAsInt(); - if (state < 0 || state >= stateCount) { - throw new IllegalArgumentException("Previous state out of bounds: " + state); + private double[] sanitizeDistribution(int bucket, double[] row) { + // Ignore invalid probabilities and states without GMM data, then renormalize. + double[] sanitized = new double[stateCount]; + double sum = 0d; + for (int state = 0; state < stateCount; state++) { + double sanitizedValue = 0d; + if (state < row.length) { + double value = row[state]; + if (value > 0d && !Double.isNaN(value) && gmmStates[bucket][state] != null) { + sanitizedValue = value; + sum += value; } - return state; } - double normalized = input.initialNormalizedValue().orElseThrow(); - return discretize(normalized); + sanitized[state] = sanitizedValue; } - - private int discretize(double normalized) { - double value = clamp01(normalized); - for (int i = 0; i < discretizationThresholds.length; i++) { - if (value <= discretizationThresholds[i]) { - return i; - } - } - return discretizationThresholds.length; + if (sum <= 0d) { + return new double[0]; } - - private StepResult simulateStep(int bucket, int currentState, SplittableRandom rng) { - double[] row = transitions[bucket][currentState]; - double[] distribution = sanitizeDistribution(bucket, row); - if (distribution.length == 0) { - return new StepResult(currentState, 0d); - } - int nextStateIndex = drawState(distribution, rng); - double normalized = sampleNormalizedValue(bucket, nextStateIndex, rng); - return new StepResult(nextStateIndex, normalized); + for (int i = 0; i < sanitized.length; i++) { + sanitized[i] /= sum; } + return sanitized; + } - private double[] sanitizeDistribution(int bucket, double[] row) { - // Ignore invalid probabilities and states without GMM data, then renormalize. - double[] sanitized = new double[stateCount]; - double sum = 0d; - for (int state = 0; state < stateCount; state++) { - double sanitizedValue = 0d; - if (state < row.length) { - double value = row[state]; - if (value > 0d && !Double.isNaN(value) && gmmStates[bucket][state] != null) { - sanitizedValue = value; - sum += value; - } - } - sanitized[state] = sanitizedValue; - } - if (sum <= 0d) { - return new double[0]; - } - for (int i = 0; i < sanitized.length; i++) { - sanitized[i] /= sum; + private int drawState(double[] distribution, SplittableRandom rng) { + double sample = rng.nextDouble(); + double cumulative = 0d; + for (int i = 0; i < distribution.length; i++) { + cumulative += distribution[i]; + if (sample <= cumulative) { + return i; } - return sanitized; - } - - private int drawState(double[] distribution, SplittableRandom rng) { - double sample = rng.nextDouble(); - double cumulative = 0d; - for (int i = 0; i < distribution.length; i++) { - cumulative += distribution[i]; - if (sample <= cumulative) { - return i; - } - } - return distribution.length - 1; } + return distribution.length - 1; + } - private double sampleNormalizedValue(int bucket, int state, SplittableRandom rng) { - GmmStateData gmm = gmmStates[bucket][state]; - if (gmm == null) { - return 0d; - } - return clamp01(gmm.sample(rng)); + private double sampleNormalizedValue(int bucket, int state, SplittableRandom rng) { + GmmStateData gmm = gmmStates[bucket][state]; + if (gmm == null) { + return 0d; } + return clamp01(gmm.sample(rng)); + } - private long deriveSeed(PowerValueSource.MarkovIdentifier input, int bucket, int state) { - long seed = input.randomSeed(); - seed = 31 * seed + bucket; - seed = 31 * seed + state; - long slot = - input.time().withZoneSameInstant(zoneId).toInstant().toEpochMilli() - / (samplingIntervalMinutes * 60_000L); - return 31 * seed + slot; - } + private long deriveSeed(PowerValueSource.MarkovIdentifier input, int bucket, int state) { + long seed = input.randomSeed(); + seed = 31 * seed + bucket; + seed = 31 * seed + state; + long slot = + input.time().withZoneSameInstant(zoneId).toInstant().toEpochMilli() + / (samplingIntervalMinutes * 60_000L); + return 31 * seed + slot; + } - private ComparableQuantity scale(double normalizedValue) { - if (!maxPowerFromModel.isGreaterThan(minPowerFromModel)) { - throw new IllegalStateException( - "Markov model normalization has non-positive range (max <= min)."); - } - double clamped = clamp01(normalizedValue); - ComparableQuantity range = maxPowerFromModel.subtract(minPowerFromModel); - return minPowerFromModel.add(range.multiply(clamped)).asType(Power.class); + private ComparableQuantity scale(double normalizedValue) { + if (!maxPowerFromModel.isGreaterThan(minPowerFromModel)) { + throw new IllegalStateException( + "Markov model normalization has non-positive range (max <= min)."); } + double clamped = clamp01(normalizedValue); + ComparableQuantity range = maxPowerFromModel.subtract(minPowerFromModel); + return minPowerFromModel.add(range.multiply(clamped)).asType(Power.class); + } - private double clamp01(double value) { - if (value < 0d) return 0d; - if (value > 1d) return 1d; - return value; - } + private double clamp01(double value) { + if (value < 0d) return 0d; + if (value > 1d) return 1d; + return value; } public record Generator(String name, String version, Map config) {} @@ -480,7 +442,7 @@ public String toString() { } public record GmmBuckets(List buckets) { - public record GmmBucket(List> states) {} + public record GmmBucket(List states) {} public record GmmState(List weights, List means, List variances) {} } diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovPowerProfile.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovPowerProfile.java deleted file mode 100644 index 343bcbb56e..0000000000 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovPowerProfile.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * © 2025. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation -*/ -package edu.ie3.datamodel.models.profile.markov; - -import edu.ie3.datamodel.models.profile.PowerProfile; -import edu.ie3.datamodel.models.profile.PowerProfileKey; -import java.util.Objects; - -/** Simple {@link PowerProfile} implementation for Markov-based load models. */ -public record MarkovPowerProfile(PowerProfileKey key) implements PowerProfile { - - public MarkovPowerProfile { - Objects.requireNonNull(key, "key"); - if (key.noKeyAssigned) { - throw new IllegalArgumentException("Profile key must not be blank."); - } - } - - public MarkovPowerProfile(String key) { - this(buildKey(key)); - } - - private static PowerProfileKey buildKey(String key) { - if (key == null || key.isBlank()) { - throw new IllegalArgumentException("Profile key must not be blank."); - } - return new PowerProfileKey(key); - } - - @Override - public PowerProfileKey getKey() { - return key; - } -} diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy index cd873c33ed..356bfc1ada 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy @@ -29,7 +29,7 @@ class MarkovLoadModelFactoryTest extends Specification { model.transitionData().stateCount() == 2 model.transitionData().values()[0][0][1] == 0.9d model.gmmBuckets().isPresent() - def gmmState = model.gmmBuckets().get().buckets().first().states().first().get() + def gmmState = model.gmmBuckets().get().buckets().first().states().first() gmmState.weights() == [0.6d] gmmState.means() == [1.0d] gmmState.variances() == [0.2d] diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy index 423c1af23b..df102a87c8 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy @@ -44,6 +44,40 @@ class JsonDataSourceTest extends Specification { content == """{"key":42}""" } + def "readTree parses JSON file into a JsonNode tree"() { + given: + def file = tempDir.resolve("data.json") + Files.writeString(file, """{"answer":42}""") + + when: + def node = dataSource.readTree(Path.of("data")) + + then: + node.get("answer").asInt() == 42 + } + + def "readTree throws SourceException when file does not exist"() { + when: + dataSource.readTree(Path.of("nonexistent")) + + then: + thrown(SourceException) + } + + def "getSourceFields returns field names from JSON file"() { + given: + def file = tempDir.resolve("fields.json") + Files.writeString(file, """{"foo":"bar","nested":{"a":1}}""") + + when: + def fields = dataSource.getSourceFields(Path.of("fields")) + + then: + fields.isPresent() + fields.get().contains("foo") + fields.get().contains("nested.a") + } + def "tabular access methods are unsupported"() { when: dataSource.getSourceFields(Entity) From 671d5ed471e584dddbd8a227861f7d6ede78c819 Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 2 Mar 2026 19:03:57 +0100 Subject: [PATCH 23/39] conversation and some added tests --- .../source/json/JsonMarkovProfileSource.java | 2 +- .../profile/markov/MarkovLoadModel.java | 2 +- .../markov/MarkovLoadModelFactoryTest.groovy | 60 +++++++++++++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java index a532df00a7..1ca609f764 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java @@ -38,7 +38,7 @@ public class JsonMarkovProfileSource extends EntitySource implements PowerValueS private final JsonDataSource dataSource; private final FileLoadProfileMetaInformation metaInformation; private final MarkovLoadModelFactory factory; - private MarkovLoadModel cachedModel; + private volatile MarkovLoadModel cachedModel; public JsonMarkovProfileSource( JsonDataSource dataSource, FileLoadProfileMetaInformation metaInformation) { diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index 07d366c0f4..005be65c69 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -313,7 +313,7 @@ private double[] sanitizeDistribution(int bucket, double[] row) { double sanitizedValue = 0d; if (state < row.length) { double value = row[state]; - if (value > 0d && !Double.isNaN(value) && gmmStates[bucket][state] != null) { + if (value > 0d && gmmStates[bucket][state] != null) { sanitizedValue = value; sum += value; } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy index 356bfc1ada..b40ce64f14 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy @@ -56,6 +56,66 @@ class MarkovLoadModelFactoryTest extends Specification { thrown(FactoryException) } + def "buildModel throws FactoryException when threshold count does not match state count"() { + given: "states=2 requires exactly 1 threshold, but 2 are provided" + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"thresholds_right": [0.5]', '"thresholds_right": [0.3, 0.7]')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + def "buildModel throws FactoryException when schema field is missing"() { + given: + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"schema": "markov.load.v1",', '')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + def "buildModel throws FactoryException when generated_at timestamp is invalid"() { + given: + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"generated_at": "2025-01-01T00:00:00Z"', '"generated_at": "not-a-timestamp"')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + def "buildModel throws FactoryException when bucket_encoding formula is missing"() { + given: + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"bucket_encoding": { "formula": "hour_of_day" }', '"bucket_encoding": {}')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + def "buildModel throws FactoryException when gmms buckets array is missing"() { + given: + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"buckets":', '"not_buckets":')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + private static String validModelJson() { return """ { From d7e0c079ceab90733fbdc1593d6d4834f192d6cc Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 2 Mar 2026 19:14:20 +0100 Subject: [PATCH 24/39] rm volatile --- .../ie3/datamodel/io/source/json/JsonMarkovProfileSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java index 1ca609f764..a532df00a7 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java @@ -38,7 +38,7 @@ public class JsonMarkovProfileSource extends EntitySource implements PowerValueS private final JsonDataSource dataSource; private final FileLoadProfileMetaInformation metaInformation; private final MarkovLoadModelFactory factory; - private volatile MarkovLoadModel cachedModel; + private MarkovLoadModel cachedModel; public JsonMarkovProfileSource( JsonDataSource dataSource, FileLoadProfileMetaInformation metaInformation) { From 82e96d3b1751c22fb4360ef633f73b4afa810eca Mon Sep 17 00:00:00 2001 From: Philipp Date: Wed, 25 Mar 2026 16:24:11 +0100 Subject: [PATCH 25/39] Fix compilation errors in Markov factory and source after validate refactor Adapt MarkovLoadModelFactory and JsonMarkovProfileSource to the API changes introduced in 5e01ecfe1 (validate moved from Factory to DataSource): - Use correct generic bound in getFields override (Class) - Add static import for CollectionUtils.newSet - Call DataSource.validate instead of factory.validate --- .../datamodel/io/factory/markov/MarkovLoadModelFactory.java | 4 +++- .../ie3/datamodel/io/source/json/JsonMarkovProfileSource.java | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java index 4441a8a4b6..d9ccc072e1 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -5,6 +5,8 @@ */ package edu.ie3.datamodel.io.factory.markov; +import static edu.ie3.datamodel.utils.CollectionUtils.newSet; + import edu.ie3.datamodel.io.factory.Factory; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.*; @@ -60,7 +62,7 @@ protected MarkovLoadModel buildModel(MarkovModelData data) { } @Override - protected List> getFields(Class entityClass) { + protected List> getFields(Class entityClass) { Set requiredFields = newSet( "schema", diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java index a532df00a7..08243dfc1a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java @@ -13,6 +13,7 @@ import edu.ie3.datamodel.io.factory.markov.MarkovModelData; import edu.ie3.datamodel.io.file.FileType; import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation; +import edu.ie3.datamodel.io.source.DataSource; import edu.ie3.datamodel.io.source.EntitySource; import edu.ie3.datamodel.io.source.PowerValueSource; import edu.ie3.datamodel.models.profile.PowerProfileKey; @@ -83,7 +84,7 @@ public void validate() throws ValidationException { try { Set fields = dataSource.getSourceFields(metaInformation.getFullFilePath()).orElse(Set.of()); - factory.validate(fields, MarkovLoadModel.class).getOrThrow(); + DataSource.validate(fields, MarkovLoadModel.class).getOrThrow(); } catch (SourceException e) { throw new FailedValidationException( "Unable to read Markov model '" From f5ee7ca94021df51519be345495e35742d0c4ad8 Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 30 Mar 2026 13:01:35 +0200 Subject: [PATCH 26/39] fixed validation and added test --- .../io/naming/FieldNamingStrategy.java | 25 +++++++++++ .../ie3/datamodel/io/naming/ModelFields.java | 43 +++++++++++++++++- .../io/source/json/JsonDataSource.java | 3 ++ .../io/naming/ModelFieldsTest.groovy | 44 +++++++++++++++++++ 4 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 src/test/groovy/edu/ie3/datamodel/io/naming/ModelFieldsTest.groovy diff --git a/src/main/java/edu/ie3/datamodel/io/naming/FieldNamingStrategy.java b/src/main/java/edu/ie3/datamodel/io/naming/FieldNamingStrategy.java index 1a469a3725..a00d1c880b 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/FieldNamingStrategy.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/FieldNamingStrategy.java @@ -216,4 +216,29 @@ public class FieldNamingStrategy { public static final String LINE = "line"; public static final String PATH_LINE_STRING = "path"; public static final String POINT = "point"; + + // markov - top-level + public static final String MARKOV_SCHEMA = "schema"; + public static final String MARKOV_GENERATED_AT = "generatedAt"; + public static final String MARKOV_GENERATOR = "generator"; + public static final String MARKOV_TIME_MODEL = "timeModel"; + public static final String MARKOV_VALUE_MODEL = "valueModel"; + public static final String MARKOV_PARAMETERS = "parameters"; + public static final String MARKOV_DATA = "data"; + + // markov - nested fields required for simulation + public static final String MARKOV_GENERATOR_NAME = "generator.name"; + public static final String MARKOV_GENERATOR_VERSION = "generator.version"; + public static final String MARKOV_BUCKET_COUNT = "timeModel.bucketCount"; + public static final String MARKOV_SAMPLING_INTERVAL = "timeModel.samplingIntervalMinutes"; + public static final String MARKOV_TIMEZONE = "timeModel.timezone"; + public static final String MARKOV_DISCRETIZATION_STATES = "valueModel.discretization.states"; + public static final String MARKOV_DISCRETIZATION_THRESHOLDS = + "valueModel.discretization.thresholdsRight"; + public static final String MARKOV_MAX_POWER_VALUE = "valueModel.normalization.maxPower.value"; + public static final String MARKOV_MAX_POWER_UNIT = "valueModel.normalization.maxPower.unit"; + public static final String MARKOV_MIN_POWER_VALUE = "valueModel.normalization.minPower.value"; + public static final String MARKOV_MIN_POWER_UNIT = "valueModel.normalization.minPower.unit"; + public static final String MARKOV_TRANSITION_VALUES = "data.transitions.values"; + public static final String MARKOV_GMM_BUCKETS = "data.gmms.buckets"; } diff --git a/src/main/java/edu/ie3/datamodel/io/naming/ModelFields.java b/src/main/java/edu/ie3/datamodel/io/naming/ModelFields.java index 8cd21edbd3..09dae5fc8a 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/ModelFields.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/ModelFields.java @@ -29,6 +29,7 @@ import edu.ie3.datamodel.models.input.thermal.DomesticHotWaterStorageInput; import edu.ie3.datamodel.models.input.thermal.ThermalBusInput; import edu.ie3.datamodel.models.input.thermal.ThermalHouseInput; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; import edu.ie3.datamodel.models.result.CongestionResult; import edu.ie3.datamodel.models.result.NodeResult; import edu.ie3.datamodel.models.result.connector.LineResult; @@ -62,6 +63,7 @@ private ModelFields() { private static final Map, Set> unsupportedFields = new HashMap<>(); private static final Map, List>> valueMandatoryFields = new HashMap<>(); + private static final Map, List>> genericMandatoryFields = new HashMap<>(); /** * Retrieves a list that contains combinations of mandatory fields for the provided class. @@ -75,7 +77,7 @@ public static List> getMandatoryFields(Class clazz) { } else if (Value.class.isAssignableFrom(clazz)) { return valueMandatoryFields.getOrDefault(clazz, Collections.emptyList()); } else { - return Collections.emptyList(); + return genericMandatoryFields.getOrDefault(clazz, Collections.emptyList()); } } @@ -134,6 +136,20 @@ public static void registerValue( ModelFields.valueMandatoryFields.putIfAbsent(entityClass, List.of(mandatoryFields)); } + /** + * Method to register mandatory fields for a class that is neither an {@link Entity} nor a {@link + * Value}. + * + *

NOTE: This method will only add fields, if no fields are registered yet! + * + * @param clazz for which fields should be registered + * @param mandatoryFields the mandatory field combinations to register + */ + @SafeVarargs + public static void registerGeneric(Class clazz, Set... mandatoryFields) { + ModelFields.genericMandatoryFields.putIfAbsent(clazz, List.of(mandatoryFields)); + } + /** * Method to register mandatory fields for a given entity class. * @@ -195,6 +211,31 @@ public static void registerUnsupported(Class entityClass, Set unsuppo // registering em fields registerMandatory(EmInput.class, assetFields, CONTROLLING_EM, CONTROL_STRATEGY); registerOptional(EmInput.class, assetOptionalFields); + + // markov load model fields + registerGeneric( + MarkovLoadModel.class, + newSet( + MARKOV_SCHEMA, + MARKOV_GENERATED_AT, + MARKOV_GENERATOR, + MARKOV_TIME_MODEL, + MARKOV_VALUE_MODEL, + MARKOV_PARAMETERS, + MARKOV_DATA, + MARKOV_GENERATOR_NAME, + MARKOV_GENERATOR_VERSION, + MARKOV_BUCKET_COUNT, + MARKOV_SAMPLING_INTERVAL, + MARKOV_TIMEZONE, + MARKOV_DISCRETIZATION_STATES, + MARKOV_DISCRETIZATION_THRESHOLDS, + MARKOV_MAX_POWER_VALUE, + MARKOV_MAX_POWER_UNIT, + MARKOV_MIN_POWER_VALUE, + MARKOV_MIN_POWER_UNIT, + MARKOV_TRANSITION_VALUES, + MARKOV_GMM_BUCKETS)); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java index f82ad74caf..af7d7c0404 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java @@ -111,6 +111,9 @@ private static void collectFields(String prefix, JsonNode node, Set coll return; } if (node.isObject()) { + if (!prefix.isEmpty()) { + collector.add(prefix); + } node.propertyNames() .forEach(name -> collectFields(join(prefix, name), node.get(name), collector)); } else if (!prefix.isEmpty()) { diff --git a/src/test/groovy/edu/ie3/datamodel/io/naming/ModelFieldsTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/naming/ModelFieldsTest.groovy new file mode 100644 index 0000000000..9c8463186a --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/naming/ModelFieldsTest.groovy @@ -0,0 +1,44 @@ +/* + * © 2026. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.naming + +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel +import spock.lang.Specification + +class ModelFieldsTest extends Specification { + + def "getMandatoryFields returns registered fields for MarkovLoadModel"() { + when: + def fields = ModelFields.getMandatoryFields(MarkovLoadModel) + + then: + fields.size() == 1 + fields[0].containsAll([ + // top-level + FieldNamingStrategy.MARKOV_SCHEMA, + FieldNamingStrategy.MARKOV_GENERATED_AT, + FieldNamingStrategy.MARKOV_GENERATOR, + FieldNamingStrategy.MARKOV_TIME_MODEL, + FieldNamingStrategy.MARKOV_VALUE_MODEL, + FieldNamingStrategy.MARKOV_PARAMETERS, + FieldNamingStrategy.MARKOV_DATA, + // nested - required for simulation + FieldNamingStrategy.MARKOV_GENERATOR_NAME, + FieldNamingStrategy.MARKOV_GENERATOR_VERSION, + FieldNamingStrategy.MARKOV_BUCKET_COUNT, + FieldNamingStrategy.MARKOV_SAMPLING_INTERVAL, + FieldNamingStrategy.MARKOV_TIMEZONE, + FieldNamingStrategy.MARKOV_DISCRETIZATION_STATES, + FieldNamingStrategy.MARKOV_DISCRETIZATION_THRESHOLDS, + FieldNamingStrategy.MARKOV_MAX_POWER_VALUE, + FieldNamingStrategy.MARKOV_MAX_POWER_UNIT, + FieldNamingStrategy.MARKOV_MIN_POWER_VALUE, + FieldNamingStrategy.MARKOV_MIN_POWER_UNIT, + FieldNamingStrategy.MARKOV_TRANSITION_VALUES, + FieldNamingStrategy.MARKOV_GMM_BUCKETS + ]) + } +} From fc0e6afa498b05af1103d5accc612d0e77140576 Mon Sep 17 00:00:00 2001 From: Philipp Date: Thu, 2 Apr 2026 04:58:00 +0200 Subject: [PATCH 27/39] refactored and ordered --- .../profile/markov/MarkovLoadModel.java | 297 +++++++++++------- 1 file changed, 192 insertions(+), 105 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index 005be65c69..8dafa244c0 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -32,10 +32,14 @@ */ public class MarkovLoadModel { + // Constants + private static final int QUARTER_HOURS_PER_DAY = 96; private static final int WEEKEND_FACTOR = QUARTER_HOURS_PER_DAY; private static final int MONTH_FACTOR = QUARTER_HOURS_PER_DAY * 2; + // Model data + private final String schema; private final ZonedDateTime generatedAt; private final Generator generator; @@ -45,6 +49,8 @@ public class MarkovLoadModel { private final TransitionData transitionData; private final Optional gmmBuckets; + // Derived runtime fields + private final ZoneId zoneId; private final double[][][] transitions; private final int bucketCount; @@ -55,6 +61,10 @@ public class MarkovLoadModel { private final ComparableQuantity maxPowerFromModel; private final ComparableQuantity minPowerFromModel; + // ===================================================================================== + // Constructor + // ===================================================================================== + public MarkovLoadModel( String schema, ZonedDateTime generatedAt, @@ -100,8 +110,72 @@ public MarkovLoadModel( .map(this::convertPowerReference) .orElseThrow( () -> new IllegalArgumentException("Markov model lacks normalization.min_power")); + + if (!this.maxPowerFromModel.isGreaterThan(this.minPowerFromModel)) { + throw new IllegalArgumentException( + "Markov model normalization has non-positive range (max <= min)."); + } + + validateTransitionDimensions(); } + private void validateTransitionDimensions() { + if (transitions.length != bucketCount) { + throw new IllegalArgumentException( + "Transition bucket count mismatch. Expected " + + bucketCount + + " but was " + + transitions.length); + } + for (int bucket = 0; bucket < transitions.length; bucket++) { + if (transitions[bucket].length != stateCount) { + throw new IllegalArgumentException( + "Transition state count mismatch in bucket " + + bucket + + ". Expected " + + stateCount + + " but was " + + transitions[bucket].length); + } + } + } + + // Constructor helpers + + private GmmStateData[][] buildGmmStates(GmmBuckets buckets) { + List bucketList = buckets.buckets(); + if (bucketList.size() != bucketCount) { + throw new IllegalArgumentException( + "GMM bucket count mismatch. Expected " + bucketCount + " but was " + bucketList.size()); + } + GmmStateData[][] lookup = new GmmStateData[bucketCount][stateCount]; + for (int bucket = 0; bucket < bucketCount; bucket++) { + List states = bucketList.get(bucket).states(); + if (states.size() != stateCount) { + throw new IllegalArgumentException( + "State count mismatch in bucket " + bucket + ". Expected " + stateCount); + } + for (int state = 0; state < stateCount; state++) { + GmmBuckets.GmmState s = states.get(state); + lookup[bucket][state] = s != null ? s.toStateData() : null; + } + } + return lookup; + } + + private ComparableQuantity convertPowerReference( + ValueModel.Normalization.PowerReference reference) { + if (!"kW".equalsIgnoreCase(reference.unit())) { + throw new IllegalArgumentException( + "Unsupported reference power unit '" + reference.unit() + "'. Only kW is supported."); + } + return Quantities.getQuantity(reference.value(), StandardUnits.ACTIVE_POWER_IN); + } + + // ===================================================================================== + // Accessors + // ===================================================================================== + public String schema() { return schema; } @@ -134,6 +208,10 @@ public Optional gmmBuckets() { return gmmBuckets; } + // ===================================================================================== + // Public API - Simulation + // ===================================================================================== + /** * Returns a supplier for a single Markov step. * @@ -145,15 +223,6 @@ public Supplier getValueSupplier( return () -> computeStep(data); } - private PowerValueSource.MarkovOutputValue computeStep(PowerValueSource.MarkovIdentifier data) { - int bucket = bucketId(data.time()); - int currentState = resolveState(data); - SplittableRandom rng = new SplittableRandom(deriveSeed(data, bucket, currentState)); - StepResult step = simulateStep(bucket, currentState, rng); - ComparableQuantity power = scale(step.normalizedValue()); - return new PowerValueSource.MarkovOutputValue(Optional.of(new PValue(power)), step.nextState()); - } - /** Convenience helper to compute a single step immediately. */ public PowerValueSource.MarkovOutputValue getPower(PowerValueSource.MarkovIdentifier data) { return getValueSupplier(data).get(); @@ -174,89 +243,26 @@ public Optional> getProfileEnergyScaling() { return Optional.empty(); } - private GmmStateData[][] buildGmmStates(GmmBuckets buckets) { - List bucketList = buckets.buckets(); - if (bucketList.size() != bucketCount) { - throw new IllegalArgumentException( - "GMM bucket count mismatch. Expected " + bucketCount + " but was " + bucketList.size()); - } - GmmStateData[][] lookup = new GmmStateData[bucketCount][stateCount]; - for (int bucket = 0; bucket < bucketCount; bucket++) { - List states = bucketList.get(bucket).states(); - if (states.size() != stateCount) { - throw new IllegalArgumentException( - "State count mismatch in bucket " + bucket + ". Expected " + stateCount); - } - for (int state = 0; state < stateCount; state++) { - GmmBuckets.GmmState s = states.get(state); - lookup[bucket][state] = s != null ? GmmStateData.from(s) : null; - } - } - return lookup; - } + // ===================================================================================== + // Simulation pipeline + // computeStep + // 1. bucketId => isWeekend + // 2. resolveState => discretize + // 3. deriveSeed + // 4. simulateStep => sanitizeDistribution => drawState => sampleNormalizedValue + // 5. scale + // ===================================================================================== - private ComparableQuantity convertPowerReference( - ValueModel.Normalization.PowerReference reference) { - if (!"kW".equalsIgnoreCase(reference.unit())) { - throw new IllegalArgumentException( - "Unsupported reference power unit '" + reference.unit() + "'. Only kW is supported."); - } - return Quantities.getQuantity(reference.value(), StandardUnits.ACTIVE_POWER_IN); + private PowerValueSource.MarkovOutputValue computeStep(PowerValueSource.MarkovIdentifier data) { + int bucket = bucketId(data.time()); + int currentState = resolveState(data); + SplittableRandom rng = new SplittableRandom(deriveSeed(data, bucket, currentState)); + StepResult step = simulateStep(bucket, currentState, rng); + ComparableQuantity power = scale(step.normalizedValue()); + return new PowerValueSource.MarkovOutputValue(Optional.of(new PValue(power)), step.nextState()); } - private record StepResult(int nextState, double normalizedValue) {} - - private static final class GmmStateData { - private final double[] weights; - private final double[] means; - private final double[] variances; - - private GmmStateData(double[] weights, double[] means, double[] variances) { - this.weights = weights; - this.means = means; - this.variances = variances; - } - - private static GmmStateData from(GmmBuckets.GmmState state) { - return new GmmStateData( - toArray(state.weights()), toArray(state.means()), toArray(state.variances())); - } - - private static double[] toArray(List values) { - double[] array = new double[values.size()]; - for (int i = 0; i < values.size(); i++) { - array[i] = values.get(i); - } - return array; - } - - private double sample(SplittableRandom rng) { - int component = drawComponent(rng.nextDouble()); - double mean = means[component]; - double variance = Math.max(0d, variances[component]); - if (variance == 0d) { - return mean; - } - return mean + Math.sqrt(variance) * nextGaussian(rng); - } - - private int drawComponent(double sample) { - double cumulative = 0d; - for (int i = 0; i < weights.length; i++) { - cumulative += weights[i]; - if (sample <= cumulative) { - return i; - } - } - return weights.length - 1; - } - - private double nextGaussian(SplittableRandom rng) { - double u1 = Math.max(Double.MIN_VALUE, rng.nextDouble()); - double u2 = rng.nextDouble(); - return Math.sqrt(-2.0d * Math.log(u1)) * Math.cos(2.0d * Math.PI * u2); - } - } + // Step 1: Determine time bucket private int bucketId(ZonedDateTime time) { ZonedDateTime zoned = time.withZoneSameInstant(zoneId); @@ -272,6 +278,8 @@ private boolean isWeekend(ZonedDateTime time) { return day == DayOfWeek.SATURDAY || day == DayOfWeek.SUNDAY; } + // Step 2: Resolve current state + private int resolveState(PowerValueSource.MarkovIdentifier input) { if (input.previousState().isPresent()) { int state = input.previousState().getAsInt(); @@ -294,6 +302,20 @@ private int discretize(double normalized) { return discretizationThresholds.length; } + // Step 3: Derive deterministic seed + + private long deriveSeed(PowerValueSource.MarkovIdentifier input, int bucket, int state) { + long seed = input.randomSeed(); + seed = 31 * seed + bucket; + seed = 31 * seed + state; + long slot = + input.time().withZoneSameInstant(zoneId).toInstant().toEpochMilli() + / (samplingIntervalMinutes * 60_000L); + return 31 * seed + slot; + } + + // Step 4: Simulate transition and sample value + private StepResult simulateStep(int bucket, int currentState, SplittableRandom rng) { double[] row = transitions[bucket][currentState]; double[] distribution = sanitizeDistribution(bucket, row); @@ -349,32 +371,80 @@ private double sampleNormalizedValue(int bucket, int state, SplittableRandom rng return clamp01(gmm.sample(rng)); } - private long deriveSeed(PowerValueSource.MarkovIdentifier input, int bucket, int state) { - long seed = input.randomSeed(); - seed = 31 * seed + bucket; - seed = 31 * seed + state; - long slot = - input.time().withZoneSameInstant(zoneId).toInstant().toEpochMilli() - / (samplingIntervalMinutes * 60_000L); - return 31 * seed + slot; - } + // Step 5: Scale normalized value to power private ComparableQuantity scale(double normalizedValue) { - if (!maxPowerFromModel.isGreaterThan(minPowerFromModel)) { - throw new IllegalStateException( - "Markov model normalization has non-positive range (max <= min)."); - } - double clamped = clamp01(normalizedValue); ComparableQuantity range = maxPowerFromModel.subtract(minPowerFromModel); - return minPowerFromModel.add(range.multiply(clamped)).asType(Power.class); + return minPowerFromModel.add(range.multiply(normalizedValue)).asType(Power.class); } + // Utility + private double clamp01(double value) { if (value < 0d) return 0d; if (value > 1d) return 1d; return value; } + // ===================================================================================== + // Inner helper types + // ===================================================================================== + + private record StepResult(int nextState, double normalizedValue) {} + + /** + * Runtime representation of a single GMM state, using primitive arrays for efficient sampling. + * Converted from {@link GmmBuckets.GmmState} during construction. + */ + private static final class GmmStateData { + private final double[] weights; + private final double[] means; + private final double[] variances; + + private GmmStateData(double[] weights, double[] means, double[] variances) { + this.weights = weights; + this.means = means; + this.variances = variances; + } + + /** + * Draws a random value from this state's Gaussian Mixture Model. First, a mixture component is + * selected (weighted by {@code weights}), then a value is sampled from that component's normal + * distribution. + */ + private double sample(SplittableRandom rng) { + int component = drawComponent(rng.nextDouble()); + double mean = means[component]; + double variance = Math.max(0d, variances[component]); + if (variance == 0d) { + return mean; + } + return mean + Math.sqrt(variance) * nextGaussian(rng); + } + + private int drawComponent(double sample) { + double cumulative = 0d; + for (int i = 0; i < weights.length; i++) { + cumulative += weights[i]; + if (sample <= cumulative) { + return i; + } + } + return weights.length - 1; + } + + /** Box-Muller transform to generate a standard normal random variate. */ + private double nextGaussian(SplittableRandom rng) { + double u1 = Math.max(Double.MIN_VALUE, rng.nextDouble()); + double u2 = rng.nextDouble(); + return Math.sqrt(-2.0d * Math.log(u1)) * Math.cos(2.0d * Math.PI * u2); + } + } + + // ===================================================================================== + // Data records (JSON structure) + // ===================================================================================== + public record Generator(String name, String version, Map config) {} public record TimeModel( @@ -444,9 +514,26 @@ public String toString() { public record GmmBuckets(List buckets) { public record GmmBucket(List states) {} - public record GmmState(List weights, List means, List variances) {} + public record GmmState(List weights, List means, List variances) { + + private GmmStateData toStateData() { + return new GmmStateData(toArray(weights), toArray(means), toArray(variances)); + } + + private static double[] toArray(List values) { + double[] array = new double[values.size()]; + for (int i = 0; i < values.size(); i++) { + array[i] = values.get(i); + } + return array; + } + } } + // ===================================================================================== + // equals / hashCode / toString + // ===================================================================================== + @Override public boolean equals(Object o) { if (this == o) return true; From f8620eaf5452dd468ea9c7200806a288c0d0d98d Mon Sep 17 00:00:00 2001 From: Philipp Date: Wed, 6 May 2026 04:07:46 +0200 Subject: [PATCH 28/39] slim down + doku --- CHANGELOG.md | 1 + .../io/factory/markov/MarkovModelData.java | 14 -- .../markov/MarkovModelParsingSupport.java | 77 +++----- .../profile/markov/MarkovLoadModel.java | 180 +++++++++++++----- 4 files changed, 155 insertions(+), 117 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b95c272c5c..e9cfe590fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased/Snapshot] ### Added +- Added support for Markov-chain-based load profiles loaded from JSON [#1472](https://github.com/ie3-institute/PowerSystemDataModel/issues/1472) ### Fixed diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java index de649ead19..bf7e5dff03 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java @@ -23,18 +23,4 @@ public MarkovModelData(JsonNode root) { public JsonNode getRoot() { return root; } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof MarkovModelData that)) return false; - return Objects.equals(getTargetClass(), that.getTargetClass()) - && Objects.equals(getFieldsToValues(), that.getFieldsToValues()) - && Objects.equals(root, that.root); - } - - @Override - public int hashCode() { - return Objects.hash(getTargetClass(), getFieldsToValues(), root); - } } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java index 16a3f5fa52..11f1bd60b7 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -21,7 +21,7 @@ interface MarkovModelParsingSupport { default Generator parseGenerator(JsonNode generatorNode) { String name = extractText(generatorNode, "name"); String version = extractText(generatorNode, "version"); - Map config = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + Map config = new LinkedHashMap<>(); JsonNode configNode = generatorNode.path("config"); if (configNode.isObject()) { for (Map.Entry entry : configNode.properties()) { @@ -230,63 +230,36 @@ default void validateTransitionShape( } default double[][][] parseTransitionValues(JsonNode valuesNode, int buckets, int stateCount) { - if (!valuesNode.isArray()) { - throw new FactoryException("Transition values must be a three dimensional array"); - } - if (valuesNode.size() != buckets) { + if (!valuesNode.isArray() || valuesNode.size() != buckets) { throw new FactoryException( - "Transition values provided " + valuesNode.size() + " buckets. Expected " + buckets); + "Transition values must be a three dimensional array with " + buckets + " buckets"); } double[][][] values = new double[buckets][stateCount][stateCount]; - int bucketIndex = 0; - for (JsonNode bucketNode : valuesNode) { - fillBucket(values, bucketNode, bucketIndex, stateCount); - bucketIndex++; - } - return values; - } - - default void fillBucket( - double[][][] values, JsonNode bucketNode, int bucketIndex, int stateCount) { - int rowIndex = 0; - for (JsonNode rowNode : bucketNode) { - fillRow(values, rowNode, bucketIndex, rowIndex, stateCount); - rowIndex++; - } - if (rowIndex != stateCount) { - throw new FactoryException( - "Bucket " + bucketIndex + " contained " + rowIndex + " rows. Expected " + stateCount); - } - } - - default void fillRow( - double[][][] values, JsonNode rowNode, int bucketIndex, int rowIndex, int stateCount) { - if (rowIndex >= stateCount) { - throw new FactoryException("Too many rows in transition matrix for bucket " + bucketIndex); - } - int columnIndex = 0; - for (JsonNode probNode : rowNode) { - if (columnIndex >= stateCount) { + for (int b = 0; b < buckets; b++) { + JsonNode bucketNode = valuesNode.get(b); + if (bucketNode.size() != stateCount) { throw new FactoryException( - "Too many columns in transition matrix for bucket " - + bucketIndex - + ", row " - + rowIndex); + "Bucket " + b + " contained " + bucketNode.size() + " rows. Expected " + stateCount); + } + for (int r = 0; r < stateCount; r++) { + JsonNode rowNode = bucketNode.get(r); + if (rowNode.size() != stateCount) { + throw new FactoryException( + "Row " + + r + + " in bucket " + + b + + " had " + + rowNode.size() + + " columns. Expected " + + stateCount); + } + for (int c = 0; c < stateCount; c++) { + values[b][r][c] = rowNode.get(c).asDouble(); + } } - values[bucketIndex][rowIndex][columnIndex] = probNode.asDouble(); - columnIndex++; - } - if (columnIndex != stateCount) { - throw new FactoryException( - "Row " - + rowIndex - + " in bucket " - + bucketIndex - + " had " - + columnIndex - + " columns. Expected " - + stateCount); } + return values; } default List readDoubleArray(JsonNode node, String field) { diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index 8dafa244c0..231bad7c0a 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -65,6 +65,14 @@ public class MarkovLoadModel { // Constructor // ===================================================================================== + /** + * Builds a model from the JSON-derived sections, deriving runtime caches (transition arrays, + * discretization thresholds, per-state GMM data) and validating overall consistency. + * + * @throws IllegalArgumentException if GMM data is missing, the normalization range is + * non-positive, or the transition tensor does not match {@code bucketCount * stateCount * + * stateCount} + */ public MarkovLoadModel( String schema, ZonedDateTime generatedAt, @@ -119,6 +127,7 @@ public MarkovLoadModel( validateTransitionDimensions(); } + /** Verifies that the parsed transition tensor matches the declared bucket and state counts. */ private void validateTransitionDimensions() { if (transitions.length != bucketCount) { throw new IllegalArgumentException( @@ -142,6 +151,11 @@ private void validateTransitionDimensions() { // Constructor helpers + /** + * Converts the JSON-shaped {@link GmmBuckets} (lists of doubles) into a runtime {@code + * GmmStateData[][]} backed by primitive arrays for fast sampling. States with no GMM data are + * preserved as {@code null} entries; {@link #sanitizeDistribution} routes around them. + */ private GmmStateData[][] buildGmmStates(GmmBuckets buckets) { List bucketList = buckets.buckets(); if (bucketList.size() != bucketCount) { @@ -163,6 +177,7 @@ private GmmStateData[][] buildGmmStates(GmmBuckets buckets) { return lookup; } + /** Converts a JSON power reference into a typed quantity. Only {@code "kW"} is accepted. */ private ComparableQuantity convertPowerReference( ValueModel.Normalization.PowerReference reference) { if (!"kW".equalsIgnoreCase(reference.unit())) { @@ -238,7 +253,7 @@ public Optional> getMaxPower() { return Optional.of(maxPowerFromModel); } - /** Markov models do not define an energy scaling factor. */ + /** Not implemented yet. Future feature */ public Optional> getProfileEnergyScaling() { return Optional.empty(); } @@ -253,6 +268,7 @@ public Optional> getProfileEnergyScaling() { // 5. scale // ===================================================================================== + /** Runs the full five-step simulation pipeline and packages the result for the caller. */ private PowerValueSource.MarkovOutputValue computeStep(PowerValueSource.MarkovIdentifier data) { int bucket = bucketId(data.time()); int currentState = resolveState(data); @@ -264,6 +280,11 @@ private PowerValueSource.MarkovOutputValue computeStep(PowerValueSource.MarkovId // Step 1: Determine time bucket + /** + * Maps a timestamp into one of {@code bucketCount} buckets, encoded as {@code month * + * MONTH_FACTOR + weekendFlag * WEEKEND_FACTOR + quarterHour}. Each bucket has its own transition + * matrix and per-state GMM, so the result drives every subsequent lookup. + */ private int bucketId(ZonedDateTime time) { ZonedDateTime zoned = time.withZoneSameInstant(zoneId); int month = zoned.getMonthValue() - 1; @@ -273,6 +294,7 @@ private int bucketId(ZonedDateTime time) { month * MONTH_FACTOR + weekendFlag * WEEKEND_FACTOR + quarterHour, bucketCount); } + /** True for Saturdays and Sundays. */ private boolean isWeekend(ZonedDateTime time) { DayOfWeek day = time.getDayOfWeek(); return day == DayOfWeek.SATURDAY || day == DayOfWeek.SUNDAY; @@ -280,6 +302,13 @@ private boolean isWeekend(ZonedDateTime time) { // Step 2: Resolve current state + /** + * Resolves the Markov state used as the starting point of this step. If the caller supplied a + * previous state, it is used directly; otherwise the supplied {@code initialNormalizedValue} is + * discretized via {@link #discretize}. + * + * @throws IllegalArgumentException if a supplied previous state is out of bounds + */ private int resolveState(PowerValueSource.MarkovIdentifier input) { if (input.previousState().isPresent()) { int state = input.previousState().getAsInt(); @@ -292,8 +321,13 @@ private int resolveState(PowerValueSource.MarkovIdentifier input) { return discretize(normalized); } + /** + * Returns the state-bin index for a normalized value, using the right-edges in {@code + * discretizationThresholds}. The input is clamped to {@code [0, 1]} to absorb minor + * floating-point drift on caller-supplied values. + */ private int discretize(double normalized) { - double value = clamp01(normalized); + double value = Math.clamp(normalized, 0d, 1d); for (int i = 0; i < discretizationThresholds.length; i++) { if (value <= discretizationThresholds[i]) { return i; @@ -304,6 +338,11 @@ private int discretize(double normalized) { // Step 3: Derive deterministic seed + /** + * Produces a deterministic RNG seed for one simulation step by mixing the request seed with the + * bucket, state and time slot. Identical inputs always yield the same output, which is required + * for reproducibility across simulation runs. + */ private long deriveSeed(PowerValueSource.MarkovIdentifier input, int bucket, int state) { long seed = input.randomSeed(); seed = 31 * seed + bucket; @@ -316,19 +355,28 @@ private long deriveSeed(PowerValueSource.MarkovIdentifier input, int bucket, int // Step 4: Simulate transition and sample value + /** + * Performs one Markov step: sanitize and renormalize the transition row, draw a next state, then + * sample its GMM. If no usable transitions remain (empty row, or every reachable state has no GMM + * data), the model stays in {@code currentState} and emits a normalized value of {@code 0}. + */ private StepResult simulateStep(int bucket, int currentState, SplittableRandom rng) { double[] row = transitions[bucket][currentState]; double[] distribution = sanitizeDistribution(bucket, row); if (distribution.length == 0) { return new StepResult(currentState, 0d); } - int nextStateIndex = drawState(distribution, rng); + int nextStateIndex = drawWeighted(distribution, rng); double normalized = sampleNormalizedValue(bucket, nextStateIndex, rng); return new StepResult(nextStateIndex, normalized); } + /** + * Filters the raw transition row so the chain never transitions into a state that lacks GMM data, + * then renormalizes the remaining probabilities. Returns an empty array if no usable mass + * remains. + */ private double[] sanitizeDistribution(int bucket, double[] row) { - // Ignore invalid probabilities and states without GMM data, then renormalize. double[] sanitized = new double[stateCount]; double sum = 0d; for (int state = 0; state < stateCount; state++) { @@ -351,41 +399,48 @@ private double[] sanitizeDistribution(int bucket, double[] row) { return sanitized; } - private int drawState(double[] distribution, SplittableRandom rng) { + /** + * Picks an index by weighted random sampling. The {@code weights} array must be non-negative and + * sum to (approximately) one; used both to draw the next Markov state and to draw a GMM + * component. + */ + private static int drawWeighted(double[] weights, SplittableRandom rng) { double sample = rng.nextDouble(); double cumulative = 0d; - for (int i = 0; i < distribution.length; i++) { - cumulative += distribution[i]; + for (int i = 0; i < weights.length; i++) { + cumulative += weights[i]; if (sample <= cumulative) { return i; } } - return distribution.length - 1; + return weights.length - 1; } + /** + * Draws a normalized power value from the GMM at {@code (bucket, state)}. Returns {@code 0} if + * that cell has no GMM data. The result is clamped to {@code [0, 1]} because Gaussian tails are + * unbounded; without clamping, {@link #scale} could return values outside {@code [minPower, + * maxPower]}. + */ private double sampleNormalizedValue(int bucket, int state, SplittableRandom rng) { GmmStateData gmm = gmmStates[bucket][state]; if (gmm == null) { return 0d; } - return clamp01(gmm.sample(rng)); + return Math.clamp(gmm.sample(rng), 0d, 1d); } // Step 5: Scale normalized value to power + /** + * Maps a normalized {@code [0, 1]} value into actual power via {@code minPower + value * + * (maxPower - minPower)}. + */ private ComparableQuantity scale(double normalizedValue) { ComparableQuantity range = maxPowerFromModel.subtract(minPowerFromModel); return minPowerFromModel.add(range.multiply(normalizedValue)).asType(Power.class); } - // Utility - - private double clamp01(double value) { - if (value < 0d) return 0d; - if (value > 1d) return 1d; - return value; - } - // ===================================================================================== // Inner helper types // ===================================================================================== @@ -413,31 +468,13 @@ private GmmStateData(double[] weights, double[] means, double[] variances) { * distribution. */ private double sample(SplittableRandom rng) { - int component = drawComponent(rng.nextDouble()); + int component = drawWeighted(weights, rng); double mean = means[component]; double variance = Math.max(0d, variances[component]); if (variance == 0d) { return mean; } - return mean + Math.sqrt(variance) * nextGaussian(rng); - } - - private int drawComponent(double sample) { - double cumulative = 0d; - for (int i = 0; i < weights.length; i++) { - cumulative += weights[i]; - if (sample <= cumulative) { - return i; - } - } - return weights.length - 1; - } - - /** Box-Muller transform to generate a standard normal random variate. */ - private double nextGaussian(SplittableRandom rng) { - double u1 = Math.max(Double.MIN_VALUE, rng.nextDouble()); - double u2 = rng.nextDouble(); - return Math.sqrt(-2.0d * Math.log(u1)) * Math.cos(2.0d * Math.PI * u2); + return mean + Math.sqrt(variance) * rng.nextGaussian(); } } @@ -445,39 +482,82 @@ private double nextGaussian(SplittableRandom rng) { // Data records (JSON structure) // ===================================================================================== + /** + * Provenance metadata for the trained model: trainer name, version (e.g. a git SHA) and the + * free-form configuration that produced this JSON. + */ public record Generator(String name, String version, Map config) {} + /** + * Temporal layout of the model: number of buckets, the (documentation-only) encoding formula used + * by the trainer, the sampling interval in minutes, and the IANA timezone the buckets are defined + * in. + */ public record TimeModel( int bucketCount, String bucketEncodingFormula, int samplingIntervalMinutes, String timezone) {} + /** Value-space configuration: physical unit, normalization range and state discretization. */ public record ValueModel( String valueUnit, Normalization normalization, Discretization discretization) { + /** + * Normalization range used to map between normalized {@code [0, 1]} values and physical power. + * Both {@code maxPower} and {@code minPower} are required at construction time; the optional + * wrapping only reflects the JSON shape. + */ public record Normalization( String method, Optional maxPower, Optional minPower) { + /** A value/unit pair from the JSON normalization block. Only {@code "kW"} is supported. */ public record PowerReference(double value, String unit) {} } + /** + * State-bin layout: the number of states and the right-edge thresholds that separate + * neighbouring states. The list contains {@code states - 1} entries. + */ public record Discretization(int states, List thresholdsRight) {} } + /** Optional metadata describing how the trainer produced the transitions and GMMs. */ public record Parameters(TransitionParameters transitions, GmmParameters gmm) { + /** + * Strategy used by the trainer when a transition row had no observations (e.g. {@code + * "self_loop"}). Carried through for traceability; not used at simulation time. + */ public record TransitionParameters(String emptyRowStrategy) {} + /** + * GMM-fitting metadata. + * + * @param valueColumn name of the column the trainer fit on + * @param verbose scikit-learn verbosity used during fitting + * @param heartbeatSeconds stuck-process watchdog interval used by the simonaMarkovLoad Python + * trainer: when set, the trainer enables {@code faulthandler.dump_traceback_later} and + * dumps a stack trace every N seconds so hangs during GMM fitting can be diagnosed. Carried + * through for JSON round-trip only; unused at simulation time. + */ public record GmmParameters( String valueColumn, Optional verbose, Optional heartbeatSeconds) {} } + /** + * The 3D transition tensor {@code values[bucket][currentState][nextState] = probability}, plus + * the dtype and encoding metadata from the JSON. {@link #equals} and {@link #hashCode} are + * overridden so two records with equal array contents compare equal; the synthetic record + * versions would otherwise compare arrays by identity. + */ public record TransitionData(String dtype, String encoding, double[][][] values) { + /** Number of time buckets, i.e. the outer dimension of {@link #values()}. */ public int bucketCount() { return values.length; } + /** Number of states, i.e. the inner dimension of {@link #values()}. */ public int stateCount() { return values.length == 0 ? 0 : values[0].length; } @@ -495,25 +575,23 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(dtype, encoding, Arrays.deepHashCode(values)); } - - @Override - public String toString() { - return "TransitionData{" - + "dtype='" - + dtype - + '\'' - + ", encoding='" - + encoding - + '\'' - + ", values=" - + Arrays.deepToString(values) - + '}'; - } } + /** + * Per-bucket Gaussian Mixture Models. The outer list has {@code bucketCount} entries (one per + * time bucket); each {@link GmmBucket} has {@code stateCount} {@link GmmState} entries (one per + * state). A {@link GmmState} may be {@code null} when no observations were available for that + * (bucket, state) pair. + */ public record GmmBuckets(List buckets) { + /** GMM data for one time bucket, indexed by state. Entries may be {@code null}. */ public record GmmBucket(List states) {} + /** + * Parameters of one Gaussian Mixture Model: per-component {@code weights}, {@code means} and + * {@code variances}. Each component is a 1-D Gaussian; the trainer chooses 1-3 components per + * (bucket, state) cell using BIC. + */ public record GmmState(List weights, List means, List variances) { private GmmStateData toStateData() { From 7edc06cb488e187a823d621d9601553b47f97bbf Mon Sep 17 00:00:00 2001 From: Philipp Date: Wed, 6 May 2026 04:37:44 +0200 Subject: [PATCH 29/39] sonar --- .../io/factory/markov/MarkovModelData.java | 14 ++++++++++++++ .../profile/markov/MarkovLoadModel.java | 19 +++++++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java index bf7e5dff03..de649ead19 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java @@ -23,4 +23,18 @@ public MarkovModelData(JsonNode root) { public JsonNode getRoot() { return root; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof MarkovModelData that)) return false; + return Objects.equals(getTargetClass(), that.getTargetClass()) + && Objects.equals(getFieldsToValues(), that.getFieldsToValues()) + && Objects.equals(root, that.root); + } + + @Override + public int hashCode() { + return Objects.hash(getTargetClass(), getFieldsToValues(), root); + } } diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index 231bad7c0a..b5e08d3934 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -565,16 +565,27 @@ public int stateCount() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof TransitionData other)) return false; - return Objects.equals(dtype, other.dtype) - && Objects.equals(encoding, other.encoding) - && Arrays.deepEquals(values, other.values); + return o instanceof TransitionData(String d, String e, double[][][] v) + && Objects.equals(dtype, d) + && Objects.equals(encoding, e) + && Arrays.deepEquals(values, v); } @Override public int hashCode() { return Objects.hash(dtype, encoding, Arrays.deepHashCode(values)); } + + @Override + public String toString() { + return "TransitionData[dtype=" + + dtype + + ", encoding=" + + encoding + + ", values=" + + Arrays.deepToString(values) + + "]"; + } } /** From fb4f1d221c636a6e761a89f8c819ff9156b90e9c Mon Sep 17 00:00:00 2001 From: Philipp Date: Wed, 6 May 2026 12:25:31 +0200 Subject: [PATCH 30/39] added todos and remove getFields --- .../markov/MarkovLoadModelFactory.java | 26 ------------------- .../profile/markov/MarkovLoadModel.java | 3 +++ 2 files changed, 3 insertions(+), 26 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java index d9ccc072e1..1be17dd81f 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -5,15 +5,11 @@ */ package edu.ie3.datamodel.io.factory.markov; -import static edu.ie3.datamodel.utils.CollectionUtils.newSet; - import edu.ie3.datamodel.io.factory.Factory; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.*; import java.time.ZonedDateTime; -import java.util.List; import java.util.Optional; -import java.util.Set; import tools.jackson.databind.JsonNode; /** @@ -60,26 +56,4 @@ protected MarkovLoadModel buildModel(MarkovModelData data) { transitionData, Optional.of(gmmBuckets)); } - - @Override - protected List> getFields(Class entityClass) { - Set requiredFields = - newSet( - "schema", - "generatedAt", - "generator.name", - "generator.version", - "timeModel.bucketCount", - "timeModel.bucketEncoding.formula", - "timeModel.samplingIntervalMinutes", - "timeModel.timezone", - "valueModel.valueUnit", - "valueModel.normalization.method", - "valueModel.discretization.states", - "valueModel.discretization.thresholdsRight", - "data.transitions.shape", - "data.transitions.values", - "data.gmms.buckets"); - return List.of(requiredFields); - } } diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index b5e08d3934..7dd0e11f1e 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -362,6 +362,7 @@ private long deriveSeed(PowerValueSource.MarkovIdentifier input, int bucket, int */ private StepResult simulateStep(int bucket, int currentState, SplittableRandom rng) { double[] row = transitions[bucket][currentState]; + // TODO: Remove distribution if possible double[] distribution = sanitizeDistribution(bucket, row); if (distribution.length == 0) { return new StepResult(currentState, 0d); @@ -519,6 +520,7 @@ public record PowerReference(double value, String unit) {} * State-bin layout: the number of states and the right-edge thresholds that separate * neighbouring states. The list contains {@code states - 1} entries. */ + // TODO: Replace Autoboxing of List public record Discretization(int states, List thresholdsRight) {} } @@ -541,6 +543,7 @@ public record TransitionParameters(String emptyRowStrategy) {} * dumps a stack trace every N seconds so hangs during GMM fitting can be diagnosed. Carried * through for JSON round-trip only; unused at simulation time. */ + // TODO: Remove boxing from Optional public record GmmParameters( String valueColumn, Optional verbose, Optional heartbeatSeconds) {} } From 7e7d9e1caec92f9b4f759230f9a09b4f972526e2 Mon Sep 17 00:00:00 2001 From: Philipp Date: Thu, 7 May 2026 05:54:40 +0200 Subject: [PATCH 31/39] TODOs --- .../markov/MarkovModelParsingSupport.java | 6 +-- .../profile/markov/MarkovLoadModel.java | 48 +++---------------- .../json/JsonMarkovProfileSourceTest.groovy | 14 ++++-- .../profile/markov/MarkovLoadModelTest.groovy | 44 ----------------- 4 files changed, 19 insertions(+), 93 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java index 11f1bd60b7..3fdd26413b 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -195,10 +195,10 @@ default ZonedDateTime parseTimestamp(String timestamp) { } } - default Optional optionalInt(JsonNode node, String field) { + default OptionalInt optionalInt(JsonNode node, String field) { JsonNode value = node.get(field); - if (value == null || value.isNull()) return Optional.empty(); - return Optional.of(value.asInt()); + if (value == null || value.isNull()) return OptionalInt.empty(); + return OptionalInt.of(value.asInt()); } default int[] parseTransitionShape(JsonNode transitionsNode) { diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index 7dd0e11f1e..f613bf3bd8 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -16,6 +16,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalInt; import java.util.SplittableRandom; import java.util.function.Supplier; import javax.measure.quantity.Energy; @@ -356,50 +357,17 @@ private long deriveSeed(PowerValueSource.MarkovIdentifier input, int bucket, int // Step 4: Simulate transition and sample value /** - * Performs one Markov step: sanitize and renormalize the transition row, draw a next state, then - * sample its GMM. If no usable transitions remain (empty row, or every reachable state has no GMM - * data), the model stays in {@code currentState} and emits a normalized value of {@code 0}. + * Performs one Markov step: draw the next state from the transition row, then sample its GMM. The + * simonaMarkovLoad trainer guarantees that every state with non-zero incoming transition + * probability also has GMM data (empty rows fall back to a self-loop on the current state), so no + * per-step row sanitization is needed. */ private StepResult simulateStep(int bucket, int currentState, SplittableRandom rng) { - double[] row = transitions[bucket][currentState]; - // TODO: Remove distribution if possible - double[] distribution = sanitizeDistribution(bucket, row); - if (distribution.length == 0) { - return new StepResult(currentState, 0d); - } - int nextStateIndex = drawWeighted(distribution, rng); + int nextStateIndex = drawWeighted(transitions[bucket][currentState], rng); double normalized = sampleNormalizedValue(bucket, nextStateIndex, rng); return new StepResult(nextStateIndex, normalized); } - /** - * Filters the raw transition row so the chain never transitions into a state that lacks GMM data, - * then renormalizes the remaining probabilities. Returns an empty array if no usable mass - * remains. - */ - private double[] sanitizeDistribution(int bucket, double[] row) { - double[] sanitized = new double[stateCount]; - double sum = 0d; - for (int state = 0; state < stateCount; state++) { - double sanitizedValue = 0d; - if (state < row.length) { - double value = row[state]; - if (value > 0d && gmmStates[bucket][state] != null) { - sanitizedValue = value; - sum += value; - } - } - sanitized[state] = sanitizedValue; - } - if (sum <= 0d) { - return new double[0]; - } - for (int i = 0; i < sanitized.length; i++) { - sanitized[i] /= sum; - } - return sanitized; - } - /** * Picks an index by weighted random sampling. The {@code weights} array must be non-negative and * sum to (approximately) one; used both to draw the next Markov state and to draw a GMM @@ -520,7 +488,6 @@ public record PowerReference(double value, String unit) {} * State-bin layout: the number of states and the right-edge thresholds that separate * neighbouring states. The list contains {@code states - 1} entries. */ - // TODO: Replace Autoboxing of List public record Discretization(int states, List thresholdsRight) {} } @@ -543,9 +510,8 @@ public record TransitionParameters(String emptyRowStrategy) {} * dumps a stack trace every N seconds so hangs during GMM fitting can be diagnosed. Carried * through for JSON round-trip only; unused at simulation time. */ - // TODO: Remove boxing from Optional public record GmmParameters( - String valueColumn, Optional verbose, Optional heartbeatSeconds) {} + String valueColumn, OptionalInt verbose, OptionalInt heartbeatSeconds) {} } /** diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy index 4dfdc6f497..9622f2cad4 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy @@ -97,8 +97,8 @@ class JsonMarkovProfileSourceTest extends Specification { source.getMaxPower().isPresent() source.getMaxPower().get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 10d output.value().isPresent() - output.nextState() == 0 - output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 10d + output.nextState() == 1 + output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 5.25d } private static String validModelJson() { @@ -154,11 +154,15 @@ class JsonMarkovProfileSourceTest extends Specification { { "states": [ { - "weights": [0.6], + "weights": [1.0], "means": [1.0], - "variances": [0.0] + "variances": [0.0] }, - null + { + "weights": [1.0], + "means": [0.5], + "variances": [0.0] + } ] } ] diff --git a/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy b/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy index 6d310d1757..61509fa922 100644 --- a/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy @@ -82,26 +82,6 @@ class MarkovLoadModelTest extends Specification { output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 2.6d } - def "supplier returns zero power when transitions row has no usable probabilities"() { - given: - def model = loadModel(emptyTransitions(), missingStateGmms()) - def input = new PowerValueSource.MarkovIdentifier( - ZonedDateTime.parse("2025-01-01T00:00:00Z"), - OptionalInt.of(0), - OptionalDouble.empty(), - 7L - ) - - when: - def supplier = model.getValueSupplier(input) - def output = supplier.get() - - then: - output.nextState() == 0 // stays in current state - output.value().isPresent() - output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 1d - } - private loadModel(String transitions, String states) { def json = modelJson(transitions, states) def root = objectMapper.readTree(json) @@ -147,30 +127,6 @@ class MarkovLoadModelTest extends Specification { """.stripIndent() } - private static String emptyTransitions() { - return """ - [ - [ - [0.0, 0.0], - [0.0, 0.0] - ] - ] - """.stripIndent() - } - - private static String missingStateGmms() { - return """ - [ - { - "weights": [1.0], - "means": [0.4], - "variances": [0.0] - }, - null - ] - """.stripIndent() - } - private static String modelJson(String transitions, String states) { def normalization = """ { From 298ca367222f7cf0afe14bf47cf7044d71b17aed Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 18 May 2026 21:56:12 +0200 Subject: [PATCH 32/39] readTheDocs added for Branch --- docs/readthedocs/io/basiciousage.md | 1 + docs/readthedocs/io/csvfiles.md | 4 + docs/readthedocs/io/markov.md | 264 ++++++++++++++++++ .../models/input/participant/load.md | 4 + 4 files changed, 273 insertions(+) create mode 100644 docs/readthedocs/io/markov.md diff --git a/docs/readthedocs/io/basiciousage.md b/docs/readthedocs/io/basiciousage.md index a1c0cf2b4d..beab9600c3 100644 --- a/docs/readthedocs/io/basiciousage.md +++ b/docs/readthedocs/io/basiciousage.md @@ -10,6 +10,7 @@ maxdepth: 2 --- naming csvfiles +markov sql influxdb ValidationUtils.md diff --git a/docs/readthedocs/io/csvfiles.md b/docs/readthedocs/io/csvfiles.md index d3ea5de530..6b08debae6 100644 --- a/docs/readthedocs/io/csvfiles.md +++ b/docs/readthedocs/io/csvfiles.md @@ -180,6 +180,10 @@ Markov-based load models (`markov_*`) do not support energy scaling. Calls to * - random - A random load proile based on: ``Kays - Agent-based simulation environment for improving the planning of distribution grids`` - Permissible head line: ``kSa,kSu,kWd,mySa,mySu,myWd,sigmaSa,sigmaSu,sigmaWd,quarterHour`` + * - markov + - Markov-chain-based stochastic load profiles from JSON files (``markov_.json``). + See [Markov-based Load Profiles](markov-load-model) for the full JSON schema documentation. + - Not a CSV format. Uses JSON with schema ``simonaMarkovLoad:psdm:1.0``. ``` diff --git a/docs/readthedocs/io/markov.md b/docs/readthedocs/io/markov.md new file mode 100644 index 0000000000..6e83a784e3 --- /dev/null +++ b/docs/readthedocs/io/markov.md @@ -0,0 +1,264 @@ +(markov-load-model)= + +# Markov-based Load Profiles + +Markov-based load profiles allow stochastic load simulation using Markov chains combined with Gaussian Mixture Models +(GMMs). The models are produced by an external Python trainer +([simonaMarkovLoad](https://github.com/ie3-institute/simonaMarkovLoad)) and consumed by PSDM via JSON files. + +## Overview + +Each Markov model encodes: + +- A set of **discrete states** representing different load levels +- **Transition matrices** per time bucket that describe the probability of moving from one state to another +- **GMM parameters** per state and bucket that allow sampling a continuous normalized power value within a state + +During simulation, one step works as follows: + +1. **Bucket lookup** - determine the current time bucket from month, weekday/weekend and quarter-hour +2. **State resolution** - use the previous state (or discretize the initial value for the first step) +3. **Seed derivation** - compute a deterministic RNG seed for reproducibility +4. **Transition + sampling** - draw the next state from the transition row, then sample a normalized value from the GMM +5. **Denormalization** - scale the normalized value back to physical power using {code}`minPower` and {code}`maxPower` + +## File Naming + +Markov model files follow the naming convention {code}`markov_.json`, e.g. {code}`markov_h0.json`. +The profile key must be unique across all load-profile sources. Do not use the same key for both a CSV profile +({code}`lpts_.csv`) and a Markov profile ({code}`markov_.json`). + +## JSON Schema + +The JSON file uses the schema identifier {code}`simonaMarkovLoad:psdm:1.0` and has the following top-level structure: + +```text +{ + "schema": "simonaMarkovLoad:psdm:1.0", + "generated_at": "2025-01-15T12:00:00Z", + "generator": { ... }, + "time_model": { ... }, + "value_model": { ... }, + "parameters": { ... }, + "data": { ... } +} +``` + +### generator + +Metadata about the tool that produced the model. + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - name + - String + - Name of the trainer (e.g. {code}`simonaMarkovLoad`) + * - version + - String + - Trainer version + * - config + - Object + - Key-value pairs of the training configuration +``` + +### time_model + +Describes how time is mapped to buckets. + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - bucket_count + - Integer + - Total number of time buckets (typically 2304 = 12 months x 2 day types x 96 quarter-hours) + * - bucket_encoding + - Object + - Contains a {code}`formula` field documenting the bucket index calculation + * - sampling_interval_minutes + - Integer + - Time step length in minutes (typically 15) + * - timezone + - String + - IANA timezone identifier (e.g. {code}`Europe/Berlin`) +``` + +The bucket index is computed as: + +``` +bucket = month * 192 + isWeekend * 96 + quarterHour +``` + +where {code}`isWeekend` is 1 for Saturday/Sunday and 0 otherwise, and {code}`quarterHour` is the 0-based quarter-hour +of the day (0..95). + +### value_model + +Defines the value space and normalization. + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - value_unit + - String + - Physical unit of the original training data (e.g. {code}`W`) + * - normalization + - Object + - See below + * - discretization + - Object + - See below +``` + +#### normalization + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - method + - String + - Normalization strategy used during training (e.g. {code}`minmax_per_series`) + * - max_power + - Object + - Upper bound: {code}`{"value": , "unit": "kW"}` + * - min_power + - Object + - Lower bound: {code}`{"value": , "unit": "kW"}` +``` + +Denormalization in PSDM uses these bounds as: + +``` +power = minPower + normalizedValue * (maxPower - minPower) +``` + +Negative {code}`min_power` values are valid and represent net feed-in (e.g. PV households). + +#### discretization + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - states + - Integer + - Number of discrete Markov states + * - thresholds_right + - Array of Double + - Right-edge thresholds for mapping normalized values to states (length = states - 1) +``` + +### parameters + +Optional metadata about how the trainer produced transitions and GMMs. + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - transitions + - Object + - Contains {code}`empty_row_strategy`: how the trainer handled transition rows with no observations + * - gmm + - Object + - Contains {code}`value_col` (column name), {code}`verbose` (logging level), and + {code}`heartbeat_seconds` (trainer watchdog interval, not used at runtime) +``` + +### data + +Contains the actual model data: transition probabilities and GMM parameters. + +#### data.transitions + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - dtype + - String + - Data type (e.g. {code}`float64`) + * - encoding + - String + - Encoding format (e.g. {code}`dense`) + * - shape + - Array of Integer + - Dimensions: {code}`[bucket_count, states, states]` + * - values + - 3D Array of Double + - Transition probabilities: {code}`values[bucket][fromState][toState]` +``` + +Each row {code}`values[bucket][state]` is a probability distribution over the next states (sums to 1.0). + +#### data.gmms + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - buckets + - Array of Bucket + - One entry per time bucket, each containing a {code}`states` array +``` + +Each bucket contains a {code}`states` array with one entry per Markov state. Each state entry is either {code}`null` +(if no GMM was fitted for that state/bucket combination) or an object with: + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - weights + - Array of Double + - Component weights (sum to 1.0) + * - means + - Array of Double + - Component means in normalized [0, 1] space + * - variances + - Array of Double + - Component variances +``` + +During sampling, a GMM component is drawn according to {code}`weights`, then a value is sampled from the corresponding +Gaussian and clamped to [0, 1] before denormalization. + +## Loading and Validation + +Markov models are loaded via {code}`JsonMarkovProfileSource`, which: + +- Reads the JSON file lazily on first access (not at construction time) +- Caches the parsed {code}`MarkovLoadModel` for subsequent calls +- Validates all 20 mandatory fields via {code}`DataSource.validate()` before use +- Is thread-safe ({code}`synchronized` lazy loading) diff --git a/docs/readthedocs/models/input/participant/load.md b/docs/readthedocs/models/input/participant/load.md index a6c1214fee..54d8496a1d 100644 --- a/docs/readthedocs/models/input/participant/load.md +++ b/docs/readthedocs/models/input/participant/load.md @@ -95,3 +95,7 @@ The profiles rely on the VDN description for interruptable loads. For more details see [here (German only)](https://www.bdew.de/media/documents/LPuVe-Praxisleitfaden.pdf). {code}`NbwTemperatureDependantLoadProfiles` provides sample temperature dependant load profiles that can be used. The `NbwTemperatureDependantLoadProfiles` consists of load profiles "ep1" for heat pumps and "ez2" for night storage heating. + +Markov-chain-based load profiles can be used for stochastic load simulation. These profiles are produced +by an external Python trainer and loaded from JSON files. For details on the JSON schema and the simulation pipeline, +see the [Markov-based Load Profiles](markov-load-model) documentation. From b92bc0eaf6d32cfc75b1a2be3f83e0ad3a35cb8e Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 3 Jul 2026 16:14:14 +0200 Subject: [PATCH 33/39] some improvements --- docs/readthedocs/io/markov.md | 11 ++-- .../markov/MarkovModelParsingSupport.java | 25 +++++---- .../io/source/json/JsonDataSource.java | 13 ++++- .../source/json/JsonMarkovProfileSource.java | 17 ++++-- .../profile/markov/MarkovLoadModel.java | 19 ++++--- .../markov/MarkovLoadModelFactoryTest.groovy | 56 ++++++++++++++++++- .../json/JsonMarkovProfileSourceTest.groovy | 31 +++++++++- .../profile/markov/MarkovLoadModelTest.groovy | 41 +++++++++++++- 8 files changed, 173 insertions(+), 40 deletions(-) diff --git a/docs/readthedocs/io/markov.md b/docs/readthedocs/io/markov.md index 6e83a784e3..6170ed9507 100644 --- a/docs/readthedocs/io/markov.md +++ b/docs/readthedocs/io/markov.md @@ -113,7 +113,7 @@ Defines the value space and normalization. - Description * - value_unit - String - - Physical unit of the original training data (e.g. {code}`W`) + - Unit of the stored values; the trainer exports {code}`normalized` ([0, 1] scale) * - normalization - Object - See below @@ -133,7 +133,7 @@ Defines the value space and normalization. - Description * - method - String - - Normalization strategy used during training (e.g. {code}`minmax_per_series`) + - Normalization strategy used during training (e.g. {code}`minmax_global`) * - max_power - Object - Upper bound: {code}`{"value": , "unit": "kW"}` @@ -164,7 +164,8 @@ Negative {code}`min_power` values are valid and represent net feed-in (e.g. PV h - Number of discrete Markov states * - thresholds_right - Array of Double - - Right-edge thresholds for mapping normalized values to states (length = states - 1) + - Right-edge thresholds for mapping normalized values to states (length = states - 1). Bins are + left-closed and right-open: a value exactly on a threshold maps to the higher state ``` ### parameters @@ -202,10 +203,10 @@ Contains the actual model data: transition probabilities and GMM parameters. - Description * - dtype - String - - Data type (e.g. {code}`float64`) + - Data type (e.g. {code}`float32`) * - encoding - String - - Encoding format (e.g. {code}`dense`) + - Encoding format (e.g. {code}`nested_lists`) * - shape - Array of Integer - Dimensions: {code}`[bucket_count, states, states]` diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java index 3fdd26413b..fd9cb2ca89 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -73,21 +73,22 @@ default ValueModel parseValueModel(JsonNode valueNode) { /** Parses optional parameter blocks (transitions and GMM). */ default Parameters parseParameters(JsonNode parametersNode) { - Parameters.TransitionParameters transitions = - new Parameters.TransitionParameters( - parametersNode.path("transitions").path("empty_row_strategy").asString("")); - if (transitions.emptyRowStrategy().isEmpty()) { - transitions = null; - } + String emptyRowStrategy = + parametersNode.path("transitions").path("empty_row_strategy").asString(""); + Optional transitions = + emptyRowStrategy.isEmpty() + ? Optional.empty() + : Optional.of(new Parameters.TransitionParameters(emptyRowStrategy)); JsonNode gmmNode = parametersNode.path("gmm"); - Parameters.GmmParameters gmm = + Optional gmm = gmmNode.isMissingNode() || gmmNode.isNull() || gmmNode.isEmpty() - ? null - : new Parameters.GmmParameters( - gmmNode.path("value_col").asString(""), - optionalInt(gmmNode, "verbose"), - optionalInt(gmmNode, "heartbeat_seconds")); + ? Optional.empty() + : Optional.of( + new Parameters.GmmParameters( + gmmNode.path("value_col").asString(""), + optionalInt(gmmNode, "verbose"), + optionalInt(gmmNode, "heartbeat_seconds"))); return new Parameters(transitions, gmm); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java index af7d7c0404..f10a4c44ac 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java @@ -89,7 +89,18 @@ public Stream> getSourceData(Class entityC @Override public Optional> getSourceFields(Path filePath) throws SourceException { JsonNode root = readTree(filePath); - return Optional.of(collectFieldNames(root)); + return Optional.of(fieldNames(root)); + } + + /** + * Returns the set of (nested, dot-separated) field names present in the given JSON tree. Allows + * callers that already hold a parsed tree to collect field names without re-reading the file. + * + * @param root the parsed JSON tree + * @return the field names + */ + public static Set fieldNames(JsonNode root) { + return collectFieldNames(root); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java index 08243dfc1a..3727b6bd9b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java @@ -40,6 +40,7 @@ public class JsonMarkovProfileSource extends EntitySource implements PowerValueS private final FileLoadProfileMetaInformation metaInformation; private final MarkovLoadModelFactory factory; private MarkovLoadModel cachedModel; + private JsonNode cachedRoot; public JsonMarkovProfileSource( JsonDataSource dataSource, FileLoadProfileMetaInformation metaInformation) { @@ -65,9 +66,8 @@ public JsonMarkovProfileSource( */ public synchronized MarkovLoadModel getModel() throws SourceException { if (cachedModel == null) { - JsonNode root = dataSource.readTree(metaInformation.getFullFilePath()); try { - cachedModel = factory.get(new MarkovModelData(root)).getOrThrow(); + cachedModel = factory.get(new MarkovModelData(readRoot())).getOrThrow(); } catch (FactoryException e) { throw new SourceException( "Unable to build Markov load model from '" @@ -75,15 +75,24 @@ public synchronized MarkovLoadModel getModel() throws SourceException { + "'.", e); } + // the parsed model supersedes the raw tree, release it for garbage collection + cachedRoot = null; } return cachedModel; } + /** Returns the parsed JSON tree, reading the underlying file only once. */ + private synchronized JsonNode readRoot() throws SourceException { + if (cachedRoot == null) { + cachedRoot = dataSource.readTree(metaInformation.getFullFilePath()); + } + return cachedRoot; + } + @Override public void validate() throws ValidationException { try { - Set fields = - dataSource.getSourceFields(metaInformation.getFullFilePath()).orElse(Set.of()); + Set fields = JsonDataSource.fieldNames(readRoot()); DataSource.validate(fields, MarkovLoadModel.class).getOrThrow(); } catch (SourceException e) { throw new FailedValidationException( diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index f613bf3bd8..60330307e7 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -155,7 +155,7 @@ private void validateTransitionDimensions() { /** * Converts the JSON-shaped {@link GmmBuckets} (lists of doubles) into a runtime {@code * GmmStateData[][]} backed by primitive arrays for fast sampling. States with no GMM data are - * preserved as {@code null} entries; {@link #sanitizeDistribution} routes around them. + * preserved as {@code null} entries; {@link #sampleNormalizedValue} returns {@code 0} for them. */ private GmmStateData[][] buildGmmStates(GmmBuckets buckets) { List bucketList = buckets.buckets(); @@ -265,7 +265,7 @@ public Optional> getProfileEnergyScaling() { // 1. bucketId => isWeekend // 2. resolveState => discretize // 3. deriveSeed - // 4. simulateStep => sanitizeDistribution => drawState => sampleNormalizedValue + // 4. simulateStep => drawWeighted => sampleNormalizedValue // 5. scale // ===================================================================================== @@ -324,13 +324,15 @@ private int resolveState(PowerValueSource.MarkovIdentifier input) { /** * Returns the state-bin index for a normalized value, using the right-edges in {@code - * discretizationThresholds}. The input is clamped to {@code [0, 1]} to absorb minor - * floating-point drift on caller-supplied values. + * discretizationThresholds}. Bins are left-closed and right-open, so a value exactly on a + * threshold belongs to the upper state - matching the trainer's {@code + * searchsorted(side="right")} assignment used when the model was fitted. The input is clamped to + * {@code [0, 1]} to absorb minor floating-point drift on caller-supplied values. */ private int discretize(double normalized) { double value = Math.clamp(normalized, 0d, 1d); for (int i = 0; i < discretizationThresholds.length; i++) { - if (value <= discretizationThresholds[i]) { + if (value < discretizationThresholds[i]) { return i; } } @@ -348,9 +350,7 @@ private long deriveSeed(PowerValueSource.MarkovIdentifier input, int bucket, int long seed = input.randomSeed(); seed = 31 * seed + bucket; seed = 31 * seed + state; - long slot = - input.time().withZoneSameInstant(zoneId).toInstant().toEpochMilli() - / (samplingIntervalMinutes * 60_000L); + long slot = input.time().toInstant().toEpochMilli() / (samplingIntervalMinutes * 60_000L); return 31 * seed + slot; } @@ -492,7 +492,8 @@ public record Discretization(int states, List thresholdsRight) {} } /** Optional metadata describing how the trainer produced the transitions and GMMs. */ - public record Parameters(TransitionParameters transitions, GmmParameters gmm) { + public record Parameters( + Optional transitions, Optional gmm) { /** * Strategy used by the trainer when a transition row had no observations (e.g. {@code diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy index b40ce64f14..31543f9f76 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy @@ -8,6 +8,7 @@ package edu.ie3.datamodel.io.factory.markov import edu.ie3.datamodel.exceptions.FactoryException import spock.lang.Specification import tools.jackson.databind.ObjectMapper +import tools.jackson.databind.node.ObjectNode class MarkovLoadModelFactoryTest extends Specification { private final ObjectMapper objectMapper = new ObjectMapper() @@ -116,6 +117,55 @@ class MarkovLoadModelFactoryTest extends Specification { thrown(FactoryException) } + def "buildModel tolerates a missing parameters block"() { + given: + def root = objectMapper.readTree(validModelJson()) + ((ObjectNode) root).remove("parameters") + + when: + def model = factory.get(new MarkovModelData(root)).getOrThrow() + + then: + model.parameters().transitions().isEmpty() + model.parameters().gmm().isEmpty() + } + + def "buildModel throws FactoryException when max_power is missing"() { + given: + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"max_power": { "value": 1.5, "unit": "kW" },', '')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + def "buildModel throws FactoryException when normalization range is non-positive"() { + given: "max_power below min_power" + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"max_power": { "value": 1.5, "unit": "kW" }', '"max_power": { "value": 0.05, "unit": "kW" }')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + def "buildModel throws FactoryException on unsupported power unit"() { + given: + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"max_power": { "value": 1.5, "unit": "kW" }', '"max_power": { "value": 1500.0, "unit": "W" }')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + private static String validModelJson() { return """ { @@ -133,7 +183,7 @@ class MarkovLoadModelFactoryTest extends Specification { "timezone": "UTC" }, "value_model": { - "value_unit": "W", + "value_unit": "normalized", "normalization": { "method": "none", "max_power": { "value": 1.5, "unit": "kW" }, @@ -154,8 +204,8 @@ class MarkovLoadModelFactoryTest extends Specification { }, "data": { "transitions": { - "dtype": "float64", - "encoding": "dense", + "dtype": "float32", + "encoding": "nested_lists", "shape": [1,2,2], "values": [ [ diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy index 9622f2cad4..ad99653657 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy @@ -6,6 +6,7 @@ package edu.ie3.datamodel.io.source.json import edu.ie3.datamodel.exceptions.SourceException +import edu.ie3.datamodel.io.connectors.JsonFileConnector import edu.ie3.datamodel.io.file.FileType import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation @@ -17,6 +18,8 @@ import spock.lang.Specification import java.nio.file.Files import java.nio.file.Path import java.time.ZonedDateTime +import java.util.concurrent.atomic.AtomicInteger +import java.util.function.Function class JsonMarkovProfileSourceTest extends Specification { @@ -60,6 +63,28 @@ class JsonMarkovProfileSourceTest extends Specification { noExceptionThrown() } + def "validate and getModel read the underlying file only once"() { + given: "a connector that counts how often the file is opened" + Files.writeString(jsonFile, validModelJson()) + def openCount = new AtomicInteger() + def countingStream = { String path -> + openCount.incrementAndGet() + new FileInputStream(path) + } as Function + def source = new JsonMarkovProfileSource( + new JsonDataSource(new JsonFileConnector(tempDir, countingStream), new FileNamingStrategy()), + new FileLoadProfileMetaInformation("profile1", jsonFile, FileType.JSON) + ) + + when: + source.validate() + source.getModel() + source.getModel() + + then: + openCount.get() == 1 + } + def "getModel throws SourceException on invalid JSON file"() { given: Files.writeString(jsonFile, "{}") @@ -118,7 +143,7 @@ class JsonMarkovProfileSourceTest extends Specification { "timezone": "UTC" }, "value_model": { - "value_unit": "W", + "value_unit": "normalized", "normalization": { "method": "none", "max_power": { "value": 10.0, "unit": "kW" }, @@ -139,8 +164,8 @@ class JsonMarkovProfileSourceTest extends Specification { }, "data": { "transitions": { - "dtype": "float64", - "encoding": "dense", + "dtype": "float32", + "encoding": "nested_lists", "shape": [1,2,2], "values": [ [ diff --git a/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy b/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy index 61509fa922..d0eefef4ea 100644 --- a/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy @@ -82,6 +82,41 @@ class MarkovLoadModelTest extends Specification { output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 2.6d } + def "initial normalized value exactly on a threshold maps to the upper state"() { + given: "trainer semantics: searchsorted(side='right') puts boundary values into the upper bin" + def model = loadModel(selfLoopTransitions(), deterministicStates()) + def input = new PowerValueSource.MarkovIdentifier( + ZonedDateTime.parse("2025-01-01T00:00:00Z"), + OptionalInt.empty(), + OptionalDouble.of(0.5d), + 13L + ) + + when: + def output = model.getValueSupplier(input).get() + + then: + output.nextState() == 1 + output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 4.2d + } + + def "supplier rejects an out-of-bounds previous state"() { + given: + def model = loadModel(deterministicTransitions(), deterministicStates()) + def input = new PowerValueSource.MarkovIdentifier( + ZonedDateTime.parse("2025-01-01T00:00:00Z"), + OptionalInt.of(5), + OptionalDouble.empty(), + 1L + ) + + when: + model.getValueSupplier(input).get() + + then: + thrown(IllegalArgumentException) + } + private loadModel(String transitions, String states) { def json = modelJson(transitions, states) def root = objectMapper.readTree(json) @@ -151,7 +186,7 @@ class MarkovLoadModelTest extends Specification { "timezone": "UTC" }, "value_model": { - "value_unit": "W", + "value_unit": "normalized", "normalization": $normalization, "discretization": { "states": 2, @@ -168,8 +203,8 @@ class MarkovLoadModelTest extends Specification { }, "data": { "transitions": { - "dtype": "float64", - "encoding": "dense", + "dtype": "float32", + "encoding": "nested_lists", "shape": [1,2,2], "values": $transitions }, From b556b59fcc3b41840c3d096dd9eda68641a222fa Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 6 Jul 2026 17:43:46 +0200 Subject: [PATCH 34/39] additions to docu --- docs/readthedocs/io/csvfiles.md | 6 +-- docs/readthedocs/io/markov.md | 43 ++++++++++++++++--- .../models/input/participant/load.md | 2 +- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/docs/readthedocs/io/csvfiles.md b/docs/readthedocs/io/csvfiles.md index df553ac556..e7dfefa745 100644 --- a/docs/readthedocs/io/csvfiles.md +++ b/docs/readthedocs/io/csvfiles.md @@ -138,7 +138,7 @@ The following keys are supported until now: - Active, reactive and heat power. Permissible head line: ``time,p,q,h`` * - v - - Voltage mangnitude in pu and angle in °. + - Voltage magnitude in pu and angle in °. Permissible head line: ``time,vMag,vAng`` * - weather - Weather information. @@ -174,11 +174,11 @@ Markov-based load models (`markov_*`) do not support energy scaling. Calls to - BDEW standard load profiles 2025 ([source](https://www.bdew.de/energie/standardlastprofile-strom/)) - Permissible head line: ``janSa,janSu,janWd,febSa,febSu,febWd,marSa,marSu,marWd,aprSa,aprSu,aprWd,maySa,maySu,mayWd,junSa,junSu,junWd,julSa,julSu,julWd,augSa,augSu,augWd,sepSa,sepSu,sepWd,octSa,octSu,octWd,novSa,novSu,novWd,decSa,decSu,decWd,quarterHour`` * - random - - A random load proile based on: ``Kays - Agent-based simulation environment for improving the planning of distribution grids`` + - A random load profile based on: ``Kays - Agent-based simulation environment for improving the planning of distribution grids`` - Permissible head line: ``kSa,kSu,kWd,mySa,mySu,myWd,sigmaSa,sigmaSu,sigmaWd,quarterHour`` * - markov - Markov-chain-based stochastic load profiles from JSON files (``markov_.json``). - See [Markov-based Load Profiles](markov-load-model) for the full JSON schema documentation. + See [Markov-based Load Profiles](#markov-load-model) for the full JSON schema documentation. - Not a CSV format. Uses JSON with schema ``simonaMarkovLoad:psdm:1.0``. ``` diff --git a/docs/readthedocs/io/markov.md b/docs/readthedocs/io/markov.md index 6170ed9507..d77f739835 100644 --- a/docs/readthedocs/io/markov.md +++ b/docs/readthedocs/io/markov.md @@ -25,12 +25,14 @@ During simulation, one step works as follows: ## File Naming Markov model files follow the naming convention {code}`markov_.json`, e.g. {code}`markov_h0.json`. +A profile key consists of 1 to 11 letters, optionally followed by up to 3 digits. The profile key must be unique across all load-profile sources. Do not use the same key for both a CSV profile ({code}`lpts_.csv`) and a Markov profile ({code}`markov_.json`). ## JSON Schema -The JSON file uses the schema identifier {code}`simonaMarkovLoad:psdm:1.0` and has the following top-level structure: +The JSON file uses the schema identifier {code}`simonaMarkovLoad:psdm:1.0` and has the following top-level structure. +Note that PSDM only requires the {code}`schema` field to be present; its value is currently not verified. ```text { @@ -97,8 +99,9 @@ The bucket index is computed as: bucket = month * 192 + isWeekend * 96 + quarterHour ``` -where {code}`isWeekend` is 1 for Saturday/Sunday and 0 otherwise, and {code}`quarterHour` is the 0-based quarter-hour -of the day (0..95). +where {code}`month` is the 0-based month (January = 0), {code}`isWeekend` is 1 for Saturday/Sunday and 0 otherwise, +and {code}`quarterHour` is the 0-based quarter-hour of the day (0..95). Timestamps are converted to the model's +timezone before the bucket index is computed. ### value_model @@ -150,6 +153,9 @@ power = minPower + normalizedValue * (maxPower - minPower) Negative {code}`min_power` values are valid and represent net feed-in (e.g. PV households). +Only {code}`kW` is supported as unit for both bounds, other units are rejected when the model is loaded. +Furthermore, {code}`max_power` must be strictly greater than {code}`min_power`. + #### discretization ```{list-table} @@ -170,7 +176,8 @@ Negative {code}`min_power` values are valid and represent net feed-in (e.g. PV h ### parameters -Optional metadata about how the trainer produced transitions and GMMs. +Metadata about how the trainer produced transitions and GMMs. The block itself has to be present to pass +source validation, but it may be empty; all fields within it are optional. ```{list-table} :widths: auto @@ -255,11 +262,37 @@ Each bucket contains a {code}`states` array with one entry per Markov state. Eac During sampling, a GMM component is drawn according to {code}`weights`, then a value is sampled from the corresponding Gaussian and clamped to [0, 1] before denormalization. +If the drawn state has no GMM data ({code}`null`), the sampled normalized value is 0, which denormalizes to +{code}`min_power`. The trainer guarantees that states without GMM data are never reachable with non-zero +transition probability, so this case does not occur. + ## Loading and Validation Markov models are loaded via {code}`JsonMarkovProfileSource`, which: - Reads the JSON file lazily on first access (not at construction time) - Caches the parsed {code}`MarkovLoadModel` for subsequent calls -- Validates all 20 mandatory fields via {code}`DataSource.validate()` before use +- Offers validation of all 20 mandatory fields via {code}`validate()`. Note that this is a separate call: + it is not triggered automatically when the model is loaded - Is thread-safe ({code}`synchronized` lazy loading) + +Markov models do not support energy scaling: {code}`getProfileEnergyScaling()` always returns +{code}`Optional.empty()`. + +## Simulation Input and Output + +Power values are requested via {code}`PowerValueSource.MarkovIdentifier`, which carries: + +- {code}`time` - the timestamp of the requested step +- {code}`previousState` - the state returned by the previous step (used for all subsequent steps) +- {code}`initialNormalizedValue` - a normalized start value that is discretized into the initial state + (used for the first step only) +- {code}`randomSeed` - the base seed for reproducible sampling + +Either {code}`previousState` or {code}`initialNormalizedValue` has to be provided. Each step returns a +{code}`MarkovOutputValue` containing the sampled power value and the {code}`nextState`, which callers pass +into the identifier of the following step. The model itself is stateless, so a single instance can serve +multiple independent chains. + +Sampling is deterministic: the RNG seed of a step is derived from the request seed, the time bucket, the +current state and the time slot, so identical inputs always produce identical results. diff --git a/docs/readthedocs/models/input/participant/load.md b/docs/readthedocs/models/input/participant/load.md index 54d8496a1d..8d940cf92c 100644 --- a/docs/readthedocs/models/input/participant/load.md +++ b/docs/readthedocs/models/input/participant/load.md @@ -98,4 +98,4 @@ The `NbwTemperatureDependantLoadProfiles` consists of load profiles "ep1" for he Markov-chain-based load profiles can be used for stochastic load simulation. These profiles are produced by an external Python trainer and loaded from JSON files. For details on the JSON schema and the simulation pipeline, -see the [Markov-based Load Profiles](markov-load-model) documentation. +see the [Markov-based Load Profiles](#markov-load-model) documentation. From 1a592194a9da9822bf4977df3225e60194244dd5 Mon Sep 17 00:00:00 2001 From: Philipp Date: Wed, 8 Jul 2026 06:33:36 +0200 Subject: [PATCH 35/39] Align validation and docs --- docs/readthedocs/io/csvfiles.md | 2 +- docs/readthedocs/io/markov.md | 4 +- .../models/input/participant/load.md | 16 +- .../markov/MarkovLoadModelFactory.java | 13 +- .../markov/MarkovModelParsingSupport.java | 85 ++++-- .../ie3/datamodel/io/naming/ModelFields.java | 1 - .../profile/markov/MarkovLoadModel.java | 248 +++++++----------- .../markov/MarkovLoadModelFactoryTest.groovy | 28 +- .../io/naming/ModelFieldsTest.groovy | 1 - .../json/JsonMarkovProfileSourceTest.groovy | 22 ++ 10 files changed, 221 insertions(+), 199 deletions(-) diff --git a/docs/readthedocs/io/csvfiles.md b/docs/readthedocs/io/csvfiles.md index e7dfefa745..3077c4f3a7 100644 --- a/docs/readthedocs/io/csvfiles.md +++ b/docs/readthedocs/io/csvfiles.md @@ -178,7 +178,7 @@ Markov-based load models (`markov_*`) do not support energy scaling. Calls to - Permissible head line: ``kSa,kSu,kWd,mySa,mySu,myWd,sigmaSa,sigmaSu,sigmaWd,quarterHour`` * - markov - Markov-chain-based stochastic load profiles from JSON files (``markov_.json``). - See [Markov-based Load Profiles](#markov-load-model) for the full JSON schema documentation. + See {ref}`Markov-based Load Profiles ` for the full JSON schema documentation. - Not a CSV format. Uses JSON with schema ``simonaMarkovLoad:psdm:1.0``. ``` diff --git a/docs/readthedocs/io/markov.md b/docs/readthedocs/io/markov.md index d77f739835..2ab58e492d 100644 --- a/docs/readthedocs/io/markov.md +++ b/docs/readthedocs/io/markov.md @@ -176,8 +176,8 @@ Furthermore, {code}`max_power` must be strictly greater than {code}`min_power`. ### parameters -Metadata about how the trainer produced transitions and GMMs. The block itself has to be present to pass -source validation, but it may be empty; all fields within it are optional. +Optional metadata about how the trainer produced transitions and GMMs. The block itself may be missing or empty; +all fields within it are optional. ```{list-table} :widths: auto diff --git a/docs/readthedocs/models/input/participant/load.md b/docs/readthedocs/models/input/participant/load.md index 8d940cf92c..13fba5a2ba 100644 --- a/docs/readthedocs/models/input/participant/load.md +++ b/docs/readthedocs/models/input/participant/load.md @@ -91,11 +91,11 @@ Wasserwirtschaft; engl. Federal Association of the Energy and Water Industry). F [the corresponding website (German only)](https://www.bdew.de/energie/standardlastprofile-strom/). Furthermore, there are {code}`TemperatureDependantLoadProfiles` which can be used to note usage of load profiles for night heating storages or heat pumps for example. -The profiles rely on the VDN description for interruptable loads. -For more details see [here (German only)](https://www.bdew.de/media/documents/LPuVe-Praxisleitfaden.pdf). -{code}`NbwTemperatureDependantLoadProfiles` provides sample temperature dependant load profiles that can be used. -The `NbwTemperatureDependantLoadProfiles` consists of load profiles "ep1" for heat pumps and "ez2" for night storage heating. - -Markov-chain-based load profiles can be used for stochastic load simulation. These profiles are produced -by an external Python trainer and loaded from JSON files. For details on the JSON schema and the simulation pipeline, -see the [Markov-based Load Profiles](#markov-load-model) documentation. +The profiles rely on the VDN description for interruptable loads. +For more details see [here (German only)](https://www.bdew.de/media/documents/LPuVe-Praxisleitfaden.pdf). +{code}`NbwTemperatureDependantLoadProfiles` provides sample temperature dependant load profiles that can be used. +The `NbwTemperatureDependantLoadProfiles` consists of load profiles "ep1" for heat pumps and "ez2" for night storage heating. + +Markov-chain-based load profiles can be used for stochastic load simulation. These profiles are produced +by an external Python trainer and loaded from JSON files. For details on the JSON schema and the simulation pipeline, +see the {ref}`Markov-based Load Profiles ` documentation. diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java index 1be17dd81f..af1675fea6 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -12,12 +12,7 @@ import java.util.Optional; import tools.jackson.databind.JsonNode; -/** - * Factory turning Markov JSON data into {@link MarkovLoadModel}s. - * - *

The JSON fields follow the simonaMarkovLoad schema (snake_case), which is mapped to the model - * records used within PSDM. - */ +/** Factory turning Markov JSON data into {@link MarkovLoadModel}s. */ public class MarkovLoadModelFactory extends Factory implements MarkovModelParsingSupport { @@ -26,11 +21,7 @@ public MarkovLoadModelFactory() { super(MarkovLoadModel.class); } - /** - * Build a {@link MarkovLoadModel} from a parsed JSON tree. - * - *

This method validates the transition shape and requires GMM buckets to be present. - */ + /** Builds a {@link MarkovLoadModel} from a parsed JSON tree. */ @Override protected MarkovLoadModel buildModel(MarkovModelData data) { JsonNode root = data.getRoot(); diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java index fd9cb2ca89..68a3c75838 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -12,10 +12,7 @@ import java.util.*; import tools.jackson.databind.JsonNode; -/** - * Shared parsing helpers for Markov model JSON documents. This is intentionally package-private as - * it is only meant to be reused across factory implementations in this package. - */ +/** Shared parsing helpers for Markov model JSON documents. */ interface MarkovModelParsingSupport { default Generator parseGenerator(JsonNode generatorNode) { @@ -31,19 +28,25 @@ default Generator parseGenerator(JsonNode generatorNode) { return new Generator(name, version, config); } - /** Extracts the time model block, including bucket count and sampling interval. */ + /** Extracts the time model block. */ default TimeModel extractTimeModel(JsonNode timeNode) { int bucketCount = extractInt(timeNode, "bucket_count"); + if (bucketCount <= 0) { + throw new FactoryException("time_model.bucket_count must be positive"); + } String formula = extractNode(timeNode, "bucket_encoding").path("formula").asString(""); if (formula.isEmpty()) { throw new FactoryException("Missing bucket encoding formula"); } int samplingInterval = extractInt(timeNode, "sampling_interval_minutes"); + if (samplingInterval <= 0) { + throw new FactoryException("time_model.sampling_interval_minutes must be positive"); + } String timezone = extractText(timeNode, "timezone"); return new TimeModel(bucketCount, formula, samplingInterval, timezone); } - /** Parses value model settings (unit, normalization, discretization thresholds). */ + /** Parses value model settings. */ default ValueModel parseValueModel(JsonNode valueNode) { String valueUnit = extractText(valueNode, "value_unit"); JsonNode normalizationNode = extractNode(valueNode, "normalization"); @@ -56,6 +59,9 @@ default ValueModel parseValueModel(JsonNode valueNode) { JsonNode discretizationNode = extractNode(valueNode, "discretization"); int states = extractInt(discretizationNode, "states"); + if (states <= 0) { + throw new FactoryException("value_model.discretization.states must be positive"); + } List thresholds = readDoubleArray(discretizationNode, "thresholds_right"); if (thresholds.size() != Math.max(0, states - 1)) { throw new FactoryException( @@ -71,7 +77,7 @@ default ValueModel parseValueModel(JsonNode valueNode) { return new ValueModel(valueUnit, normalization, discretization); } - /** Parses optional parameter blocks (transitions and GMM). */ + /** Parses optional parameter metadata. */ default Parameters parseParameters(JsonNode parametersNode) { String emptyRowStrategy = parametersNode.path("transitions").path("empty_row_strategy").asString(""); @@ -93,11 +99,7 @@ default Parameters parseParameters(JsonNode parametersNode) { return new Parameters(transitions, gmm); } - /** - * Parses the transition matrix section. - * - *

The expected shape is [bucket, state, state]. - */ + /** Parses the transition tensor. */ default TransitionData parseTransitions( JsonNode dataNode, int expectedBucketCount, int stateCount) { JsonNode transitionsNode = extractNode(dataNode, "transitions"); @@ -116,7 +118,7 @@ default TransitionData parseTransitions( return new TransitionData(dtype, encoding, values); } - /** Parses GMM buckets. Individual states may be null, which disables sampling for that state. */ + /** Parses GMM buckets. Individual states may be null. */ default GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { if (gmmsNode == null || gmmsNode.isMissingNode() || gmmsNode.isNull()) { throw new FactoryException("Missing field 'gmms'"); @@ -168,13 +170,7 @@ default String extractText(JsonNode node, String field) { default double extractDouble(JsonNode node, String field) { JsonNode value = node.get(field); - if (value == null || value.isMissingNode() || value.isNull()) { - throw new FactoryException("Missing field '" + field + "'"); - } - if (!value.isNumber()) { - throw new FactoryException("Field '" + field + "' must be a double"); - } - return value.asDouble(); + return extractDoubleValue(value, field); } default int extractInt(JsonNode node, String field) { @@ -199,6 +195,9 @@ default ZonedDateTime parseTimestamp(String timestamp) { default OptionalInt optionalInt(JsonNode node, String field) { JsonNode value = node.get(field); if (value == null || value.isNull()) return OptionalInt.empty(); + if (!value.canConvertToInt()) { + throw new FactoryException("Field '" + field + "' must be an integer"); + } return OptionalInt.of(value.asInt()); } @@ -207,7 +206,11 @@ default int[] parseTransitionShape(JsonNode transitionsNode) { if (!shapeNode.isArray() || shapeNode.size() != 3) { throw new FactoryException("Transition shape must contain three dimensions"); } - return new int[] {shapeNode.get(0).asInt(), shapeNode.get(1).asInt(), shapeNode.get(2).asInt()}; + return new int[] { + extractInt(shapeNode, 0, "Transition shape"), + extractInt(shapeNode, 1, "Transition shape"), + extractInt(shapeNode, 2, "Transition shape") + }; } default void validateTransitionShape( @@ -238,12 +241,19 @@ default double[][][] parseTransitionValues(JsonNode valuesNode, int buckets, int double[][][] values = new double[buckets][stateCount][stateCount]; for (int b = 0; b < buckets; b++) { JsonNode bucketNode = valuesNode.get(b); + if (!bucketNode.isArray()) { + throw new FactoryException("Bucket " + b + " in transition values must be an array"); + } if (bucketNode.size() != stateCount) { throw new FactoryException( "Bucket " + b + " contained " + bucketNode.size() + " rows. Expected " + stateCount); } for (int r = 0; r < stateCount; r++) { JsonNode rowNode = bucketNode.get(r); + if (!rowNode.isArray()) { + throw new FactoryException( + "Row " + r + " in bucket " + b + " of transition values must be an array"); + } if (rowNode.size() != stateCount) { throw new FactoryException( "Row " @@ -256,7 +266,9 @@ default double[][][] parseTransitionValues(JsonNode valuesNode, int buckets, int + stateCount); } for (int c = 0; c < stateCount; c++) { - values[b][r][c] = rowNode.get(c).asDouble(); + values[b][r][c] = + extractDoubleValue( + rowNode.get(c), "data.transitions.values[" + b + "][" + r + "][" + c + "]"); } } } @@ -269,10 +281,37 @@ default List readDoubleArray(JsonNode node, String field) { throw new FactoryException("Field '" + field + "' must be an array"); } List values = new ArrayList<>(); - arrayNode.forEach(element -> values.add(element.asDouble())); + for (int i = 0; i < arrayNode.size(); i++) { + values.add(extractDoubleValue(arrayNode.get(i), field + "[" + i + "]")); + } return List.copyOf(values); } + default int extractInt(JsonNode node, int index, String field) { + JsonNode value = node.get(index); + if (value == null || value.isMissingNode() || value.isNull()) { + throw new FactoryException("Missing field '" + field + "[" + index + "]'"); + } + if (!value.canConvertToInt()) { + throw new FactoryException("Field '" + field + "[" + index + "]' must be an integer"); + } + return value.asInt(); + } + + default double extractDoubleValue(JsonNode node, String field) { + if (node == null || node.isMissingNode() || node.isNull()) { + throw new FactoryException("Missing field '" + field + "'"); + } + if (!node.isNumber()) { + throw new FactoryException("Field '" + field + "' must be a double"); + } + double value = node.asDouble(); + if (!Double.isFinite(value)) { + throw new FactoryException("Field '" + field + "' must be finite"); + } + return value; + } + default Optional parsePowerReference( JsonNode parent, String field) { JsonNode referenceNode = parent.get(field); diff --git a/src/main/java/edu/ie3/datamodel/io/naming/ModelFields.java b/src/main/java/edu/ie3/datamodel/io/naming/ModelFields.java index 2ef708197a..5bd98c5376 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/ModelFields.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/ModelFields.java @@ -231,7 +231,6 @@ public static void registerUnsupported(Class entityClass, Set unsuppo MARKOV_GENERATOR, MARKOV_TIME_MODEL, MARKOV_VALUE_MODEL, - MARKOV_PARAMETERS, MARKOV_DATA, MARKOV_GENERATOR_NAME, MARKOV_GENERATOR_VERSION, diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index 60330307e7..9f3f817b74 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -27,19 +27,15 @@ /** * Container for Markov-chain-based load models produced by simonaMarkovLoad. * - *

The model bundles the static data (transition matrices, GMM parameters, normalization) with - * the simulation helpers needed to generate stepwise power values. Each simulation step should use - * a fresh supplier obtained via {@link #getValueSupplier(PowerValueSource.MarkovIdentifier)}. + *

The model keeps the trained transition and GMM data together with the helpers used to sample + * one power value per simulation step. */ public class MarkovLoadModel { - // Constants - private static final int QUARTER_HOURS_PER_DAY = 96; private static final int WEEKEND_FACTOR = QUARTER_HOURS_PER_DAY; private static final int MONTH_FACTOR = QUARTER_HOURS_PER_DAY * 2; - - // Model data + private static final double PROBABILITY_TOLERANCE = 1e-5; private final String schema; private final ZonedDateTime generatedAt; @@ -50,8 +46,6 @@ public class MarkovLoadModel { private final TransitionData transitionData; private final Optional gmmBuckets; - // Derived runtime fields - private final ZoneId zoneId; private final double[][][] transitions; private final int bucketCount; @@ -62,13 +56,11 @@ public class MarkovLoadModel { private final ComparableQuantity maxPowerFromModel; private final ComparableQuantity minPowerFromModel; - // ===================================================================================== - // Constructor - // ===================================================================================== - /** - * Builds a model from the JSON-derived sections, deriving runtime caches (transition arrays, - * discretization thresholds, per-state GMM data) and validating overall consistency. + * Builds a model from the parsed JSON sections and prepares the runtime lookup arrays. + * + *

Semantic invariants such as transition row sums and GMM component consistency are checked + * here as well, so directly constructed models are protected just like JSON-parsed models. * * @throws IllegalArgumentException if GMM data is missing, the normalization range is * non-positive, or the transition tensor does not match {@code bucketCount * stateCount * @@ -128,7 +120,6 @@ public MarkovLoadModel( validateTransitionDimensions(); } - /** Verifies that the parsed transition tensor matches the declared bucket and state counts. */ private void validateTransitionDimensions() { if (transitions.length != bucketCount) { throw new IllegalArgumentException( @@ -147,16 +138,43 @@ private void validateTransitionDimensions() { + " but was " + transitions[bucket].length); } + for (int state = 0; state < transitions[bucket].length; state++) { + double[] row = transitions[bucket][state]; + if (row.length != stateCount) { + throw new IllegalArgumentException( + "Transition next-state count mismatch in bucket " + + bucket + + ", state " + + state + + ". Expected " + + stateCount + + " but was " + + row.length); + } + validateProbabilityVector(row, "Transition row in bucket " + bucket + ", state " + state); + } } } - // Constructor helpers + private static void validateProbabilityVector(double[] weights, String context) { + if (weights.length == 0) { + throw new IllegalArgumentException(context + " must not be empty."); + } + double sum = 0d; + for (double weight : weights) { + if (!Double.isFinite(weight)) { + throw new IllegalArgumentException(context + " contains a non-finite probability."); + } + if (weight < 0d) { + throw new IllegalArgumentException(context + " contains a negative probability."); + } + sum += weight; + } + if (Math.abs(sum - 1d) > PROBABILITY_TOLERANCE) { + throw new IllegalArgumentException(context + " must sum to 1.0, but summed to " + sum + "."); + } + } - /** - * Converts the JSON-shaped {@link GmmBuckets} (lists of doubles) into a runtime {@code - * GmmStateData[][]} backed by primitive arrays for fast sampling. States with no GMM data are - * preserved as {@code null} entries; {@link #sampleNormalizedValue} returns {@code 0} for them. - */ private GmmStateData[][] buildGmmStates(GmmBuckets buckets) { List bucketList = buckets.buckets(); if (bucketList.size() != bucketCount) { @@ -178,7 +196,6 @@ private GmmStateData[][] buildGmmStates(GmmBuckets buckets) { return lookup; } - /** Converts a JSON power reference into a typed quantity. Only {@code "kW"} is accepted. */ private ComparableQuantity convertPowerReference( ValueModel.Normalization.PowerReference reference) { if (!"kW".equalsIgnoreCase(reference.unit())) { @@ -188,10 +205,6 @@ private ComparableQuantity convertPowerReference( return Quantities.getQuantity(reference.value(), StandardUnits.ACTIVE_POWER_IN); } - // ===================================================================================== - // Accessors - // ===================================================================================== - public String schema() { return schema; } @@ -224,10 +237,6 @@ public Optional gmmBuckets() { return gmmBuckets; } - // ===================================================================================== - // Public API - Simulation - // ===================================================================================== - /** * Returns a supplier for a single Markov step. * @@ -259,17 +268,6 @@ public Optional> getProfileEnergyScaling() { return Optional.empty(); } - // ===================================================================================== - // Simulation pipeline - // computeStep - // 1. bucketId => isWeekend - // 2. resolveState => discretize - // 3. deriveSeed - // 4. simulateStep => drawWeighted => sampleNormalizedValue - // 5. scale - // ===================================================================================== - - /** Runs the full five-step simulation pipeline and packages the result for the caller. */ private PowerValueSource.MarkovOutputValue computeStep(PowerValueSource.MarkovIdentifier data) { int bucket = bucketId(data.time()); int currentState = resolveState(data); @@ -279,13 +277,7 @@ private PowerValueSource.MarkovOutputValue computeStep(PowerValueSource.MarkovId return new PowerValueSource.MarkovOutputValue(Optional.of(new PValue(power)), step.nextState()); } - // Step 1: Determine time bucket - - /** - * Maps a timestamp into one of {@code bucketCount} buckets, encoded as {@code month * - * MONTH_FACTOR + weekendFlag * WEEKEND_FACTOR + quarterHour}. Each bucket has its own transition - * matrix and per-state GMM, so the result drives every subsequent lookup. - */ + /** Maps a timestamp into the trainer's bucket layout: month, weekday/weekend and quarter-hour. */ private int bucketId(ZonedDateTime time) { ZonedDateTime zoned = time.withZoneSameInstant(zoneId); int month = zoned.getMonthValue() - 1; @@ -301,12 +293,8 @@ private boolean isWeekend(ZonedDateTime time) { return day == DayOfWeek.SATURDAY || day == DayOfWeek.SUNDAY; } - // Step 2: Resolve current state - /** - * Resolves the Markov state used as the starting point of this step. If the caller supplied a - * previous state, it is used directly; otherwise the supplied {@code initialNormalizedValue} is - * discretized via {@link #discretize}. + * Uses the previous state if present; otherwise discretizes the initial normalized value. * * @throws IllegalArgumentException if a supplied previous state is out of bounds */ @@ -323,11 +311,8 @@ private int resolveState(PowerValueSource.MarkovIdentifier input) { } /** - * Returns the state-bin index for a normalized value, using the right-edges in {@code - * discretizationThresholds}. Bins are left-closed and right-open, so a value exactly on a - * threshold belongs to the upper state - matching the trainer's {@code - * searchsorted(side="right")} assignment used when the model was fitted. The input is clamped to - * {@code [0, 1]} to absorb minor floating-point drift on caller-supplied values. + * Returns the state-bin index for a normalized value. Boundary values map to the upper bin, + * matching the trainer's {@code searchsorted(side="right")} assignment. */ private int discretize(double normalized) { double value = Math.clamp(normalized, 0d, 1d); @@ -339,13 +324,7 @@ private int discretize(double normalized) { return discretizationThresholds.length; } - // Step 3: Derive deterministic seed - - /** - * Produces a deterministic RNG seed for one simulation step by mixing the request seed with the - * bucket, state and time slot. Identical inputs always yield the same output, which is required - * for reproducibility across simulation runs. - */ + /** Produces a deterministic RNG seed from request seed, bucket, state and time slot. */ private long deriveSeed(PowerValueSource.MarkovIdentifier input, int bucket, int state) { long seed = input.randomSeed(); seed = 31 * seed + bucket; @@ -354,25 +333,14 @@ private long deriveSeed(PowerValueSource.MarkovIdentifier input, int bucket, int return 31 * seed + slot; } - // Step 4: Simulate transition and sample value - - /** - * Performs one Markov step: draw the next state from the transition row, then sample its GMM. The - * simonaMarkovLoad trainer guarantees that every state with non-zero incoming transition - * probability also has GMM data (empty rows fall back to a self-loop on the current state), so no - * per-step row sanitization is needed. - */ + /** Draws the next state from the transition row and samples the corresponding GMM. */ private StepResult simulateStep(int bucket, int currentState, SplittableRandom rng) { int nextStateIndex = drawWeighted(transitions[bucket][currentState], rng); double normalized = sampleNormalizedValue(bucket, nextStateIndex, rng); return new StepResult(nextStateIndex, normalized); } - /** - * Picks an index by weighted random sampling. The {@code weights} array must be non-negative and - * sum to (approximately) one; used both to draw the next Markov state and to draw a GMM - * component. - */ + /** Picks an index by weighted random sampling. */ private static int drawWeighted(double[] weights, SplittableRandom rng) { double sample = rng.nextDouble(); double cumulative = 0d; @@ -386,10 +354,8 @@ private static int drawWeighted(double[] weights, SplittableRandom rng) { } /** - * Draws a normalized power value from the GMM at {@code (bucket, state)}. Returns {@code 0} if - * that cell has no GMM data. The result is clamped to {@code [0, 1]} because Gaussian tails are - * unbounded; without clamping, {@link #scale} could return values outside {@code [minPower, - * maxPower]}. + * Draws a normalized power value from the GMM at {@code (bucket, state)} and clamps Gaussian + * tails to the model range. */ private double sampleNormalizedValue(int bucket, int state, SplittableRandom rng) { GmmStateData gmm = gmmStates[bucket][state]; @@ -399,27 +365,15 @@ private double sampleNormalizedValue(int bucket, int state, SplittableRandom rng return Math.clamp(gmm.sample(rng), 0d, 1d); } - // Step 5: Scale normalized value to power - - /** - * Maps a normalized {@code [0, 1]} value into actual power via {@code minPower + value * - * (maxPower - minPower)}. - */ + /** Maps a normalized value to physical power using the model's min/max range. */ private ComparableQuantity scale(double normalizedValue) { ComparableQuantity range = maxPowerFromModel.subtract(minPowerFromModel); return minPowerFromModel.add(range.multiply(normalizedValue)).asType(Power.class); } - // ===================================================================================== - // Inner helper types - // ===================================================================================== - private record StepResult(int nextState, double normalizedValue) {} - /** - * Runtime representation of a single GMM state, using primitive arrays for efficient sampling. - * Converted from {@link GmmBuckets.GmmState} during construction. - */ + /** Runtime representation of a single GMM state. */ private static final class GmmStateData { private final double[] weights; private final double[] means; @@ -431,11 +385,6 @@ private GmmStateData(double[] weights, double[] means, double[] variances) { this.variances = variances; } - /** - * Draws a random value from this state's Gaussian Mixture Model. First, a mixture component is - * selected (weighted by {@code weights}), then a value is sampled from that component's normal - * distribution. - */ private double sample(SplittableRandom rng) { int component = drawWeighted(weights, rng); double mean = means[component]; @@ -447,21 +396,10 @@ private double sample(SplittableRandom rng) { } } - // ===================================================================================== - // Data records (JSON structure) - // ===================================================================================== - - /** - * Provenance metadata for the trained model: trainer name, version (e.g. a git SHA) and the - * free-form configuration that produced this JSON. - */ + /** Provenance metadata for the trained model. */ public record Generator(String name, String version, Map config) {} - /** - * Temporal layout of the model: number of buckets, the (documentation-only) encoding formula used - * by the trainer, the sampling interval in minutes, and the IANA timezone the buckets are defined - * in. - */ + /** Temporal layout of the model. */ public record TimeModel( int bucketCount, String bucketEncodingFormula, @@ -473,9 +411,8 @@ public record ValueModel( String valueUnit, Normalization normalization, Discretization discretization) { /** - * Normalization range used to map between normalized {@code [0, 1]} values and physical power. - * Both {@code maxPower} and {@code minPower} are required at construction time; the optional - * wrapping only reflects the JSON shape. + * Normalization range used to map between normalized values and physical power. The optional + * wrapping reflects the JSON shape, both bounds are required at construction time. */ public record Normalization( String method, Optional maxPower, Optional minPower) { @@ -484,10 +421,7 @@ public record Normalization( public record PowerReference(double value, String unit) {} } - /** - * State-bin layout: the number of states and the right-edge thresholds that separate - * neighbouring states. The list contains {@code states - 1} entries. - */ + /** State-bin layout and right-edge thresholds. */ public record Discretization(int states, List thresholdsRight) {} } @@ -496,8 +430,8 @@ public record Parameters( Optional transitions, Optional gmm) { /** - * Strategy used by the trainer when a transition row had no observations (e.g. {@code - * "self_loop"}). Carried through for traceability; not used at simulation time. + * Strategy used by the trainer when a transition row had no observations. Carried through for + * traceability. Not used at simulation time. */ public record TransitionParameters(String emptyRowStrategy) {} @@ -506,28 +440,21 @@ public record TransitionParameters(String emptyRowStrategy) {} * * @param valueColumn name of the column the trainer fit on * @param verbose scikit-learn verbosity used during fitting - * @param heartbeatSeconds stuck-process watchdog interval used by the simonaMarkovLoad Python - * trainer: when set, the trainer enables {@code faulthandler.dump_traceback_later} and - * dumps a stack trace every N seconds so hangs during GMM fitting can be diagnosed. Carried - * through for JSON round-trip only; unused at simulation time. + * @param heartbeatSeconds trainer watchdog interval, carried through for JSON round-trip only */ public record GmmParameters( String valueColumn, OptionalInt verbose, OptionalInt heartbeatSeconds) {} } /** - * The 3D transition tensor {@code values[bucket][currentState][nextState] = probability}, plus - * the dtype and encoding metadata from the JSON. {@link #equals} and {@link #hashCode} are - * overridden so two records with equal array contents compare equal; the synthetic record - * versions would otherwise compare arrays by identity. + * Transition tensor {@code values[bucket][currentState][nextState] = probability}. Equality uses + * array contents instead of array identity. */ public record TransitionData(String dtype, String encoding, double[][][] values) { - /** Number of time buckets, i.e. the outer dimension of {@link #values()}. */ public int bucketCount() { return values.length; } - /** Number of states, i.e. the inner dimension of {@link #values()}. */ public int stateCount() { return values.length == 0 ? 0 : values[0].length; } @@ -558,27 +485,52 @@ public String toString() { } } - /** - * Per-bucket Gaussian Mixture Models. The outer list has {@code bucketCount} entries (one per - * time bucket); each {@link GmmBucket} has {@code stateCount} {@link GmmState} entries (one per - * state). A {@link GmmState} may be {@code null} when no observations were available for that - * (bucket, state) pair. - */ + /** Per-bucket Gaussian Mixture Models. State entries may be {@code null}. */ public record GmmBuckets(List buckets) { - /** GMM data for one time bucket, indexed by state. Entries may be {@code null}. */ public record GmmBucket(List states) {} - /** - * Parameters of one Gaussian Mixture Model: per-component {@code weights}, {@code means} and - * {@code variances}. Each component is a 1-D Gaussian; the trainer chooses 1-3 components per - * (bucket, state) cell using BIC. - */ + /** Parameters of one Gaussian Mixture Model. */ public record GmmState(List weights, List means, List variances) { + public GmmState { + validateGmmComponents(weights, means, variances); + } private GmmStateData toStateData() { return new GmmStateData(toArray(weights), toArray(means), toArray(variances)); } + private static void validateGmmComponents( + List weights, List means, List variances) { + Objects.requireNonNull(weights, "weights"); + Objects.requireNonNull(means, "means"); + Objects.requireNonNull(variances, "variances"); + + if (weights.isEmpty()) { + throw new IllegalArgumentException("GMM state must contain at least one component."); + } + if (weights.size() != means.size() || weights.size() != variances.size()) { + throw new IllegalArgumentException( + "GMM weights, means and variances must have the same number of entries."); + } + + double[] weightArray = toArray(weights); + validateProbabilityVector(weightArray, "GMM component weights"); + + for (int i = 0; i < means.size(); i++) { + Double mean = means.get(i); + Double variance = variances.get(i); + if (mean == null || !Double.isFinite(mean)) { + throw new IllegalArgumentException("GMM means must be finite."); + } + if (variance == null || !Double.isFinite(variance)) { + throw new IllegalArgumentException("GMM variances must be finite."); + } + if (variance < 0d) { + throw new IllegalArgumentException("GMM variances must be non-negative."); + } + } + } + private static double[] toArray(List values) { double[] array = new double[values.size()]; for (int i = 0; i < values.size(); i++) { @@ -589,10 +541,6 @@ private static double[] toArray(List values) { } } - // ===================================================================================== - // equals / hashCode / toString - // ===================================================================================== - @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy index 31543f9f76..7154dfddcf 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy @@ -31,7 +31,7 @@ class MarkovLoadModelFactoryTest extends Specification { model.transitionData().values()[0][0][1] == 0.9d model.gmmBuckets().isPresent() def gmmState = model.gmmBuckets().get().buckets().first().states().first() - gmmState.weights() == [0.6d] + gmmState.weights() == [1.0d] gmmState.means() == [1.0d] gmmState.variances() == [0.2d] model.valueModel().normalization().maxPower().isPresent() @@ -57,6 +57,18 @@ class MarkovLoadModelFactoryTest extends Specification { thrown(FactoryException) } + def "buildModel throws FactoryException when transition row does not sum to one"() { + given: + def invalidJson = objectMapper.readTree(validModelJson() + .replace('[0.1, 0.9]', '[0.1, 0.8]')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + def "buildModel throws FactoryException when threshold count does not match state count"() { given: "states=2 requires exactly 1 threshold, but 2 are provided" def invalidJson = objectMapper.readTree(validModelJson() @@ -117,6 +129,18 @@ class MarkovLoadModelFactoryTest extends Specification { thrown(FactoryException) } + def "buildModel throws FactoryException when GMM component arrays differ in size"() { + given: + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"variances": [0.2]', '"variances": [0.2, 0.3]')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + def "buildModel tolerates a missing parameters block"() { given: def root = objectMapper.readTree(validModelJson()) @@ -219,7 +243,7 @@ class MarkovLoadModelFactoryTest extends Specification { { "states": [ { - "weights": [0.6], + "weights": [1.0], "means": [1.0], "variances": [0.2] }, diff --git a/src/test/groovy/edu/ie3/datamodel/io/naming/ModelFieldsTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/naming/ModelFieldsTest.groovy index 9c8463186a..c15ab6243e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/naming/ModelFieldsTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/naming/ModelFieldsTest.groovy @@ -23,7 +23,6 @@ class ModelFieldsTest extends Specification { FieldNamingStrategy.MARKOV_GENERATOR, FieldNamingStrategy.MARKOV_TIME_MODEL, FieldNamingStrategy.MARKOV_VALUE_MODEL, - FieldNamingStrategy.MARKOV_PARAMETERS, FieldNamingStrategy.MARKOV_DATA, // nested - required for simulation FieldNamingStrategy.MARKOV_GENERATOR_NAME, diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy index ad99653657..a543323eaf 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy @@ -14,6 +14,8 @@ import edu.ie3.datamodel.io.source.PowerValueSource import edu.ie3.datamodel.models.StandardUnits import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel import spock.lang.Specification +import tools.jackson.databind.ObjectMapper +import tools.jackson.databind.node.ObjectNode import java.nio.file.Files import java.nio.file.Path @@ -85,6 +87,26 @@ class JsonMarkovProfileSourceTest extends Specification { openCount.get() == 1 } + def "validate tolerates a missing optional parameters block"() { + given: + def root = new ObjectMapper().readTree(validModelJson()) + ((ObjectNode) root).remove("parameters") + Files.writeString(jsonFile, root.toString()) + def source = new JsonMarkovProfileSource( + new JsonDataSource(tempDir, new FileNamingStrategy()), + new FileLoadProfileMetaInformation("profile1", jsonFile, FileType.JSON) + ) + + when: + source.validate() + def model = source.getModel() + + then: + noExceptionThrown() + model.parameters().transitions().isEmpty() + model.parameters().gmm().isEmpty() + } + def "getModel throws SourceException on invalid JSON file"() { given: Files.writeString(jsonFile, "{}") From 220ef76c4012a8d9c4fdce832b8e32b5f733d657 Mon Sep 17 00:00:00 2001 From: Philipp Date: Wed, 8 Jul 2026 15:50:00 +0200 Subject: [PATCH 36/39] Meeting Changes --- docs/readthedocs/io/markov.md | 2 +- .../EntityPersistenceNamingStrategy.java | 7 ++-- .../LoadProfileMetaInformation.java | 22 ++++++++++++- .../models/profile/PowerProfileKey.java | 33 ++++++++++++++++--- ...EntityPersistenceNamingStrategyTest.groovy | 9 +++-- .../io/naming/FileNamingStrategyTest.groovy | 7 ++-- .../models/profile/PowerProfileKeyTest.groovy | 10 ++++++ 7 files changed, 77 insertions(+), 13 deletions(-) diff --git a/docs/readthedocs/io/markov.md b/docs/readthedocs/io/markov.md index 2ab58e492d..a50218ff17 100644 --- a/docs/readthedocs/io/markov.md +++ b/docs/readthedocs/io/markov.md @@ -272,7 +272,7 @@ Markov models are loaded via {code}`JsonMarkovProfileSource`, which: - Reads the JSON file lazily on first access (not at construction time) - Caches the parsed {code}`MarkovLoadModel` for subsequent calls -- Offers validation of all 20 mandatory fields via {code}`validate()`. Note that this is a separate call: +- Offers validation of all mandatory fields via {code}`validate()`. Note that this is a separate call: it is not triggered automatically when the model is loaded - Is thread-safe ({code}`synchronized` lazy loading) diff --git a/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java b/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java index 5a893eea9c..7cd31efbab 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java @@ -57,10 +57,11 @@ public class EntityPersistenceNamingStrategy { /** * Regex to match the naming convention of a file for a repetitive load profile time series. The - * profile is accessible via the named capturing group "profile", the uuid by the group "uuid" + * profile type is accessible via the named capturing group "type", the profile by the group + * "profile" */ private static final String LOAD_PROFILE_TIME_SERIES = - "(?:lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})"; + "(?lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})"; /** * Pattern to identify load profile time series in this instance of the naming strategy (takes @@ -162,7 +163,7 @@ public LoadProfileMetaInformation loadProfileTimesSeriesMetaInformation(String f throw new IllegalArgumentException( "Cannot extract meta information on load profile time series from '" + fileName + "'."); - return new LoadProfileMetaInformation(matcher.group("profile")); + return new LoadProfileMetaInformation(matcher.group("profile"), matcher.group("type")); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java index d25d8fadbe..49ed3a9bff 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java @@ -14,8 +14,16 @@ public class LoadProfileMetaInformation extends TimeSeriesMetaInformation { private final PowerProfileKey profileKey; public LoadProfileMetaInformation(String profileKey) { + this(profileKey, PowerProfileKey.Type.TS); + } + + public LoadProfileMetaInformation(String profileKey, String type) { + this(profileKey, parseType(type)); + } + + public LoadProfileMetaInformation(String profileKey, PowerProfileKey.Type type) { super(UUID.randomUUID()); - this.profileKey = new PowerProfileKey(profileKey); + this.profileKey = new PowerProfileKey(profileKey, type); } public LoadProfileMetaInformation(PowerProfileKey powerProfileKey) { @@ -27,6 +35,18 @@ public PowerProfileKey getProfileKey() { return profileKey; } + private static PowerProfileKey.Type parseType(String type) { + return switch (type) { + case "lpts" -> PowerProfileKey.Type.TS; + case "markov" -> PowerProfileKey.Type.MARKOV; + default -> + throw new IllegalArgumentException( + "The given type '" + + type + + "' is not supported for load profile time series meta information."); + }; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/src/main/java/edu/ie3/datamodel/models/profile/PowerProfileKey.java b/src/main/java/edu/ie3/datamodel/models/profile/PowerProfileKey.java index 501fe22bc2..38222d1a74 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/PowerProfileKey.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/PowerProfileKey.java @@ -12,10 +12,16 @@ /** Interface defining a power profile. */ public final class PowerProfileKey implements Serializable { + private final Type type; private final String value; public final boolean noKeyAssigned; public PowerProfileKey(String key) { + this(key, Type.TS); + } + + public PowerProfileKey(String key, Type type) { + this.type = type; if (key == null || key.isEmpty()) { this.value = "No profile assigned."; this.noKeyAssigned = true; @@ -29,6 +35,10 @@ public String getValue() { return value; } + public Type getType() { + return type; + } + public boolean equals(PowerProfile other) { return equals(other.getKey()); } @@ -41,24 +51,39 @@ public boolean equalsAny(PowerProfile... others) { public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; PowerProfileKey that = (PowerProfileKey) o; - return noKeyAssigned == that.noKeyAssigned && Objects.equals(value, that.value); + return noKeyAssigned == that.noKeyAssigned + && type == that.type + && Objects.equals(value, that.value); } @Override public int hashCode() { - return Objects.hash(value, noKeyAssigned); + return Objects.hash(type, value, noKeyAssigned); } @Override public String toString() { - return "PowerProfileKey{" + "value='" + value + '\'' + ", noKeyAssigned=" + noKeyAssigned + '}'; + return "PowerProfileKey{" + + "type=" + + type + + ", value='" + + value + + '\'' + + ", noKeyAssigned=" + + noKeyAssigned + + '}'; } // static - public static final PowerProfileKey NO_KEY_ASSIGNED = new PowerProfileKey(""); + public static final PowerProfileKey NO_KEY_ASSIGNED = new PowerProfileKey("", Type.TS); public static String toUniformKey(String key) { return key.toLowerCase().replaceAll("[-_]*", ""); } + + public enum Type { + TS, + MARKOV + } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy index 7f1dfb8230..5a8a63859b 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy @@ -21,6 +21,7 @@ import edu.ie3.datamodel.models.input.system.type.* import edu.ie3.datamodel.models.input.thermal.CylindricalStorageInput import edu.ie3.datamodel.models.input.thermal.ThermalHouseInput import edu.ie3.datamodel.models.profile.BdewStandardLoadProfile +import edu.ie3.datamodel.models.profile.PowerProfileKey import edu.ie3.datamodel.models.result.NodeResult import edu.ie3.datamodel.models.result.connector.LineResult import edu.ie3.datamodel.models.result.connector.SwitchResult @@ -87,8 +88,10 @@ class EntityPersistenceNamingStrategyTest extends Specification { matcher.matches() then: "it also has correct capturing groups" - matcher.groupCount() == 1 - matcher.group(1) == "g3" + matcher.groupCount() == 2 + matcher.group(1) == "lpts" + matcher.group("type") == "lpts" + matcher.group(2) == "g3" matcher.group("profile") == "g3" } @@ -102,6 +105,7 @@ class EntityPersistenceNamingStrategyTest extends Specification { then: matcher.matches() + matcher.group("type") == "markov" matcher.group("profile") == "demo1" } @@ -141,6 +145,7 @@ class EntityPersistenceNamingStrategyTest extends Specification { then: meta.profileKey.getValue() == "demo2" + meta.profileKey.type == PowerProfileKey.Type.MARKOV } def "The EntityPersistenceNamingStrategy is able to prepare the prefix properly"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy index ff6fc1334e..ff20b75533 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy @@ -24,6 +24,7 @@ import edu.ie3.datamodel.models.input.system.type.* import edu.ie3.datamodel.models.input.thermal.CylindricalStorageInput import edu.ie3.datamodel.models.input.thermal.ThermalHouseInput import edu.ie3.datamodel.models.profile.BdewStandardLoadProfile +import edu.ie3.datamodel.models.profile.PowerProfileKey import edu.ie3.datamodel.models.result.NodeResult import edu.ie3.datamodel.models.result.connector.LineResult import edu.ie3.datamodel.models.result.connector.SwitchResult @@ -684,7 +685,7 @@ class FileNamingStrategyTest extends Specification { def actual = strategy.loadProfileTimeSeriesPattern.pattern() then: - actual == "test_grid" + escapedFileSeparator + "input" + escapedFileSeparator + "participants" + escapedFileSeparator + "time_series" + escapedFileSeparator + "(?:lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})" + actual == "test_grid" + escapedFileSeparator + "input" + escapedFileSeparator + "participants" + escapedFileSeparator + "time_series" + escapedFileSeparator + "(?lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})" } def "A FileNamingStrategy with FlatHierarchy returns correct individual time series file name pattern"() { @@ -706,7 +707,7 @@ class FileNamingStrategyTest extends Specification { def actual = strategy.loadProfileTimeSeriesPattern.pattern() then: - actual == "(?:lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})" + actual == "(?lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})" } def "Trying to extract time series meta information throws an Exception, if it is provided a malformed string"() { @@ -837,6 +838,7 @@ class FileNamingStrategyTest extends Specification { LoadProfileMetaInformation.isAssignableFrom(metaInformation.getClass()) (metaInformation as LoadProfileMetaInformation).with { profileKey.value == "g3" + profileKey.type == PowerProfileKey.Type.TS } } @@ -852,6 +854,7 @@ class FileNamingStrategyTest extends Specification { LoadProfileMetaInformation.isAssignableFrom(metaInformation.getClass()) (metaInformation as LoadProfileMetaInformation).with { profileKey.value == "g3" + profileKey.type == PowerProfileKey.Type.TS } } } diff --git a/src/test/groovy/edu/ie3/datamodel/models/profile/PowerProfileKeyTest.groovy b/src/test/groovy/edu/ie3/datamodel/models/profile/PowerProfileKeyTest.groovy index 78984dab26..472297bb8e 100644 --- a/src/test/groovy/edu/ie3/datamodel/models/profile/PowerProfileKeyTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/models/profile/PowerProfileKeyTest.groovy @@ -121,4 +121,14 @@ class PowerProfileKeyTest extends Specification { "" || PowerProfileKey.NO_KEY_ASSIGNED null || PowerProfileKey.NO_KEY_ASSIGNED } + + def "Power profile keys with same value but different type are distinct"() { + given: + def timeSeriesKey = new PowerProfileKey("demo", PowerProfileKey.Type.TS) + def markovKey = new PowerProfileKey("demo", PowerProfileKey.Type.MARKOV) + + expect: + timeSeriesKey != markovKey + timeSeriesKey.hashCode() != markovKey.hashCode() + } } From 60ce3dad59a173fb3349aedb6c18e3a2a8b66580 Mon Sep 17 00:00:00 2001 From: Philipp Date: Thu, 9 Jul 2026 12:03:30 +0200 Subject: [PATCH 37/39] Review Comments --- .../markov/MarkovLoadModelFactory.java | 16 +- .../markov/MarkovModelParsingSupport.java | 166 +++++++++++------- .../io/naming/FieldNamingStrategy.java | 24 +++ .../LoadProfileMetaInformation.java | 14 +- .../datamodel/io/source/PowerValueSource.java | 4 - .../source/json/JsonMarkovProfileSource.java | 7 +- .../models/profile/PowerProfileKey.java | 14 +- .../markov/MarkovLoadModelFactoryTest.groovy | 73 +------- .../json/JsonMarkovProfileSourceTest.groovy | 101 +++-------- .../models/profile/PowerProfileKeyTest.groovy | 18 ++ .../profile/markov/MarkovLoadModelTest.groovy | 65 +------ .../markov/MarkovModelJsonTestSupport.groovy | 102 +++++++++++ 12 files changed, 300 insertions(+), 304 deletions(-) create mode 100644 src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovModelJsonTestSupport.groovy diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java index af1675fea6..549cbc7fab 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -25,17 +25,17 @@ public MarkovLoadModelFactory() { @Override protected MarkovLoadModel buildModel(MarkovModelData data) { JsonNode root = data.getRoot(); - String schema = extractText(root, "schema"); - ZonedDateTime generatedAt = parseTimestamp(extractText(root, "generated_at")); - Generator generator = parseGenerator(extractNode(root, "generator")); - TimeModel timeModel = extractTimeModel(extractNode(root, "time_model")); - ValueModel valueModel = parseValueModel(extractNode(root, "value_model")); - Parameters parameters = parseParameters(root.path("parameters")); + String schema = extractText(root, MARKOV_SCHEMA); + ZonedDateTime generatedAt = parseTimestamp(extractText(root, jsonField(MARKOV_GENERATED_AT))); + Generator generator = parseGenerator(extractNode(root, MARKOV_GENERATOR)); + TimeModel timeModel = extractTimeModel(extractNode(root, jsonField(MARKOV_TIME_MODEL))); + ValueModel valueModel = parseValueModel(extractNode(root, jsonField(MARKOV_VALUE_MODEL))); + Parameters parameters = parseParameters(root.path(MARKOV_PARAMETERS)); - JsonNode dataNode = extractNode(root, "data"); + JsonNode dataNode = extractNode(root, MARKOV_DATA); TransitionData transitionData = parseTransitions(dataNode, timeModel.bucketCount(), valueModel.discretization().states()); - GmmBuckets gmmBuckets = parseGmmBuckets(extractNode(dataNode, "gmms")); + GmmBuckets gmmBuckets = parseGmmBuckets(extractNode(dataNode, jsonLeafField(MARKOV_GMMS))); return new MarkovLoadModel( schema, diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java index 68a3c75838..96c5e6e737 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -5,8 +5,11 @@ */ package edu.ie3.datamodel.io.factory.markov; +import static edu.ie3.datamodel.io.naming.FieldNamingStrategy.*; + import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.*; +import edu.ie3.util.StringUtils; import java.time.ZonedDateTime; import java.time.format.DateTimeParseException; import java.util.*; @@ -16,10 +19,10 @@ interface MarkovModelParsingSupport { default Generator parseGenerator(JsonNode generatorNode) { - String name = extractText(generatorNode, "name"); - String version = extractText(generatorNode, "version"); + String name = extractText(generatorNode, jsonLeafField(MARKOV_GENERATOR_NAME)); + String version = extractText(generatorNode, jsonLeafField(MARKOV_GENERATOR_VERSION)); Map config = new LinkedHashMap<>(); - JsonNode configNode = generatorNode.path("config"); + JsonNode configNode = generatorNode.path(jsonLeafField(MARKOV_GENERATOR_CONFIG)); if (configNode.isObject()) { for (Map.Entry entry : configNode.properties()) { config.put(entry.getKey(), entry.getValue().asString()); @@ -30,47 +33,56 @@ default Generator parseGenerator(JsonNode generatorNode) { /** Extracts the time model block. */ default TimeModel extractTimeModel(JsonNode timeNode) { - int bucketCount = extractInt(timeNode, "bucket_count"); + int bucketCount = extractInt(timeNode, jsonLeafField(MARKOV_BUCKET_COUNT)); if (bucketCount <= 0) { - throw new FactoryException("time_model.bucket_count must be positive"); + throw new FactoryException(jsonField(MARKOV_BUCKET_COUNT) + " must be greater than 0."); } - String formula = extractNode(timeNode, "bucket_encoding").path("formula").asString(""); + String formula = + extractNode(timeNode, jsonLeafField(MARKOV_BUCKET_ENCODING)) + .path(jsonLeafField(MARKOV_BUCKET_ENCODING_FORMULA)) + .asString(""); if (formula.isEmpty()) { - throw new FactoryException("Missing bucket encoding formula"); + throw new FactoryException("Missing bucket encoding formula."); } - int samplingInterval = extractInt(timeNode, "sampling_interval_minutes"); + int samplingInterval = extractInt(timeNode, jsonLeafField(MARKOV_SAMPLING_INTERVAL)); if (samplingInterval <= 0) { - throw new FactoryException("time_model.sampling_interval_minutes must be positive"); + throw new FactoryException(jsonField(MARKOV_SAMPLING_INTERVAL) + " must be greater than 0."); } - String timezone = extractText(timeNode, "timezone"); + String timezone = extractText(timeNode, jsonLeafField(MARKOV_TIMEZONE)); return new TimeModel(bucketCount, formula, samplingInterval, timezone); } /** Parses value model settings. */ default ValueModel parseValueModel(JsonNode valueNode) { - String valueUnit = extractText(valueNode, "value_unit"); - JsonNode normalizationNode = extractNode(valueNode, "normalization"); - String normalizationMethod = extractText(normalizationNode, "method"); + String valueUnit = extractText(valueNode, jsonLeafField(MARKOV_VALUE_UNIT)); + JsonNode normalizationNode = extractNode(valueNode, jsonLeafField(MARKOV_NORMALIZATION)); + String normalizationMethod = + extractText(normalizationNode, jsonLeafField(MARKOV_NORMALIZATION_METHOD)); ValueModel.Normalization normalization = new ValueModel.Normalization( normalizationMethod, - parsePowerReference(normalizationNode, "max_power"), - parsePowerReference(normalizationNode, "min_power")); + parsePowerReference(normalizationNode, jsonLeafField(MARKOV_MAX_POWER)), + parsePowerReference(normalizationNode, jsonLeafField(MARKOV_MIN_POWER))); - JsonNode discretizationNode = extractNode(valueNode, "discretization"); - int states = extractInt(discretizationNode, "states"); + JsonNode discretizationNode = extractNode(valueNode, jsonLeafField(MARKOV_DISCRETIZATION)); + int states = extractInt(discretizationNode, jsonLeafField(MARKOV_DISCRETIZATION_STATES)); if (states <= 0) { - throw new FactoryException("value_model.discretization.states must be positive"); + throw new FactoryException( + jsonField(MARKOV_DISCRETIZATION_STATES) + " must be greater than 0."); } - List thresholds = readDoubleArray(discretizationNode, "thresholds_right"); + String thresholdsField = jsonLeafField(MARKOV_DISCRETIZATION_THRESHOLDS); + List thresholds = readDoubleArray(discretizationNode, thresholdsField); if (thresholds.size() != Math.max(0, states - 1)) { throw new FactoryException( - "Discretization thresholds_right must contain " + "Discretization " + + thresholdsField + + " must contain " + Math.max(0, states - 1) + " entries for " + states + " states, but found " - + thresholds.size()); + + thresholds.size() + + "."); } ValueModel.Discretization discretization = new ValueModel.Discretization(states, thresholds); @@ -80,21 +92,24 @@ default ValueModel parseValueModel(JsonNode valueNode) { /** Parses optional parameter metadata. */ default Parameters parseParameters(JsonNode parametersNode) { String emptyRowStrategy = - parametersNode.path("transitions").path("empty_row_strategy").asString(""); + parametersNode + .path(jsonLeafField(MARKOV_PARAMETERS_TRANSITIONS)) + .path(jsonLeafField(MARKOV_EMPTY_ROW_STRATEGY)) + .asString(""); Optional transitions = emptyRowStrategy.isEmpty() ? Optional.empty() : Optional.of(new Parameters.TransitionParameters(emptyRowStrategy)); - JsonNode gmmNode = parametersNode.path("gmm"); + JsonNode gmmNode = parametersNode.path(jsonLeafField(MARKOV_PARAMETERS_GMM)); Optional gmm = gmmNode.isMissingNode() || gmmNode.isNull() || gmmNode.isEmpty() ? Optional.empty() : Optional.of( new Parameters.GmmParameters( - gmmNode.path("value_col").asString(""), - optionalInt(gmmNode, "verbose"), - optionalInt(gmmNode, "heartbeat_seconds"))); + gmmNode.path(jsonLeafField(MARKOV_GMM_VALUE_COLUMN)).asString(""), + optionalInt(gmmNode, jsonLeafField(MARKOV_GMM_VERBOSE)), + optionalInt(gmmNode, jsonLeafField(MARKOV_GMM_HEARTBEAT_SECONDS)))); return new Parameters(transitions, gmm); } @@ -102,9 +117,9 @@ default Parameters parseParameters(JsonNode parametersNode) { /** Parses the transition tensor. */ default TransitionData parseTransitions( JsonNode dataNode, int expectedBucketCount, int stateCount) { - JsonNode transitionsNode = extractNode(dataNode, "transitions"); - String dtype = extractText(transitionsNode, "dtype"); - String encoding = extractText(transitionsNode, "encoding"); + JsonNode transitionsNode = extractNode(dataNode, jsonLeafField(MARKOV_TRANSITIONS)); + String dtype = extractText(transitionsNode, jsonLeafField(MARKOV_TRANSITION_DTYPE)); + String encoding = extractText(transitionsNode, jsonLeafField(MARKOV_TRANSITION_ENCODING)); int[] shape = parseTransitionShape(transitionsNode); int buckets = shape[0]; @@ -112,7 +127,7 @@ default TransitionData parseTransitions( int columns = shape[2]; validateTransitionShape(expectedBucketCount, stateCount, buckets, rows, columns); - JsonNode valuesNode = extractNode(transitionsNode, "values"); + JsonNode valuesNode = extractNode(transitionsNode, jsonLeafField(MARKOV_TRANSITION_VALUES)); double[][][] values = parseTransitionValues(valuesNode, buckets, stateCount); return new TransitionData(dtype, encoding, values); @@ -121,17 +136,18 @@ default TransitionData parseTransitions( /** Parses GMM buckets. Individual states may be null. */ default GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { if (gmmsNode == null || gmmsNode.isMissingNode() || gmmsNode.isNull()) { - throw new FactoryException("Missing field 'gmms'"); + throw new FactoryException("Missing field '" + jsonLeafField(MARKOV_GMMS) + "'."); } - JsonNode bucketsNode = gmmsNode.get("buckets"); + JsonNode bucketsNode = gmmsNode.get(jsonLeafField(MARKOV_GMM_BUCKETS)); if (bucketsNode == null || !bucketsNode.isArray()) { - throw new FactoryException("data.gmms.buckets must be an array"); + throw new FactoryException(jsonField(MARKOV_GMM_BUCKETS) + " must be an array."); } List buckets = new ArrayList<>(); for (JsonNode bucketNode : bucketsNode) { - JsonNode statesNode = bucketNode.get("states"); + JsonNode statesNode = bucketNode.get(jsonLeafField(MARKOV_GMM_STATES)); if (statesNode == null || !statesNode.isArray()) { - throw new FactoryException("Each GMM bucket must contain an array 'states'"); + throw new FactoryException( + "Each GMM bucket must contain an array '" + jsonLeafField(MARKOV_GMM_STATES) + "'."); } List states = new ArrayList<>(); for (JsonNode stateNode : statesNode) { @@ -139,9 +155,9 @@ default GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { states.add(null); continue; } - List weights = readDoubleArray(stateNode, "weights"); - List means = readDoubleArray(stateNode, "means"); - List variances = readDoubleArray(stateNode, "variances"); + List weights = readDoubleArray(stateNode, jsonLeafField(MARKOV_GMM_WEIGHTS)); + List means = readDoubleArray(stateNode, jsonLeafField(MARKOV_GMM_MEANS)); + List variances = readDoubleArray(stateNode, jsonLeafField(MARKOV_GMM_VARIANCES)); states.add(new GmmBuckets.GmmState(weights, means, variances)); } buckets.add(new GmmBuckets.GmmBucket(Collections.unmodifiableList(states))); @@ -152,7 +168,7 @@ default GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { default JsonNode extractNode(JsonNode node, String field) { JsonNode value = node.get(field); if (value == null || value.isMissingNode()) { - throw new FactoryException("Missing field '" + field + "'"); + throw new FactoryException("Missing field '" + field + "'."); } return value; } @@ -160,10 +176,10 @@ default JsonNode extractNode(JsonNode node, String field) { default String extractText(JsonNode node, String field) { JsonNode value = node.get(field); if (value == null || value.isMissingNode() || value.isNull()) { - throw new FactoryException("Missing field '" + field + "'"); + throw new FactoryException("Missing field '" + field + "'."); } if (!value.isString()) { - throw new FactoryException("Field '" + field + "' must be textual"); + throw new FactoryException("Field '" + field + "' must be textual."); } return value.asString(); } @@ -176,10 +192,10 @@ default double extractDouble(JsonNode node, String field) { default int extractInt(JsonNode node, String field) { JsonNode value = node.get(field); if (value == null || value.isMissingNode() || value.isNull()) { - throw new FactoryException("Missing field '" + field + "'"); + throw new FactoryException("Missing field '" + field + "'."); } if (!value.canConvertToInt()) { - throw new FactoryException("Field '" + field + "' must be an integer"); + throw new FactoryException("Field '" + field + "' must be an integer."); } return value.asInt(); } @@ -188,7 +204,9 @@ default ZonedDateTime parseTimestamp(String timestamp) { try { return ZonedDateTime.parse(timestamp); } catch (DateTimeParseException e) { - throw new FactoryException("Unable to parse generated_at timestamp '" + timestamp + "'", e); + throw new FactoryException( + "Unable to parse " + jsonField(MARKOV_GENERATED_AT) + " timestamp '" + timestamp + "'.", + e); } } @@ -196,15 +214,15 @@ default OptionalInt optionalInt(JsonNode node, String field) { JsonNode value = node.get(field); if (value == null || value.isNull()) return OptionalInt.empty(); if (!value.canConvertToInt()) { - throw new FactoryException("Field '" + field + "' must be an integer"); + throw new FactoryException("Field '" + field + "' must be an integer."); } return OptionalInt.of(value.asInt()); } default int[] parseTransitionShape(JsonNode transitionsNode) { - JsonNode shapeNode = extractNode(transitionsNode, "shape"); + JsonNode shapeNode = extractNode(transitionsNode, jsonLeafField(MARKOV_TRANSITION_SHAPE)); if (!shapeNode.isArray() || shapeNode.size() != 3) { - throw new FactoryException("Transition shape must contain three dimensions"); + throw new FactoryException("Transition shape must contain three dimensions."); } return new int[] { extractInt(shapeNode, 0, "Transition shape"), @@ -220,7 +238,8 @@ default void validateTransitionShape( "Transition bucket count mismatch. Expected " + expectedBucketCount + " but was " - + buckets); + + buckets + + "."); } if (rows != stateCount || columns != stateCount) { throw new FactoryException( @@ -229,30 +248,37 @@ default void validateTransitionShape( + " but was rows=" + rows + ", columns=" - + columns); + + columns + + "."); } } default double[][][] parseTransitionValues(JsonNode valuesNode, int buckets, int stateCount) { if (!valuesNode.isArray() || valuesNode.size() != buckets) { throw new FactoryException( - "Transition values must be a three dimensional array with " + buckets + " buckets"); + "Transition values must be a three dimensional array with " + buckets + " buckets."); } double[][][] values = new double[buckets][stateCount][stateCount]; for (int b = 0; b < buckets; b++) { JsonNode bucketNode = valuesNode.get(b); if (!bucketNode.isArray()) { - throw new FactoryException("Bucket " + b + " in transition values must be an array"); + throw new FactoryException("Bucket " + b + " in transition values must be an array."); } if (bucketNode.size() != stateCount) { throw new FactoryException( - "Bucket " + b + " contained " + bucketNode.size() + " rows. Expected " + stateCount); + "Bucket " + + b + + " contained " + + bucketNode.size() + + " rows. Expected " + + stateCount + + "."); } for (int r = 0; r < stateCount; r++) { JsonNode rowNode = bucketNode.get(r); if (!rowNode.isArray()) { throw new FactoryException( - "Row " + r + " in bucket " + b + " of transition values must be an array"); + "Row " + r + " in bucket " + b + " of transition values must be an array."); } if (rowNode.size() != stateCount) { throw new FactoryException( @@ -263,12 +289,14 @@ default double[][][] parseTransitionValues(JsonNode valuesNode, int buckets, int + " had " + rowNode.size() + " columns. Expected " - + stateCount); + + stateCount + + "."); } for (int c = 0; c < stateCount; c++) { values[b][r][c] = extractDoubleValue( - rowNode.get(c), "data.transitions.values[" + b + "][" + r + "][" + c + "]"); + rowNode.get(c), + jsonField(MARKOV_TRANSITION_VALUES) + "[" + b + "][" + r + "][" + c + "]"); } } } @@ -278,7 +306,7 @@ default double[][][] parseTransitionValues(JsonNode valuesNode, int buckets, int default List readDoubleArray(JsonNode node, String field) { JsonNode arrayNode = node.get(field); if (arrayNode == null || !arrayNode.isArray()) { - throw new FactoryException("Field '" + field + "' must be an array"); + throw new FactoryException("Field '" + field + "' must be an array."); } List values = new ArrayList<>(); for (int i = 0; i < arrayNode.size(); i++) { @@ -290,24 +318,24 @@ default List readDoubleArray(JsonNode node, String field) { default int extractInt(JsonNode node, int index, String field) { JsonNode value = node.get(index); if (value == null || value.isMissingNode() || value.isNull()) { - throw new FactoryException("Missing field '" + field + "[" + index + "]'"); + throw new FactoryException("Missing field '" + field + "[" + index + "]'."); } if (!value.canConvertToInt()) { - throw new FactoryException("Field '" + field + "[" + index + "]' must be an integer"); + throw new FactoryException("Field '" + field + "[" + index + "]' must be an integer."); } return value.asInt(); } default double extractDoubleValue(JsonNode node, String field) { if (node == null || node.isMissingNode() || node.isNull()) { - throw new FactoryException("Missing field '" + field + "'"); + throw new FactoryException("Missing field '" + field + "'."); } if (!node.isNumber()) { - throw new FactoryException("Field '" + field + "' must be a double"); + throw new FactoryException("Field '" + field + "' must be a double."); } double value = node.asDouble(); if (!Double.isFinite(value)) { - throw new FactoryException("Field '" + field + "' must be finite"); + throw new FactoryException("Field '" + field + "' must be finite."); } return value; } @@ -319,10 +347,20 @@ default Optional parsePowerReference( return Optional.empty(); } if (!referenceNode.isObject()) { - throw new FactoryException("Field '" + field + "' must be an object"); + throw new FactoryException("Field '" + field + "' must be an object."); } - double value = extractDouble(referenceNode, "value"); - String unit = extractText(referenceNode, "unit"); + double value = extractDouble(referenceNode, jsonLeafField(MARKOV_MAX_POWER_VALUE)); + String unit = extractText(referenceNode, jsonLeafField(MARKOV_MAX_POWER_UNIT)); return Optional.of(new ValueModel.Normalization.PowerReference(value, unit)); } + + default String jsonField(String field) { + return StringUtils.camelCaseToSnakeCase(field); + } + + default String jsonLeafField(String field) { + int lastSeparatorIndex = field.lastIndexOf('.'); + String leaf = lastSeparatorIndex < 0 ? field : field.substring(lastSeparatorIndex + 1); + return jsonField(leaf); + } } diff --git a/src/main/java/edu/ie3/datamodel/io/naming/FieldNamingStrategy.java b/src/main/java/edu/ie3/datamodel/io/naming/FieldNamingStrategy.java index f72f98501a..20daa4c700 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/FieldNamingStrategy.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/FieldNamingStrategy.java @@ -224,16 +224,40 @@ public class FieldNamingStrategy { // markov - nested fields required for simulation public static final String MARKOV_GENERATOR_NAME = "generator.name"; public static final String MARKOV_GENERATOR_VERSION = "generator.version"; + public static final String MARKOV_GENERATOR_CONFIG = "generator.config"; public static final String MARKOV_BUCKET_COUNT = "timeModel.bucketCount"; + public static final String MARKOV_BUCKET_ENCODING = "timeModel.bucketEncoding"; + public static final String MARKOV_BUCKET_ENCODING_FORMULA = "timeModel.bucketEncoding.formula"; public static final String MARKOV_SAMPLING_INTERVAL = "timeModel.samplingIntervalMinutes"; public static final String MARKOV_TIMEZONE = "timeModel.timezone"; + public static final String MARKOV_VALUE_UNIT = "valueModel.valueUnit"; + public static final String MARKOV_NORMALIZATION = "valueModel.normalization"; + public static final String MARKOV_NORMALIZATION_METHOD = "valueModel.normalization.method"; + public static final String MARKOV_MAX_POWER = "valueModel.normalization.maxPower"; + public static final String MARKOV_DISCRETIZATION = "valueModel.discretization"; public static final String MARKOV_DISCRETIZATION_STATES = "valueModel.discretization.states"; public static final String MARKOV_DISCRETIZATION_THRESHOLDS = "valueModel.discretization.thresholdsRight"; public static final String MARKOV_MAX_POWER_VALUE = "valueModel.normalization.maxPower.value"; public static final String MARKOV_MAX_POWER_UNIT = "valueModel.normalization.maxPower.unit"; + public static final String MARKOV_MIN_POWER = "valueModel.normalization.minPower"; public static final String MARKOV_MIN_POWER_VALUE = "valueModel.normalization.minPower.value"; public static final String MARKOV_MIN_POWER_UNIT = "valueModel.normalization.minPower.unit"; + public static final String MARKOV_PARAMETERS_TRANSITIONS = "parameters.transitions"; + public static final String MARKOV_EMPTY_ROW_STRATEGY = "parameters.transitions.emptyRowStrategy"; + public static final String MARKOV_PARAMETERS_GMM = "parameters.gmm"; + public static final String MARKOV_GMM_VALUE_COLUMN = "parameters.gmm.valueCol"; + public static final String MARKOV_GMM_VERBOSE = "parameters.gmm.verbose"; + public static final String MARKOV_GMM_HEARTBEAT_SECONDS = "parameters.gmm.heartbeatSeconds"; + public static final String MARKOV_TRANSITIONS = "data.transitions"; + public static final String MARKOV_TRANSITION_DTYPE = "data.transitions.dtype"; + public static final String MARKOV_TRANSITION_ENCODING = "data.transitions.encoding"; + public static final String MARKOV_TRANSITION_SHAPE = "data.transitions.shape"; public static final String MARKOV_TRANSITION_VALUES = "data.transitions.values"; + public static final String MARKOV_GMMS = "data.gmms"; public static final String MARKOV_GMM_BUCKETS = "data.gmms.buckets"; + public static final String MARKOV_GMM_STATES = "data.gmms.buckets.states"; + public static final String MARKOV_GMM_WEIGHTS = "data.gmms.buckets.states.weights"; + public static final String MARKOV_GMM_MEANS = "data.gmms.buckets.states.means"; + public static final String MARKOV_GMM_VARIANCES = "data.gmms.buckets.states.variances"; } diff --git a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java index 49ed3a9bff..7f3101fa01 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java @@ -18,7 +18,7 @@ public LoadProfileMetaInformation(String profileKey) { } public LoadProfileMetaInformation(String profileKey, String type) { - this(profileKey, parseType(type)); + this(profileKey, PowerProfileKey.Type.parse(type)); } public LoadProfileMetaInformation(String profileKey, PowerProfileKey.Type type) { @@ -35,18 +35,6 @@ public PowerProfileKey getProfileKey() { return profileKey; } - private static PowerProfileKey.Type parseType(String type) { - return switch (type) { - case "lpts" -> PowerProfileKey.Type.TS; - case "markov" -> PowerProfileKey.Type.MARKOV; - default -> - throw new IllegalArgumentException( - "The given type '" - + type - + "' is not supported for load profile time series meta information."); - }; - } - @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java b/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java index 1bda47c848..530e8a6a8e 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java @@ -8,7 +8,6 @@ import edu.ie3.datamodel.models.profile.PowerProfileKey; import edu.ie3.datamodel.models.value.PValue; import java.time.ZonedDateTime; -import java.util.Objects; import java.util.Optional; import java.util.OptionalDouble; import java.util.OptionalInt; @@ -95,9 +94,6 @@ record MarkovIdentifier( implements PowerValueIdentifier { public MarkovIdentifier { - Objects.requireNonNull(time, "time"); - Objects.requireNonNull(previousState, "previousState"); - Objects.requireNonNull(initialNormalizedValue, "initialNormalizedValue"); if (previousState.isEmpty() && initialNormalizedValue.isEmpty()) { throw new IllegalArgumentException( "Need either previous state or an initial normalized value to start the Markov chain."); diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java index 3727b6bd9b..eb6599b4ef 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java @@ -19,7 +19,6 @@ import edu.ie3.datamodel.models.profile.PowerProfileKey; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; import java.time.ZonedDateTime; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Supplier; @@ -51,9 +50,9 @@ public JsonMarkovProfileSource( JsonDataSource dataSource, FileLoadProfileMetaInformation metaInformation, MarkovLoadModelFactory factory) { - this.dataSource = Objects.requireNonNull(dataSource, "dataSource"); - this.metaInformation = Objects.requireNonNull(metaInformation, "metaInformation"); - this.factory = Objects.requireNonNull(factory, "factory"); + this.dataSource = dataSource; + this.metaInformation = metaInformation; + this.factory = factory; if (metaInformation.getFileType() != FileType.JSON) { throw new IllegalArgumentException("Markov profile source requires JSON meta information."); } diff --git a/src/main/java/edu/ie3/datamodel/models/profile/PowerProfileKey.java b/src/main/java/edu/ie3/datamodel/models/profile/PowerProfileKey.java index 38222d1a74..aa9fa3cee3 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/PowerProfileKey.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/PowerProfileKey.java @@ -84,6 +84,18 @@ public static String toUniformKey(String key) { public enum Type { TS, - MARKOV + MARKOV; + + public static Type parse(String type) { + return switch (type) { + case "lpts" -> TS; + case "markov" -> MARKOV; + default -> + throw new IllegalArgumentException( + "The given type '" + + type + + "' is not supported for load profile time series meta information."); + }; + } } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy index 7154dfddcf..ae678dde55 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy @@ -6,12 +6,10 @@ package edu.ie3.datamodel.io.factory.markov import edu.ie3.datamodel.exceptions.FactoryException -import spock.lang.Specification -import tools.jackson.databind.ObjectMapper +import edu.ie3.datamodel.models.profile.markov.MarkovModelJsonTestSupport import tools.jackson.databind.node.ObjectNode -class MarkovLoadModelFactoryTest extends Specification { - private final ObjectMapper objectMapper = new ObjectMapper() +class MarkovLoadModelFactoryTest extends MarkovModelJsonTestSupport { private final MarkovLoadModelFactory factory = new MarkovLoadModelFactory() def "buildModel returns parsed Markov load model from valid JSON"() { @@ -189,71 +187,4 @@ class MarkovLoadModelFactoryTest extends Specification { then: thrown(FactoryException) } - - private static String validModelJson() { - return """ - { - "schema": "markov.load.v1", - "generated_at": "2025-01-01T00:00:00Z", - "generator": { - "name": "simonaMarkovLoad", - "version": "1.0.0", - "config": { "foo": "bar" } - }, - "time_model": { - "bucket_count": 1, - "bucket_encoding": { "formula": "hour_of_day" }, - "sampling_interval_minutes": 60, - "timezone": "UTC" - }, - "value_model": { - "value_unit": "normalized", - "normalization": { - "method": "none", - "max_power": { "value": 1.5, "unit": "kW" }, - "min_power": { "value": 0.1, "unit": "kW" } - }, - "discretization": { - "states": 2, - "thresholds_right": [0.5] - } - }, - "parameters": { - "transitions": { "empty_row_strategy": "fill" }, - "gmm": { - "value_col": "p", - "verbose": 1, - "heartbeat_seconds": 5 - } - }, - "data": { - "transitions": { - "dtype": "float32", - "encoding": "nested_lists", - "shape": [1,2,2], - "values": [ - [ - [0.1, 0.9], - [0.3, 0.7] - ] - ] - }, - "gmms": { - "buckets": [ - { - "states": [ - { - "weights": [1.0], - "means": [1.0], - "variances": [0.2] - }, - null - ] - } - ] - } - } - } - """.stripIndent() - } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy index a543323eaf..737bf09bf9 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy @@ -13,8 +13,7 @@ import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation import edu.ie3.datamodel.io.source.PowerValueSource import edu.ie3.datamodel.models.StandardUnits import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel -import spock.lang.Specification -import tools.jackson.databind.ObjectMapper +import edu.ie3.datamodel.models.profile.markov.MarkovModelJsonTestSupport import tools.jackson.databind.node.ObjectNode import java.nio.file.Files @@ -23,7 +22,7 @@ import java.time.ZonedDateTime import java.util.concurrent.atomic.AtomicInteger import java.util.function.Function -class JsonMarkovProfileSourceTest extends Specification { +class JsonMarkovProfileSourceTest extends MarkovModelJsonTestSupport { Path tempDir Path jsonFile @@ -41,7 +40,7 @@ class JsonMarkovProfileSourceTest extends Specification { } } - def "getModel reads and caches Markov model from JSON file"() { + def "validate and getModel read Markov model from JSON file"() { given: Files.writeString(jsonFile, validModelJson()) def source = new JsonMarkovProfileSource( @@ -50,18 +49,18 @@ class JsonMarkovProfileSourceTest extends Specification { ) when: - MarkovLoadModel modelFirst = source.getModel() - MarkovLoadModel modelSecond = source.getModel() + source.validate() then: - modelFirst.is(modelSecond) // cached instance reused - modelFirst.schema() == "markov.load.v1" noExceptionThrown() - when: "validation is executed on the same file" - source.validate() + when: + MarkovLoadModel modelFirst = source.getModel() + MarkovLoadModel modelSecond = source.getModel() then: + modelFirst.is(modelSecond) // cached instance reused + modelFirst.schema() == "markov.load.v1" noExceptionThrown() } @@ -89,7 +88,7 @@ class JsonMarkovProfileSourceTest extends Specification { def "validate tolerates a missing optional parameters block"() { given: - def root = new ObjectMapper().readTree(validModelJson()) + def root = objectMapper.readTree(validModelJson()) ((ObjectNode) root).remove("parameters") Files.writeString(jsonFile, root.toString()) def source = new JsonMarkovProfileSource( @@ -148,74 +147,24 @@ class JsonMarkovProfileSourceTest extends Specification { output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 5.25d } - private static String validModelJson() { + protected static String validModelJson() { + return markovModelJson(defaultTransitionValues(), sourceGmmStates(), "10.0", "0.5") + } + + private static String sourceGmmStates() { return """ - { - "schema": "markov.load.v1", - "generated_at": "2025-01-01T00:00:00Z", - "generator": { - "name": "simonaMarkovLoad", - "version": "1.0.0", - "config": { "foo": "bar" } - }, - "time_model": { - "bucket_count": 1, - "bucket_encoding": { "formula": "hour_of_day" }, - "sampling_interval_minutes": 60, - "timezone": "UTC" - }, - "value_model": { - "value_unit": "normalized", - "normalization": { - "method": "none", - "max_power": { "value": 10.0, "unit": "kW" }, - "min_power": { "value": 0.5, "unit": "kW" } - }, - "discretization": { - "states": 2, - "thresholds_right": [0.5] - } - }, - "parameters": { - "transitions": { "empty_row_strategy": "fill" }, - "gmm": { - "value_col": "p", - "verbose": 1, - "heartbeat_seconds": 5 - } + [ + { + "weights": [1.0], + "means": [1.0], + "variances": [0.0] }, - "data": { - "transitions": { - "dtype": "float32", - "encoding": "nested_lists", - "shape": [1,2,2], - "values": [ - [ - [0.1, 0.9], - [0.3, 0.7] - ] - ] - }, - "gmms": { - "buckets": [ - { - "states": [ - { - "weights": [1.0], - "means": [1.0], - "variances": [0.0] - }, - { - "weights": [1.0], - "means": [0.5], - "variances": [0.0] - } - ] - } - ] - } + { + "weights": [1.0], + "means": [0.5], + "variances": [0.0] } - } + ] """.stripIndent() } } diff --git a/src/test/groovy/edu/ie3/datamodel/models/profile/PowerProfileKeyTest.groovy b/src/test/groovy/edu/ie3/datamodel/models/profile/PowerProfileKeyTest.groovy index 472297bb8e..6290645f42 100644 --- a/src/test/groovy/edu/ie3/datamodel/models/profile/PowerProfileKeyTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/models/profile/PowerProfileKeyTest.groovy @@ -131,4 +131,22 @@ class PowerProfileKeyTest extends Specification { timeSeriesKey != markovKey timeSeriesKey.hashCode() != markovKey.hashCode() } + + def "Power profile key type is parsed correctly from file prefix"() { + expect: + PowerProfileKey.Type.parse(type) == expected + + where: + type || expected + "lpts" || PowerProfileKey.Type.TS + "markov" || PowerProfileKey.Type.MARKOV + } + + def "Power profile key type parsing rejects unsupported file prefix"() { + when: + PowerProfileKey.Type.parse("unsupported") + + then: + thrown(IllegalArgumentException) + } } diff --git a/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy b/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy index d0eefef4ea..70cd5742ff 100644 --- a/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy @@ -9,14 +9,11 @@ import edu.ie3.datamodel.io.factory.markov.MarkovLoadModelFactory import edu.ie3.datamodel.io.factory.markov.MarkovModelData import edu.ie3.datamodel.io.source.PowerValueSource import edu.ie3.datamodel.models.StandardUnits -import spock.lang.Specification -import tools.jackson.databind.ObjectMapper import java.time.ZonedDateTime -class MarkovLoadModelTest extends Specification { +class MarkovLoadModelTest extends MarkovModelJsonTestSupport { - private final ObjectMapper objectMapper = new ObjectMapper() private final MarkovLoadModelFactory factory = new MarkovLoadModelFactory() def "supplier scales deterministic normalized values and exposes next state"() { @@ -118,7 +115,7 @@ class MarkovLoadModelTest extends Specification { } private loadModel(String transitions, String states) { - def json = modelJson(transitions, states) + def json = markovModelJson(transitions, states, "5.0", "1.0") def root = objectMapper.readTree(json) factory.get(new MarkovModelData(root)).getOrThrow() } @@ -161,62 +158,4 @@ class MarkovLoadModelTest extends Specification { ] """.stripIndent() } - - private static String modelJson(String transitions, String states) { - def normalization = """ - { - "method": "none", - "max_power": { "value": 5.0, "unit": "kW" }, - "min_power": { "value": 1.0, "unit": "kW" } - } - """ - return """ - { - "schema": "markov.load.v1", - "generated_at": "2025-01-01T00:00:00Z", - "generator": { - "name": "simonaMarkovLoad", - "version": "1.0.0", - "config": { "foo": "bar" } - }, - "time_model": { - "bucket_count": 1, - "bucket_encoding": { "formula": "hour_of_day" }, - "sampling_interval_minutes": 60, - "timezone": "UTC" - }, - "value_model": { - "value_unit": "normalized", - "normalization": $normalization, - "discretization": { - "states": 2, - "thresholds_right": [0.5] - } - }, - "parameters": { - "transitions": { "empty_row_strategy": "fill" }, - "gmm": { - "value_col": "p", - "verbose": 1, - "heartbeat_seconds": 5 - } - }, - "data": { - "transitions": { - "dtype": "float32", - "encoding": "nested_lists", - "shape": [1,2,2], - "values": $transitions - }, - "gmms": { - "buckets": [ - { - "states": $states - } - ] - } - } - } - """.stripIndent() - } } diff --git a/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovModelJsonTestSupport.groovy b/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovModelJsonTestSupport.groovy new file mode 100644 index 0000000000..aac14b2538 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovModelJsonTestSupport.groovy @@ -0,0 +1,102 @@ +/* + * © 2026. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.models.profile.markov + +import spock.lang.Specification +import tools.jackson.databind.ObjectMapper + +abstract class MarkovModelJsonTestSupport extends Specification { + + protected final ObjectMapper objectMapper = new ObjectMapper() + + protected static String validModelJson() { + return markovModelJson(defaultTransitionValues(), defaultGmmStates()) + } + + protected static String markovModelJson( + String transitions, + String states, + String maxPower = "1.5", + String minPower = "0.1" + ) { + return """ + { + "schema": "markov.load.v1", + "generated_at": "2025-01-01T00:00:00Z", + "generator": { + "name": "simonaMarkovLoad", + "version": "1.0.0", + "config": { "foo": "bar" } + }, + "time_model": { + "bucket_count": 1, + "bucket_encoding": { "formula": "hour_of_day" }, + "sampling_interval_minutes": 60, + "timezone": "UTC" + }, + "value_model": { + "value_unit": "normalized", + "normalization": { + "method": "none", + "max_power": { "value": $maxPower, "unit": "kW" }, + "min_power": { "value": $minPower, "unit": "kW" } + }, + "discretization": { + "states": 2, + "thresholds_right": [0.5] + } + }, + "parameters": { + "transitions": { "empty_row_strategy": "fill" }, + "gmm": { + "value_col": "p", + "verbose": 1, + "heartbeat_seconds": 5 + } + }, + "data": { + "transitions": { + "dtype": "float32", + "encoding": "nested_lists", + "shape": [1,2,2], + "values": $transitions + }, + "gmms": { + "buckets": [ + { + "states": $states + } + ] + } + } + } + """.stripIndent() + } + + protected static String defaultTransitionValues() { + return """ + [ + [ + [0.1, 0.9], + [0.3, 0.7] + ] + ] + """.stripIndent() + } + + protected static String defaultGmmStates() { + return """ + [ + { + "weights": [1.0], + "means": [1.0], + "variances": [0.2] + }, + null + ] + """.stripIndent() + } +} From a9d81ab4ddf2635ad784d2e6c2202e39dd33b52d Mon Sep 17 00:00:00 2001 From: Philipp Date: Thu, 9 Jul 2026 12:07:11 +0200 Subject: [PATCH 38/39] Changed Comment --- .../datamodel/io/naming/EntityPersistenceNamingStrategy.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java b/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java index 7cd31efbab..9e507dde87 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java @@ -56,9 +56,8 @@ public class EntityPersistenceNamingStrategy { protected final Pattern individualTimeSeriesPattern; /** - * Regex to match the naming convention of a file for a repetitive load profile time series. The - * profile type is accessible via the named capturing group "type", the profile by the group - * "profile" + * Regex to match the naming convention of a load profile source file. The profile type is + * accessible via the named capturing group "type", the profile by the group "profile" */ private static final String LOAD_PROFILE_TIME_SERIES = "(?lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})"; From dfbe093deec3deaa0d25b98077b2667a6a0a56cc Mon Sep 17 00:00:00 2001 From: Philipp Date: Thu, 9 Jul 2026 21:42:52 +0200 Subject: [PATCH 39/39] SonarQube Complexity Warning --- .../markov/MarkovModelParsingSupport.java | 72 +++++++++++-------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java index 96c5e6e737..16f30bf209 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -260,38 +260,9 @@ default double[][][] parseTransitionValues(JsonNode valuesNode, int buckets, int } double[][][] values = new double[buckets][stateCount][stateCount]; for (int b = 0; b < buckets; b++) { - JsonNode bucketNode = valuesNode.get(b); - if (!bucketNode.isArray()) { - throw new FactoryException("Bucket " + b + " in transition values must be an array."); - } - if (bucketNode.size() != stateCount) { - throw new FactoryException( - "Bucket " - + b - + " contained " - + bucketNode.size() - + " rows. Expected " - + stateCount - + "."); - } + JsonNode bucketNode = extractTransitionBucket(valuesNode, b, stateCount); for (int r = 0; r < stateCount; r++) { - JsonNode rowNode = bucketNode.get(r); - if (!rowNode.isArray()) { - throw new FactoryException( - "Row " + r + " in bucket " + b + " of transition values must be an array."); - } - if (rowNode.size() != stateCount) { - throw new FactoryException( - "Row " - + r - + " in bucket " - + b - + " had " - + rowNode.size() - + " columns. Expected " - + stateCount - + "."); - } + JsonNode rowNode = extractTransitionRow(bucketNode, b, r, stateCount); for (int c = 0; c < stateCount; c++) { values[b][r][c] = extractDoubleValue( @@ -303,6 +274,45 @@ default double[][][] parseTransitionValues(JsonNode valuesNode, int buckets, int return values; } + default JsonNode extractTransitionBucket(JsonNode valuesNode, int bucket, int stateCount) { + JsonNode bucketNode = valuesNode.get(bucket); + if (!bucketNode.isArray()) { + throw new FactoryException("Bucket " + bucket + " in transition values must be an array."); + } + if (bucketNode.size() != stateCount) { + throw new FactoryException( + "Bucket " + + bucket + + " contained " + + bucketNode.size() + + " rows. Expected " + + stateCount + + "."); + } + return bucketNode; + } + + default JsonNode extractTransitionRow(JsonNode bucketNode, int bucket, int row, int stateCount) { + JsonNode rowNode = bucketNode.get(row); + if (!rowNode.isArray()) { + throw new FactoryException( + "Row " + row + " in bucket " + bucket + " of transition values must be an array."); + } + if (rowNode.size() != stateCount) { + throw new FactoryException( + "Row " + + row + + " in bucket " + + bucket + + " had " + + rowNode.size() + + " columns. Expected " + + stateCount + + "."); + } + return rowNode; + } + default List readDoubleArray(JsonNode node, String field) { JsonNode arrayNode = node.get(field); if (arrayNode == null || !arrayNode.isArray()) {