diff --git a/CHANGELOG.md b/CHANGELOG.md index 01a0f960c..83feac6d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased/Snapshot] ### Added +- Added support for Markov-chain-based load profiles loaded from JSON [#1472](https://github.com/ie3-institute/PowerSystemDataModel/issues/1472) ### Fixed diff --git a/build.gradle b/build.gradle index 5cc6c26f2..227243036 100644 --- a/build.gradle +++ b/build.gradle @@ -102,6 +102,8 @@ dependencies { implementation 'commons-io:commons-io:2.22.0' // I/O functionalities implementation 'commons-codec:commons-codec:1.22.0' // needed by commons-compress implementation 'org.apache.commons:commons-compress:1.28.0' // I/O functionalities + + implementation 'tools.jackson.core:jackson-databind:3.0.3' } tasks.withType(JavaCompile).configureEach { diff --git a/docs/readthedocs/io/basiciousage.md b/docs/readthedocs/io/basiciousage.md index a1c0cf2b4..beab9600c 100644 --- a/docs/readthedocs/io/basiciousage.md +++ b/docs/readthedocs/io/basiciousage.md @@ -10,6 +10,7 @@ maxdepth: 2 --- naming csvfiles +markov sql influxdb ValidationUtils.md diff --git a/docs/readthedocs/io/csvfiles.md b/docs/readthedocs/io/csvfiles.md index 4904a22f5..3077c4f3a 100644 --- a/docs/readthedocs/io/csvfiles.md +++ b/docs/readthedocs/io/csvfiles.md @@ -89,6 +89,100 @@ You may extend / alter the naming with pre- or suffix by calling `new EntityPers - *prefix_* lpts *_profileKey* *_suffix* ``` +#### Individual Time Series + +Let's spend a few more words on the individual time series: +Those files are meant to carry different types of content - one might give information about wholesale market prices, +the other is a record of power values provided by a real system. +To be able to understand, what's inside of the file, the *columnScheme* part of the file name gives insight of its +content. + +For example, you have an IndividualTimeSeries CSV file for energy prices, then you use the key `c` from the table below +for columnScheme `its_c_2fcb3e53-b94a-4b96-bea4-c469e499f1a1.csv`. +The CSV file must then have the appropriate format for the key `c` : + +```text + time,price + 2020-01-01T00:00:00Z,100.0 +``` + +The CSV file requires a unique identification number. +The UUID (Universally Unique Identifier) can be created [here](https://www.uuidgenerator.net/). +You can also use the Method `java.util.UUID#randomUUID` to create a UUID. +This is the UUID from the example above `2fcb3e53-b94a-4b96-bea4-c469e499f1a1`. + +The following keys are supported until now: +```{list-table} + :widths: auto + :class: wrapping + :header-rows: 1 + + * - Key + - Information and supported head line. + * - c + - An energy price (e.g. in €/MWh; c stands for charge). + Permissible head line: ``time,price`` + * - p + - Active power. + Permissible head line: ``time,p`` + * - pq + - Active and reactive power. + Permissible head line: ``time,p,q`` + * - h + - Heat power demand. + Permissible head line: ``time,h`` + * - ph + - Active and heat power. + Permissible head line: ``time,p,h`` + * - pqh + - Active, reactive and heat power. + Permissible head line: ``time,p,q,h`` + * - v + - Voltage magnitude in pu and angle in °. + Permissible head line: ``time,vMag,vAng`` + * - weather + - Weather information. + Permissible head line: ``time,coordinate,direct_irradiation,diffuse_irradiation,temperature,wind_velocity,wind_direction`` + +``` + + +##### Load Profile Time Series + +The following profiles are supported until now: + +Note: +Load profile keys must be unique across all load-profile sources. Do not use the same profile key +for both CSV (lpts_*) and Markov JSON (markov_*), e.g. avoid having both lpts_h0.csv and +markov_h0.json. + +Markov-based load models (`markov_*`) do not support energy scaling. Calls to +`getProfileEnergyScaling()` will always return `Optional.empty()`. + +```{list-table} + :widths: auto + :class: wrapping + :header-rows: 1 + + * - Key + - Information + - Supported head line. + * - e.g.: H0 + - BDEW standard load profiles 1999 ([source](https://www.bdew.de/energie/standardlastprofile-strom/)) + - Permissible head line: ``SuSa,SuSu,SuWd,TrSa,TrSu,TrWd,WiSa,WiSu,WiWd,quarterHour`` + * - e.g.: h25 + - BDEW standard load profiles 2025 ([source](https://www.bdew.de/energie/standardlastprofile-strom/)) + - Permissible head line: ``janSa,janSu,janWd,febSa,febSu,febWd,marSa,marSu,marWd,aprSa,aprSu,aprWd,maySa,maySu,mayWd,junSa,junSu,junWd,julSa,julSu,julWd,augSa,augSu,augWd,sepSa,sepSu,sepWd,octSa,octSu,octWd,novSa,novSu,novWd,decSa,decSu,decWd,quarterHour`` + * - random + - A random load profile based on: ``Kays - Agent-based simulation environment for improving the planning of distribution grids`` + - Permissible head line: ``kSa,kSu,kWd,mySa,mySu,myWd,sigmaSa,sigmaSu,sigmaWd,quarterHour`` + * - markov + - Markov-chain-based stochastic load profiles from JSON files (``markov_.json``). + See {ref}`Markov-based Load Profiles ` for the full JSON schema documentation. + - Not a CSV format. Uses JSON with schema ``simonaMarkovLoad:psdm:1.0``. + +``` + ### Results ```{list-table} @@ -240,4 +334,4 @@ occur. We consider either regular directories or compressed [tarball archives](https://en.wikipedia.org/wiki/Tar_(computing)) (`*.tar.gz`) as source of input files. -The class `TarballUtils` offers some helpful functions to compress or extract input data files for easier shipping. \ No newline at end of file +The class `TarballUtils` offers some helpful functions to compress or extract input data files for easier shipping. diff --git a/docs/readthedocs/io/markov.md b/docs/readthedocs/io/markov.md new file mode 100644 index 000000000..a50218ff1 --- /dev/null +++ b/docs/readthedocs/io/markov.md @@ -0,0 +1,298 @@ +(markov-load-model)= + +# Markov-based Load Profiles + +Markov-based load profiles allow stochastic load simulation using Markov chains combined with Gaussian Mixture Models +(GMMs). The models are produced by an external Python trainer +([simonaMarkovLoad](https://github.com/ie3-institute/simonaMarkovLoad)) and consumed by PSDM via JSON files. + +## Overview + +Each Markov model encodes: + +- A set of **discrete states** representing different load levels +- **Transition matrices** per time bucket that describe the probability of moving from one state to another +- **GMM parameters** per state and bucket that allow sampling a continuous normalized power value within a state + +During simulation, one step works as follows: + +1. **Bucket lookup** - determine the current time bucket from month, weekday/weekend and quarter-hour +2. **State resolution** - use the previous state (or discretize the initial value for the first step) +3. **Seed derivation** - compute a deterministic RNG seed for reproducibility +4. **Transition + sampling** - draw the next state from the transition row, then sample a normalized value from the GMM +5. **Denormalization** - scale the normalized value back to physical power using {code}`minPower` and {code}`maxPower` + +## File Naming + +Markov model files follow the naming convention {code}`markov_.json`, e.g. {code}`markov_h0.json`. +A profile key consists of 1 to 11 letters, optionally followed by up to 3 digits. +The profile key must be unique across all load-profile sources. Do not use the same key for both a CSV profile +({code}`lpts_.csv`) and a Markov profile ({code}`markov_.json`). + +## JSON Schema + +The JSON file uses the schema identifier {code}`simonaMarkovLoad:psdm:1.0` and has the following top-level structure. +Note that PSDM only requires the {code}`schema` field to be present; its value is currently not verified. + +```text +{ + "schema": "simonaMarkovLoad:psdm:1.0", + "generated_at": "2025-01-15T12:00:00Z", + "generator": { ... }, + "time_model": { ... }, + "value_model": { ... }, + "parameters": { ... }, + "data": { ... } +} +``` + +### generator + +Metadata about the tool that produced the model. + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - name + - String + - Name of the trainer (e.g. {code}`simonaMarkovLoad`) + * - version + - String + - Trainer version + * - config + - Object + - Key-value pairs of the training configuration +``` + +### time_model + +Describes how time is mapped to buckets. + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - bucket_count + - Integer + - Total number of time buckets (typically 2304 = 12 months x 2 day types x 96 quarter-hours) + * - bucket_encoding + - Object + - Contains a {code}`formula` field documenting the bucket index calculation + * - sampling_interval_minutes + - Integer + - Time step length in minutes (typically 15) + * - timezone + - String + - IANA timezone identifier (e.g. {code}`Europe/Berlin`) +``` + +The bucket index is computed as: + +``` +bucket = month * 192 + isWeekend * 96 + quarterHour +``` + +where {code}`month` is the 0-based month (January = 0), {code}`isWeekend` is 1 for Saturday/Sunday and 0 otherwise, +and {code}`quarterHour` is the 0-based quarter-hour of the day (0..95). Timestamps are converted to the model's +timezone before the bucket index is computed. + +### value_model + +Defines the value space and normalization. + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - value_unit + - String + - Unit of the stored values; the trainer exports {code}`normalized` ([0, 1] scale) + * - normalization + - Object + - See below + * - discretization + - Object + - See below +``` + +#### normalization + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - method + - String + - Normalization strategy used during training (e.g. {code}`minmax_global`) + * - max_power + - Object + - Upper bound: {code}`{"value": , "unit": "kW"}` + * - min_power + - Object + - Lower bound: {code}`{"value": , "unit": "kW"}` +``` + +Denormalization in PSDM uses these bounds as: + +``` +power = minPower + normalizedValue * (maxPower - minPower) +``` + +Negative {code}`min_power` values are valid and represent net feed-in (e.g. PV households). + +Only {code}`kW` is supported as unit for both bounds, other units are rejected when the model is loaded. +Furthermore, {code}`max_power` must be strictly greater than {code}`min_power`. + +#### discretization + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - states + - Integer + - Number of discrete Markov states + * - thresholds_right + - Array of Double + - Right-edge thresholds for mapping normalized values to states (length = states - 1). Bins are + left-closed and right-open: a value exactly on a threshold maps to the higher state +``` + +### parameters + +Optional metadata about how the trainer produced transitions and GMMs. The block itself may be missing or empty; +all fields within it are optional. + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - transitions + - Object + - Contains {code}`empty_row_strategy`: how the trainer handled transition rows with no observations + * - gmm + - Object + - Contains {code}`value_col` (column name), {code}`verbose` (logging level), and + {code}`heartbeat_seconds` (trainer watchdog interval, not used at runtime) +``` + +### data + +Contains the actual model data: transition probabilities and GMM parameters. + +#### data.transitions + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - dtype + - String + - Data type (e.g. {code}`float32`) + * - encoding + - String + - Encoding format (e.g. {code}`nested_lists`) + * - shape + - Array of Integer + - Dimensions: {code}`[bucket_count, states, states]` + * - values + - 3D Array of Double + - Transition probabilities: {code}`values[bucket][fromState][toState]` +``` + +Each row {code}`values[bucket][state]` is a probability distribution over the next states (sums to 1.0). + +#### data.gmms + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - buckets + - Array of Bucket + - One entry per time bucket, each containing a {code}`states` array +``` + +Each bucket contains a {code}`states` array with one entry per Markov state. Each state entry is either {code}`null` +(if no GMM was fitted for that state/bucket combination) or an object with: + +```{list-table} + :widths: auto + :header-rows: 1 + + * - Field + - Type + - Description + * - weights + - Array of Double + - Component weights (sum to 1.0) + * - means + - Array of Double + - Component means in normalized [0, 1] space + * - variances + - Array of Double + - Component variances +``` + +During sampling, a GMM component is drawn according to {code}`weights`, then a value is sampled from the corresponding +Gaussian and clamped to [0, 1] before denormalization. + +If the drawn state has no GMM data ({code}`null`), the sampled normalized value is 0, which denormalizes to +{code}`min_power`. The trainer guarantees that states without GMM data are never reachable with non-zero +transition probability, so this case does not occur. + +## Loading and Validation + +Markov models are loaded via {code}`JsonMarkovProfileSource`, which: + +- Reads the JSON file lazily on first access (not at construction time) +- Caches the parsed {code}`MarkovLoadModel` for subsequent calls +- Offers validation of all mandatory fields via {code}`validate()`. Note that this is a separate call: + it is not triggered automatically when the model is loaded +- Is thread-safe ({code}`synchronized` lazy loading) + +Markov models do not support energy scaling: {code}`getProfileEnergyScaling()` always returns +{code}`Optional.empty()`. + +## Simulation Input and Output + +Power values are requested via {code}`PowerValueSource.MarkovIdentifier`, which carries: + +- {code}`time` - the timestamp of the requested step +- {code}`previousState` - the state returned by the previous step (used for all subsequent steps) +- {code}`initialNormalizedValue` - a normalized start value that is discretized into the initial state + (used for the first step only) +- {code}`randomSeed` - the base seed for reproducible sampling + +Either {code}`previousState` or {code}`initialNormalizedValue` has to be provided. Each step returns a +{code}`MarkovOutputValue` containing the sampled power value and the {code}`nextState`, which callers pass +into the identifier of the following step. The model itself is stateless, so a single instance can serve +multiple independent chains. + +Sampling is deterministic: the RNG seed of a step is derived from the request seed, the time bucket, the +current state and the time slot, so identical inputs always produce identical results. diff --git a/docs/readthedocs/models/input/participant/load.md b/docs/readthedocs/models/input/participant/load.md index a6c1214fe..13fba5a2b 100644 --- a/docs/readthedocs/models/input/participant/load.md +++ b/docs/readthedocs/models/input/participant/load.md @@ -91,7 +91,11 @@ Wasserwirtschaft; engl. Federal Association of the Energy and Water Industry). F [the corresponding website (German only)](https://www.bdew.de/energie/standardlastprofile-strom/). Furthermore, there are {code}`TemperatureDependantLoadProfiles` which can be used to note usage of load profiles for night heating storages or heat pumps for example. -The profiles rely on the VDN description for interruptable loads. -For more details see [here (German only)](https://www.bdew.de/media/documents/LPuVe-Praxisleitfaden.pdf). -{code}`NbwTemperatureDependantLoadProfiles` provides sample temperature dependant load profiles that can be used. -The `NbwTemperatureDependantLoadProfiles` consists of load profiles "ep1" for heat pumps and "ez2" for night storage heating. +The profiles rely on the VDN description for interruptable loads. +For more details see [here (German only)](https://www.bdew.de/media/documents/LPuVe-Praxisleitfaden.pdf). +{code}`NbwTemperatureDependantLoadProfiles` provides sample temperature dependant load profiles that can be used. +The `NbwTemperatureDependantLoadProfiles` consists of load profiles "ep1" for heat pumps and "ez2" for night storage heating. + +Markov-chain-based load profiles can be used for stochastic load simulation. These profiles are produced +by an external Python trainer and loaded from JSON files. For details on the JSON schema and the simulation pipeline, +see the {ref}`Markov-based Load Profiles ` documentation. diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java new file mode 100644 index 000000000..1eaea3188 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java @@ -0,0 +1,39 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.connectors; + +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.function.Function; + +/** Connector for JSON-based sources and sinks. */ +public class JsonFileConnector extends FileConnector { + private static final String FILE_ENDING = ".json"; + + public JsonFileConnector(Path baseDirectory) { + super(baseDirectory); + } + + public JsonFileConnector(Path baseDirectory, Function customInputStream) { + super(baseDirectory, customInputStream); + } + + /** + * Opens an input stream for the given JSON file. + * + * @param filePath relative path without ending + * @return input stream for the file + */ + public InputStream initInputStream(Path filePath) throws FileNotFoundException { + return openInputStream(filePath); + } + + @Override + protected String getFileEnding() { + return FILE_ENDING; + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java new file mode 100644 index 000000000..549cbc7fa --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -0,0 +1,50 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.factory.markov; + +import edu.ie3.datamodel.io.factory.Factory; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.*; +import java.time.ZonedDateTime; +import java.util.Optional; +import tools.jackson.databind.JsonNode; + +/** Factory turning Markov JSON data into {@link MarkovLoadModel}s. */ +public class MarkovLoadModelFactory + extends Factory + implements MarkovModelParsingSupport { + + public MarkovLoadModelFactory() { + super(MarkovLoadModel.class); + } + + /** Builds a {@link MarkovLoadModel} from a parsed JSON tree. */ + @Override + protected MarkovLoadModel buildModel(MarkovModelData data) { + JsonNode root = data.getRoot(); + String schema = extractText(root, MARKOV_SCHEMA); + ZonedDateTime generatedAt = parseTimestamp(extractText(root, jsonField(MARKOV_GENERATED_AT))); + Generator generator = parseGenerator(extractNode(root, MARKOV_GENERATOR)); + TimeModel timeModel = extractTimeModel(extractNode(root, jsonField(MARKOV_TIME_MODEL))); + ValueModel valueModel = parseValueModel(extractNode(root, jsonField(MARKOV_VALUE_MODEL))); + Parameters parameters = parseParameters(root.path(MARKOV_PARAMETERS)); + + JsonNode dataNode = extractNode(root, MARKOV_DATA); + TransitionData transitionData = + parseTransitions(dataNode, timeModel.bucketCount(), valueModel.discretization().states()); + GmmBuckets gmmBuckets = parseGmmBuckets(extractNode(dataNode, jsonLeafField(MARKOV_GMMS))); + + return new MarkovLoadModel( + schema, + generatedAt, + generator, + timeModel, + valueModel, + parameters, + transitionData, + Optional.of(gmmBuckets)); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java new file mode 100644 index 000000000..de649ead1 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java @@ -0,0 +1,40 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.factory.markov; + +import edu.ie3.datamodel.io.factory.FactoryData; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; +import java.util.Collections; +import java.util.Objects; +import tools.jackson.databind.JsonNode; + +/** Factory data wrapper around a parsed Markov-load JSON tree. */ +public class MarkovModelData extends FactoryData { + private final JsonNode root; + + public MarkovModelData(JsonNode root) { + super(Collections.emptyMap(), MarkovLoadModel.class); + this.root = Objects.requireNonNull(root, "root"); + } + + public JsonNode getRoot() { + return root; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof MarkovModelData that)) return false; + return Objects.equals(getTargetClass(), that.getTargetClass()) + && Objects.equals(getFieldsToValues(), that.getFieldsToValues()) + && Objects.equals(root, that.root); + } + + @Override + public int hashCode() { + return Objects.hash(getTargetClass(), getFieldsToValues(), root); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java new file mode 100644 index 000000000..16f30bf20 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -0,0 +1,376 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.factory.markov; + +import static edu.ie3.datamodel.io.naming.FieldNamingStrategy.*; + +import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.*; +import edu.ie3.util.StringUtils; +import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; +import java.util.*; +import tools.jackson.databind.JsonNode; + +/** Shared parsing helpers for Markov model JSON documents. */ +interface MarkovModelParsingSupport { + + default Generator parseGenerator(JsonNode generatorNode) { + String name = extractText(generatorNode, jsonLeafField(MARKOV_GENERATOR_NAME)); + String version = extractText(generatorNode, jsonLeafField(MARKOV_GENERATOR_VERSION)); + Map config = new LinkedHashMap<>(); + JsonNode configNode = generatorNode.path(jsonLeafField(MARKOV_GENERATOR_CONFIG)); + if (configNode.isObject()) { + for (Map.Entry entry : configNode.properties()) { + config.put(entry.getKey(), entry.getValue().asString()); + } + } + return new Generator(name, version, config); + } + + /** Extracts the time model block. */ + default TimeModel extractTimeModel(JsonNode timeNode) { + int bucketCount = extractInt(timeNode, jsonLeafField(MARKOV_BUCKET_COUNT)); + if (bucketCount <= 0) { + throw new FactoryException(jsonField(MARKOV_BUCKET_COUNT) + " must be greater than 0."); + } + String formula = + extractNode(timeNode, jsonLeafField(MARKOV_BUCKET_ENCODING)) + .path(jsonLeafField(MARKOV_BUCKET_ENCODING_FORMULA)) + .asString(""); + if (formula.isEmpty()) { + throw new FactoryException("Missing bucket encoding formula."); + } + int samplingInterval = extractInt(timeNode, jsonLeafField(MARKOV_SAMPLING_INTERVAL)); + if (samplingInterval <= 0) { + throw new FactoryException(jsonField(MARKOV_SAMPLING_INTERVAL) + " must be greater than 0."); + } + String timezone = extractText(timeNode, jsonLeafField(MARKOV_TIMEZONE)); + return new TimeModel(bucketCount, formula, samplingInterval, timezone); + } + + /** Parses value model settings. */ + default ValueModel parseValueModel(JsonNode valueNode) { + String valueUnit = extractText(valueNode, jsonLeafField(MARKOV_VALUE_UNIT)); + JsonNode normalizationNode = extractNode(valueNode, jsonLeafField(MARKOV_NORMALIZATION)); + String normalizationMethod = + extractText(normalizationNode, jsonLeafField(MARKOV_NORMALIZATION_METHOD)); + ValueModel.Normalization normalization = + new ValueModel.Normalization( + normalizationMethod, + parsePowerReference(normalizationNode, jsonLeafField(MARKOV_MAX_POWER)), + parsePowerReference(normalizationNode, jsonLeafField(MARKOV_MIN_POWER))); + + JsonNode discretizationNode = extractNode(valueNode, jsonLeafField(MARKOV_DISCRETIZATION)); + int states = extractInt(discretizationNode, jsonLeafField(MARKOV_DISCRETIZATION_STATES)); + if (states <= 0) { + throw new FactoryException( + jsonField(MARKOV_DISCRETIZATION_STATES) + " must be greater than 0."); + } + String thresholdsField = jsonLeafField(MARKOV_DISCRETIZATION_THRESHOLDS); + List thresholds = readDoubleArray(discretizationNode, thresholdsField); + if (thresholds.size() != Math.max(0, states - 1)) { + throw new FactoryException( + "Discretization " + + thresholdsField + + " must contain " + + Math.max(0, states - 1) + + " entries for " + + states + + " states, but found " + + thresholds.size() + + "."); + } + ValueModel.Discretization discretization = new ValueModel.Discretization(states, thresholds); + + return new ValueModel(valueUnit, normalization, discretization); + } + + /** Parses optional parameter metadata. */ + default Parameters parseParameters(JsonNode parametersNode) { + String emptyRowStrategy = + parametersNode + .path(jsonLeafField(MARKOV_PARAMETERS_TRANSITIONS)) + .path(jsonLeafField(MARKOV_EMPTY_ROW_STRATEGY)) + .asString(""); + Optional transitions = + emptyRowStrategy.isEmpty() + ? Optional.empty() + : Optional.of(new Parameters.TransitionParameters(emptyRowStrategy)); + + JsonNode gmmNode = parametersNode.path(jsonLeafField(MARKOV_PARAMETERS_GMM)); + Optional gmm = + gmmNode.isMissingNode() || gmmNode.isNull() || gmmNode.isEmpty() + ? Optional.empty() + : Optional.of( + new Parameters.GmmParameters( + gmmNode.path(jsonLeafField(MARKOV_GMM_VALUE_COLUMN)).asString(""), + optionalInt(gmmNode, jsonLeafField(MARKOV_GMM_VERBOSE)), + optionalInt(gmmNode, jsonLeafField(MARKOV_GMM_HEARTBEAT_SECONDS)))); + + return new Parameters(transitions, gmm); + } + + /** Parses the transition tensor. */ + default TransitionData parseTransitions( + JsonNode dataNode, int expectedBucketCount, int stateCount) { + JsonNode transitionsNode = extractNode(dataNode, jsonLeafField(MARKOV_TRANSITIONS)); + String dtype = extractText(transitionsNode, jsonLeafField(MARKOV_TRANSITION_DTYPE)); + String encoding = extractText(transitionsNode, jsonLeafField(MARKOV_TRANSITION_ENCODING)); + + int[] shape = parseTransitionShape(transitionsNode); + int buckets = shape[0]; + int rows = shape[1]; + int columns = shape[2]; + validateTransitionShape(expectedBucketCount, stateCount, buckets, rows, columns); + + JsonNode valuesNode = extractNode(transitionsNode, jsonLeafField(MARKOV_TRANSITION_VALUES)); + double[][][] values = parseTransitionValues(valuesNode, buckets, stateCount); + + return new TransitionData(dtype, encoding, values); + } + + /** Parses GMM buckets. Individual states may be null. */ + default GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { + if (gmmsNode == null || gmmsNode.isMissingNode() || gmmsNode.isNull()) { + throw new FactoryException("Missing field '" + jsonLeafField(MARKOV_GMMS) + "'."); + } + JsonNode bucketsNode = gmmsNode.get(jsonLeafField(MARKOV_GMM_BUCKETS)); + if (bucketsNode == null || !bucketsNode.isArray()) { + throw new FactoryException(jsonField(MARKOV_GMM_BUCKETS) + " must be an array."); + } + List buckets = new ArrayList<>(); + for (JsonNode bucketNode : bucketsNode) { + JsonNode statesNode = bucketNode.get(jsonLeafField(MARKOV_GMM_STATES)); + if (statesNode == null || !statesNode.isArray()) { + throw new FactoryException( + "Each GMM bucket must contain an array '" + jsonLeafField(MARKOV_GMM_STATES) + "'."); + } + List states = new ArrayList<>(); + for (JsonNode stateNode : statesNode) { + if (stateNode == null || stateNode.isNull()) { + states.add(null); + continue; + } + List weights = readDoubleArray(stateNode, jsonLeafField(MARKOV_GMM_WEIGHTS)); + List means = readDoubleArray(stateNode, jsonLeafField(MARKOV_GMM_MEANS)); + List variances = readDoubleArray(stateNode, jsonLeafField(MARKOV_GMM_VARIANCES)); + states.add(new GmmBuckets.GmmState(weights, means, variances)); + } + buckets.add(new GmmBuckets.GmmBucket(Collections.unmodifiableList(states))); + } + return new GmmBuckets(List.copyOf(buckets)); + } + + default JsonNode extractNode(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode()) { + throw new FactoryException("Missing field '" + field + "'."); + } + return value; + } + + default String extractText(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode() || value.isNull()) { + throw new FactoryException("Missing field '" + field + "'."); + } + if (!value.isString()) { + throw new FactoryException("Field '" + field + "' must be textual."); + } + return value.asString(); + } + + default double extractDouble(JsonNode node, String field) { + JsonNode value = node.get(field); + return extractDoubleValue(value, field); + } + + default int extractInt(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode() || value.isNull()) { + throw new FactoryException("Missing field '" + field + "'."); + } + if (!value.canConvertToInt()) { + throw new FactoryException("Field '" + field + "' must be an integer."); + } + return value.asInt(); + } + + default ZonedDateTime parseTimestamp(String timestamp) { + try { + return ZonedDateTime.parse(timestamp); + } catch (DateTimeParseException e) { + throw new FactoryException( + "Unable to parse " + jsonField(MARKOV_GENERATED_AT) + " timestamp '" + timestamp + "'.", + e); + } + } + + default OptionalInt optionalInt(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isNull()) return OptionalInt.empty(); + if (!value.canConvertToInt()) { + throw new FactoryException("Field '" + field + "' must be an integer."); + } + return OptionalInt.of(value.asInt()); + } + + default int[] parseTransitionShape(JsonNode transitionsNode) { + JsonNode shapeNode = extractNode(transitionsNode, jsonLeafField(MARKOV_TRANSITION_SHAPE)); + if (!shapeNode.isArray() || shapeNode.size() != 3) { + throw new FactoryException("Transition shape must contain three dimensions."); + } + return new int[] { + extractInt(shapeNode, 0, "Transition shape"), + extractInt(shapeNode, 1, "Transition shape"), + extractInt(shapeNode, 2, "Transition shape") + }; + } + + default void validateTransitionShape( + int expectedBucketCount, int stateCount, int buckets, int rows, int columns) { + if (buckets != expectedBucketCount) { + throw new FactoryException( + "Transition bucket count mismatch. Expected " + + expectedBucketCount + + " but was " + + buckets + + "."); + } + if (rows != stateCount || columns != stateCount) { + throw new FactoryException( + "Transition state dimension mismatch. Expected " + + stateCount + + " but was rows=" + + rows + + ", columns=" + + columns + + "."); + } + } + + default double[][][] parseTransitionValues(JsonNode valuesNode, int buckets, int stateCount) { + if (!valuesNode.isArray() || valuesNode.size() != buckets) { + throw new FactoryException( + "Transition values must be a three dimensional array with " + buckets + " buckets."); + } + double[][][] values = new double[buckets][stateCount][stateCount]; + for (int b = 0; b < buckets; b++) { + JsonNode bucketNode = extractTransitionBucket(valuesNode, b, stateCount); + for (int r = 0; r < stateCount; r++) { + JsonNode rowNode = extractTransitionRow(bucketNode, b, r, stateCount); + for (int c = 0; c < stateCount; c++) { + values[b][r][c] = + extractDoubleValue( + rowNode.get(c), + jsonField(MARKOV_TRANSITION_VALUES) + "[" + b + "][" + r + "][" + c + "]"); + } + } + } + return values; + } + + default JsonNode extractTransitionBucket(JsonNode valuesNode, int bucket, int stateCount) { + JsonNode bucketNode = valuesNode.get(bucket); + if (!bucketNode.isArray()) { + throw new FactoryException("Bucket " + bucket + " in transition values must be an array."); + } + if (bucketNode.size() != stateCount) { + throw new FactoryException( + "Bucket " + + bucket + + " contained " + + bucketNode.size() + + " rows. Expected " + + stateCount + + "."); + } + return bucketNode; + } + + default JsonNode extractTransitionRow(JsonNode bucketNode, int bucket, int row, int stateCount) { + JsonNode rowNode = bucketNode.get(row); + if (!rowNode.isArray()) { + throw new FactoryException( + "Row " + row + " in bucket " + bucket + " of transition values must be an array."); + } + if (rowNode.size() != stateCount) { + throw new FactoryException( + "Row " + + row + + " in bucket " + + bucket + + " had " + + rowNode.size() + + " columns. Expected " + + stateCount + + "."); + } + return rowNode; + } + + default List readDoubleArray(JsonNode node, String field) { + JsonNode arrayNode = node.get(field); + if (arrayNode == null || !arrayNode.isArray()) { + throw new FactoryException("Field '" + field + "' must be an array."); + } + List values = new ArrayList<>(); + for (int i = 0; i < arrayNode.size(); i++) { + values.add(extractDoubleValue(arrayNode.get(i), field + "[" + i + "]")); + } + return List.copyOf(values); + } + + default int extractInt(JsonNode node, int index, String field) { + JsonNode value = node.get(index); + if (value == null || value.isMissingNode() || value.isNull()) { + throw new FactoryException("Missing field '" + field + "[" + index + "]'."); + } + if (!value.canConvertToInt()) { + throw new FactoryException("Field '" + field + "[" + index + "]' must be an integer."); + } + return value.asInt(); + } + + default double extractDoubleValue(JsonNode node, String field) { + if (node == null || node.isMissingNode() || node.isNull()) { + throw new FactoryException("Missing field '" + field + "'."); + } + if (!node.isNumber()) { + throw new FactoryException("Field '" + field + "' must be a double."); + } + double value = node.asDouble(); + if (!Double.isFinite(value)) { + throw new FactoryException("Field '" + field + "' must be finite."); + } + return value; + } + + default Optional parsePowerReference( + JsonNode parent, String field) { + JsonNode referenceNode = parent.get(field); + if (referenceNode == null || referenceNode.isMissingNode() || referenceNode.isNull()) { + return Optional.empty(); + } + if (!referenceNode.isObject()) { + throw new FactoryException("Field '" + field + "' must be an object."); + } + double value = extractDouble(referenceNode, jsonLeafField(MARKOV_MAX_POWER_VALUE)); + String unit = extractText(referenceNode, jsonLeafField(MARKOV_MAX_POWER_UNIT)); + return Optional.of(new ValueModel.Normalization.PowerReference(value, unit)); + } + + default String jsonField(String field) { + return StringUtils.camelCaseToSnakeCase(field); + } + + default String jsonLeafField(String field) { + int lastSeparatorIndex = field.lastIndexOf('.'); + String leaf = lastSeparatorIndex < 0 ? field : field.substring(lastSeparatorIndex + 1); + return jsonField(leaf); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/file/FileType.java b/src/main/java/edu/ie3/datamodel/io/file/FileType.java index b9ac1f3f7..08886b255 100644 --- a/src/main/java/edu/ie3/datamodel/io/file/FileType.java +++ b/src/main/java/edu/ie3/datamodel/io/file/FileType.java @@ -10,7 +10,8 @@ import java.util.stream.Collectors; public enum FileType { - CSV(".csv"); + CSV(".csv"), + JSON(".json"); public final String fileEnding; diff --git a/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java b/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java index 3a0741d4b..9e507dde8 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java @@ -56,11 +56,11 @@ public class EntityPersistenceNamingStrategy { protected final Pattern individualTimeSeriesPattern; /** - * Regex to match the naming convention of a file for a repetitive load profile time series. The - * profile is accessible via the named capturing group "profile", the uuid by the group "uuid" + * Regex to match the naming convention of a load profile source file. The profile type is + * accessible via the named capturing group "type", the profile by the group "profile" */ private static final String LOAD_PROFILE_TIME_SERIES = - "lpts_(?[a-zA-Z]{1,11}[0-9]{0,3})"; + "(?lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})"; /** * Pattern to identify load profile time series in this instance of the naming strategy (takes @@ -162,7 +162,7 @@ public LoadProfileMetaInformation loadProfileTimesSeriesMetaInformation(String f throw new IllegalArgumentException( "Cannot extract meta information on load profile time series from '" + fileName + "'."); - return new LoadProfileMetaInformation(matcher.group("profile")); + return new LoadProfileMetaInformation(matcher.group("profile"), matcher.group("type")); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/naming/FieldNamingStrategy.java b/src/main/java/edu/ie3/datamodel/io/naming/FieldNamingStrategy.java index ae3ec377a..20daa4c70 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/FieldNamingStrategy.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/FieldNamingStrategy.java @@ -211,4 +211,53 @@ public class FieldNamingStrategy { public static final String TAPPOS = "tapPos"; public static final String TIME = "time"; public static final String VALUE = "value"; + + // markov - top-level + public static final String MARKOV_SCHEMA = "schema"; + public static final String MARKOV_GENERATED_AT = "generatedAt"; + public static final String MARKOV_GENERATOR = "generator"; + public static final String MARKOV_TIME_MODEL = "timeModel"; + public static final String MARKOV_VALUE_MODEL = "valueModel"; + public static final String MARKOV_PARAMETERS = "parameters"; + public static final String MARKOV_DATA = "data"; + + // markov - nested fields required for simulation + public static final String MARKOV_GENERATOR_NAME = "generator.name"; + public static final String MARKOV_GENERATOR_VERSION = "generator.version"; + public static final String MARKOV_GENERATOR_CONFIG = "generator.config"; + public static final String MARKOV_BUCKET_COUNT = "timeModel.bucketCount"; + public static final String MARKOV_BUCKET_ENCODING = "timeModel.bucketEncoding"; + public static final String MARKOV_BUCKET_ENCODING_FORMULA = "timeModel.bucketEncoding.formula"; + public static final String MARKOV_SAMPLING_INTERVAL = "timeModel.samplingIntervalMinutes"; + public static final String MARKOV_TIMEZONE = "timeModel.timezone"; + public static final String MARKOV_VALUE_UNIT = "valueModel.valueUnit"; + public static final String MARKOV_NORMALIZATION = "valueModel.normalization"; + public static final String MARKOV_NORMALIZATION_METHOD = "valueModel.normalization.method"; + public static final String MARKOV_MAX_POWER = "valueModel.normalization.maxPower"; + public static final String MARKOV_DISCRETIZATION = "valueModel.discretization"; + public static final String MARKOV_DISCRETIZATION_STATES = "valueModel.discretization.states"; + public static final String MARKOV_DISCRETIZATION_THRESHOLDS = + "valueModel.discretization.thresholdsRight"; + public static final String MARKOV_MAX_POWER_VALUE = "valueModel.normalization.maxPower.value"; + public static final String MARKOV_MAX_POWER_UNIT = "valueModel.normalization.maxPower.unit"; + public static final String MARKOV_MIN_POWER = "valueModel.normalization.minPower"; + public static final String MARKOV_MIN_POWER_VALUE = "valueModel.normalization.minPower.value"; + public static final String MARKOV_MIN_POWER_UNIT = "valueModel.normalization.minPower.unit"; + public static final String MARKOV_PARAMETERS_TRANSITIONS = "parameters.transitions"; + public static final String MARKOV_EMPTY_ROW_STRATEGY = "parameters.transitions.emptyRowStrategy"; + public static final String MARKOV_PARAMETERS_GMM = "parameters.gmm"; + public static final String MARKOV_GMM_VALUE_COLUMN = "parameters.gmm.valueCol"; + public static final String MARKOV_GMM_VERBOSE = "parameters.gmm.verbose"; + public static final String MARKOV_GMM_HEARTBEAT_SECONDS = "parameters.gmm.heartbeatSeconds"; + public static final String MARKOV_TRANSITIONS = "data.transitions"; + public static final String MARKOV_TRANSITION_DTYPE = "data.transitions.dtype"; + public static final String MARKOV_TRANSITION_ENCODING = "data.transitions.encoding"; + public static final String MARKOV_TRANSITION_SHAPE = "data.transitions.shape"; + public static final String MARKOV_TRANSITION_VALUES = "data.transitions.values"; + public static final String MARKOV_GMMS = "data.gmms"; + public static final String MARKOV_GMM_BUCKETS = "data.gmms.buckets"; + public static final String MARKOV_GMM_STATES = "data.gmms.buckets.states"; + public static final String MARKOV_GMM_WEIGHTS = "data.gmms.buckets.states.weights"; + public static final String MARKOV_GMM_MEANS = "data.gmms.buckets.states.means"; + public static final String MARKOV_GMM_VARIANCES = "data.gmms.buckets.states.variances"; } diff --git a/src/main/java/edu/ie3/datamodel/io/naming/ModelFields.java b/src/main/java/edu/ie3/datamodel/io/naming/ModelFields.java index 9db871e44..5bd98c537 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/ModelFields.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/ModelFields.java @@ -27,6 +27,7 @@ import edu.ie3.datamodel.models.input.thermal.DomesticHotWaterStorageInput; import edu.ie3.datamodel.models.input.thermal.ThermalBusInput; import edu.ie3.datamodel.models.input.thermal.ThermalHouseInput; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; import edu.ie3.datamodel.models.result.CongestionResult; import edu.ie3.datamodel.models.result.NodeResult; import edu.ie3.datamodel.models.result.connector.LineResult; @@ -60,6 +61,7 @@ private ModelFields() { private static final Map, Set> unsupportedFields = new HashMap<>(); private static final Map, List>> valueMandatoryFields = new HashMap<>(); + private static final Map, List>> genericMandatoryFields = new HashMap<>(); /** * Retrieves a list that contains combinations of mandatory fields for the provided class. @@ -73,7 +75,7 @@ public static List> getMandatoryFields(Class clazz) { } else if (Value.class.isAssignableFrom(clazz)) { return valueMandatoryFields.getOrDefault(clazz, Collections.emptyList()); } else { - return Collections.emptyList(); + return genericMandatoryFields.getOrDefault(clazz, Collections.emptyList()); } } @@ -145,6 +147,20 @@ public static void registerValue( ModelFields.valueMandatoryFields.putIfAbsent(entityClass, List.of(mandatoryFields)); } + /** + * Method to register mandatory fields for a class that is neither an {@link Entity} nor a {@link + * Value}. + * + *

NOTE: This method will only add fields, if no fields are registered yet! + * + * @param clazz for which fields should be registered + * @param mandatoryFields the mandatory field combinations to register + */ + @SafeVarargs + public static void registerGeneric(Class clazz, Set... mandatoryFields) { + ModelFields.genericMandatoryFields.putIfAbsent(clazz, List.of(mandatoryFields)); + } + /** * Method to register mandatory fields for a given entity class. * @@ -205,6 +221,30 @@ public static void registerUnsupported(Class entityClass, Set unsuppo // registering em fields registerMandatory(EmInput.class, assetFields, CONTROLLING_EM, CONTROL_STRATEGY); registerOptional(EmInput.class, assetOptionalFields); + + // markov load model fields + registerGeneric( + MarkovLoadModel.class, + newSet( + MARKOV_SCHEMA, + MARKOV_GENERATED_AT, + MARKOV_GENERATOR, + MARKOV_TIME_MODEL, + MARKOV_VALUE_MODEL, + MARKOV_DATA, + MARKOV_GENERATOR_NAME, + MARKOV_GENERATOR_VERSION, + MARKOV_BUCKET_COUNT, + MARKOV_SAMPLING_INTERVAL, + MARKOV_TIMEZONE, + MARKOV_DISCRETIZATION_STATES, + MARKOV_DISCRETIZATION_THRESHOLDS, + MARKOV_MAX_POWER_VALUE, + MARKOV_MAX_POWER_UNIT, + MARKOV_MIN_POWER_VALUE, + MARKOV_MIN_POWER_UNIT, + MARKOV_TRANSITION_VALUES, + MARKOV_GMM_BUCKETS)); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java index d25d8fadb..7f3101fa0 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java @@ -14,8 +14,16 @@ public class LoadProfileMetaInformation extends TimeSeriesMetaInformation { private final PowerProfileKey profileKey; public LoadProfileMetaInformation(String profileKey) { + this(profileKey, PowerProfileKey.Type.TS); + } + + public LoadProfileMetaInformation(String profileKey, String type) { + this(profileKey, PowerProfileKey.Type.parse(type)); + } + + public LoadProfileMetaInformation(String profileKey, PowerProfileKey.Type type) { super(UUID.randomUUID()); - this.profileKey = new PowerProfileKey(profileKey); + this.profileKey = new PowerProfileKey(profileKey, type); } public LoadProfileMetaInformation(PowerProfileKey powerProfileKey) { diff --git a/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java b/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java index 61a00c8ac..530e8a6a8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/PowerValueSource.java @@ -9,13 +9,17 @@ import edu.ie3.datamodel.models.value.PValue; import java.time.ZonedDateTime; import java.util.Optional; +import java.util.OptionalDouble; +import java.util.OptionalInt; import java.util.function.Supplier; import javax.measure.quantity.Energy; import javax.measure.quantity.Power; import tech.units.indriya.ComparableQuantity; /** Interface defining base functionality for power value sources. */ -public sealed interface PowerValueSource +public sealed interface PowerValueSource< + I extends PowerValueSource.PowerValueIdentifier, + O extends PowerValueSource.PowerOutputValue> permits PowerValueSource.MarkovBased, PowerValueSource.TimeSeriesBased { /** Returns the profile of this source. */ @@ -29,7 +33,7 @@ public sealed interface PowerValueSource> getValueSupplier(I data); + Supplier getValueSupplier(I data); /** * Method to determine the next timestamp for which data is present. @@ -49,10 +53,11 @@ public sealed interface PowerValueSource {} + non-sealed interface TimeSeriesBased + extends PowerValueSource {} /** Interface for markov-chain-based power value sources. */ - non-sealed interface MarkovBased extends PowerValueSource {} + non-sealed interface MarkovBased extends PowerValueSource {} // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= // input data @@ -61,7 +66,7 @@ non-sealed interface MarkovBased extends PowerValueSource * Interface for the input data of {@link #getValueSupplier(PowerValueIdentifier)}. The data is * used to determine the next power. */ - sealed interface PowerValueIdentifier permits PowerValueSource.TimeSeriesInputValue { + sealed interface PowerValueIdentifier permits TimeSeriesInputValue, MarkovIdentifier { /** Returns the timestamp for which a power value is needed. */ ZonedDateTime time(); } @@ -72,4 +77,50 @@ sealed interface PowerValueIdentifier permits PowerValueSource.TimeSeriesInputVa * @param time */ record TimeSeriesInputValue(ZonedDateTime time) implements PowerValueIdentifier {} + + /** + * Input data for Markov-based power value sources, containing everything needed for a single + * simonaMarkovLoad step. + * + *

Provide either {@code previousState} (typical for subsequent steps) or an {@code + * initialNormalizedValue} (for the first step). The {@code randomSeed} enables reproducible + * sampling. + */ + record MarkovIdentifier( + ZonedDateTime time, + OptionalInt previousState, + OptionalDouble initialNormalizedValue, + long randomSeed) + implements PowerValueIdentifier { + + public MarkovIdentifier { + if (previousState.isEmpty() && initialNormalizedValue.isEmpty()) { + throw new IllegalArgumentException( + "Need either previous state or an initial normalized value to start the Markov chain."); + } + } + } + + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= + // output data + + /** Interface for the output data of {@link #getValueSupplier(PowerValueIdentifier)}. */ + sealed interface PowerOutputValue + permits PowerValueSource.TimeSeriesOutputValue, PowerValueSource.MarkovOutputValue { + Optional value(); + } + + /** + * Interface for time-series-based power values. + * + * @param value + */ + record TimeSeriesOutputValue(Optional value) implements PowerOutputValue { + public static Supplier from(Supplier> supplier) { + return () -> new TimeSeriesOutputValue(supplier.get()); + } + } + + /** Output data for Markov-based power values, including the next state. */ + record MarkovOutputValue(Optional value, int nextState) implements PowerOutputValue {} } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvLoadProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvLoadProfileSource.java index 927740bb1..d90bed2e4 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvLoadProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvLoadProfileSource.java @@ -13,7 +13,6 @@ import edu.ie3.datamodel.io.source.LoadProfileSource; import edu.ie3.datamodel.models.timeseries.repetitive.LoadProfileEntry; import edu.ie3.datamodel.models.timeseries.repetitive.LoadProfileTimeSeries; -import edu.ie3.datamodel.models.value.PValue; import edu.ie3.datamodel.models.value.load.LoadValues; import edu.ie3.datamodel.utils.Try; import java.nio.file.Path; @@ -71,8 +70,8 @@ public Set> getEntries() { } @Override - public Supplier> getValueSupplier(TimeSeriesInputValue data) { - return loadProfileTimeSeries.supplyValue(data.time()); + public Supplier getValueSupplier(TimeSeriesInputValue data) { + return TimeSeriesOutputValue.from(loadProfileTimeSeries.supplyValue(data.time())); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java new file mode 100644 index 000000000..f10a4c44a --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java @@ -0,0 +1,143 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.source.json; + +import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.io.connectors.JsonFileConnector; +import edu.ie3.datamodel.io.naming.FileNamingStrategy; +import edu.ie3.datamodel.io.source.file.FileDataSource; +import edu.ie3.datamodel.models.Entity; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Stream; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; + +/** Data source abstraction for JSON files. */ +public class JsonDataSource extends FileDataSource { + + private final JsonFileConnector connector; + private final ObjectMapper objectMapper = new ObjectMapper(); + + public JsonDataSource(Path directoryPath, FileNamingStrategy fileNamingStrategy) { + this(new JsonFileConnector(directoryPath), fileNamingStrategy); + } + + public JsonDataSource(JsonFileConnector connector, FileNamingStrategy fileNamingStrategy) { + super(connector.getBaseDirectory(), fileNamingStrategy); + this.connector = connector; + } + + /** + * Opens an input stream for the provided file path. + * + * @param filePath relative path without ending + * @return input stream + * @throws SourceException if the file cannot be opened + */ + public InputStream initInputStream(Path filePath) throws SourceException { + try { + return connector.initInputStream(filePath); + } catch (FileNotFoundException e) { + throw new SourceException("Unable to open JSON file '" + filePath + "'.", e); + } + } + + /** + * Reads and parses a JSON file into a {@link JsonNode} tree. + * + * @param filePath path to the JSON file + * @return the parsed JSON tree + * @throws SourceException if the file cannot be read or parsed + */ + public JsonNode readTree(Path filePath) throws SourceException { + try (InputStream inputStream = initInputStream(filePath)) { + return objectMapper.readTree(inputStream); + } catch (IOException e) { + throw new SourceException("Unable to read JSON from '" + filePath + "'.", e); + } + } + + @Override + public Optional> getSourceFields(Class entityClass) + throws SourceException { + throw unsupportedTabularAccess("getSourceFields(Class)"); + } + + @Override + public Stream> getSourceData(Class entityClass) + throws SourceException { + throw unsupportedTabularAccess("getSourceData(Class)"); + } + + /** + * Returns the set of field names present in the JSON file at the given path. + * + * @param filePath path to the JSON file + * @return an optional containing the field names, or empty if none found + * @throws SourceException if the file cannot be read + */ + @Override + public Optional> getSourceFields(Path filePath) throws SourceException { + JsonNode root = readTree(filePath); + return Optional.of(fieldNames(root)); + } + + /** + * Returns the set of (nested, dot-separated) field names present in the given JSON tree. Allows + * callers that already hold a parsed tree to collect field names without re-reading the file. + * + * @param root the parsed JSON tree + * @return the field names + */ + public static Set fieldNames(JsonNode root) { + return collectFieldNames(root); + } + + @Override + public Stream> getSourceData(Path filePath) throws SourceException { + throw unsupportedTabularAccess("getSourceData(Path)"); + } + + private static Set collectFieldNames(JsonNode node) { + Set fields = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + collectFields("", node, fields); + return fields; + } + + private static void collectFields(String prefix, JsonNode node, Set collector) { + if (node.isArray()) { + if (!prefix.isEmpty()) { + collector.add(prefix); + } + return; + } + if (node.isObject()) { + if (!prefix.isEmpty()) { + collector.add(prefix); + } + node.propertyNames() + .forEach(name -> collectFields(join(prefix, name), node.get(name), collector)); + } else if (!prefix.isEmpty()) { + collector.add(prefix); + } + } + + private static String join(String prefix, String name) { + return prefix.isEmpty() ? name : prefix + "." + name; + } + + private UnsupportedOperationException unsupportedTabularAccess(String method) { + return new UnsupportedOperationException( + "JsonDataSource does not support '" + method + "', as JSON sources are not tabular."); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java new file mode 100644 index 000000000..eb6599b4e --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java @@ -0,0 +1,139 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.source.json; + +import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.exceptions.FailedValidationException; +import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.exceptions.ValidationException; +import edu.ie3.datamodel.io.factory.markov.MarkovLoadModelFactory; +import edu.ie3.datamodel.io.factory.markov.MarkovModelData; +import edu.ie3.datamodel.io.file.FileType; +import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation; +import edu.ie3.datamodel.io.source.DataSource; +import edu.ie3.datamodel.io.source.EntitySource; +import edu.ie3.datamodel.io.source.PowerValueSource; +import edu.ie3.datamodel.models.profile.PowerProfileKey; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; +import java.time.ZonedDateTime; +import java.util.Optional; +import java.util.Set; +import java.util.function.Supplier; +import javax.measure.quantity.Energy; +import javax.measure.quantity.Power; +import tech.units.indriya.ComparableQuantity; +import tools.jackson.databind.JsonNode; + +/** + * Source that reads Markov-based load models from JSON files. + * + *

The JSON file is parsed lazily and cached. All power value requests are delegated to the + * parsed {@link MarkovLoadModel}. + */ +public class JsonMarkovProfileSource extends EntitySource implements PowerValueSource.MarkovBased { + + private final JsonDataSource dataSource; + private final FileLoadProfileMetaInformation metaInformation; + private final MarkovLoadModelFactory factory; + private MarkovLoadModel cachedModel; + private JsonNode cachedRoot; + + public JsonMarkovProfileSource( + JsonDataSource dataSource, FileLoadProfileMetaInformation metaInformation) { + this(dataSource, metaInformation, new MarkovLoadModelFactory()); + } + + public JsonMarkovProfileSource( + JsonDataSource dataSource, + FileLoadProfileMetaInformation metaInformation, + MarkovLoadModelFactory factory) { + this.dataSource = dataSource; + this.metaInformation = metaInformation; + this.factory = factory; + if (metaInformation.getFileType() != FileType.JSON) { + throw new IllegalArgumentException("Markov profile source requires JSON meta information."); + } + } + + /** + * Returns the parsed Markov model, parsing the underlying file if needed. + * + * @throws SourceException if reading or parsing fails + */ + public synchronized MarkovLoadModel getModel() throws SourceException { + if (cachedModel == null) { + try { + cachedModel = factory.get(new MarkovModelData(readRoot())).getOrThrow(); + } catch (FactoryException e) { + throw new SourceException( + "Unable to build Markov load model from '" + + metaInformation.getProfileKey().getValue() + + "'.", + e); + } + // the parsed model supersedes the raw tree, release it for garbage collection + cachedRoot = null; + } + return cachedModel; + } + + /** Returns the parsed JSON tree, reading the underlying file only once. */ + private synchronized JsonNode readRoot() throws SourceException { + if (cachedRoot == null) { + cachedRoot = dataSource.readTree(metaInformation.getFullFilePath()); + } + return cachedRoot; + } + + @Override + public void validate() throws ValidationException { + try { + Set fields = JsonDataSource.fieldNames(readRoot()); + DataSource.validate(fields, MarkovLoadModel.class).getOrThrow(); + } catch (SourceException e) { + throw new FailedValidationException( + "Unable to read Markov model '" + + metaInformation.getProfileKey().getValue() + + "' for validation.", + e); + } + } + + @Override + public PowerProfileKey getProfileKey() { + return metaInformation.getProfileKey(); + } + + /** Delegates to the cached {@link MarkovLoadModel} for a single simulation step. */ + @Override + public Supplier getValueSupplier(MarkovIdentifier data) { + return getModelUnchecked().getValueSupplier(data); + } + + @Override + public Optional getNextTimeKey(ZonedDateTime time) { + return getModelUnchecked().getNextTimeKey(time); + } + + @Override + public Optional> getMaxPower() { + return getModelUnchecked().getMaxPower(); + } + + @Override + public Optional> getProfileEnergyScaling() { + return getModelUnchecked().getProfileEnergyScaling(); + } + + private MarkovLoadModel getModelUnchecked() { + try { + return getModel(); + } catch (SourceException e) { + throw new IllegalStateException( + "Unable to load Markov model '" + metaInformation.getProfileKey().getValue() + "'.", e); + } + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSource.java index 6be4a0972..2463b4529 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSource.java @@ -15,7 +15,6 @@ import edu.ie3.datamodel.io.source.LoadProfileSource; import edu.ie3.datamodel.models.timeseries.repetitive.LoadProfileEntry; import edu.ie3.datamodel.models.timeseries.repetitive.LoadProfileTimeSeries; -import edu.ie3.datamodel.models.value.PValue; import edu.ie3.datamodel.models.value.Value; import edu.ie3.datamodel.models.value.load.LoadValues; import edu.ie3.datamodel.utils.TimeSeriesUtils; @@ -97,10 +96,11 @@ public Set> getEntries() { } @Override - public Supplier> getValueSupplier(TimeSeriesInputValue data) { + public Supplier getValueSupplier(TimeSeriesInputValue data) { ZonedDateTime time = data.time(); Optional loadValueOption = queryForValue(time); - return () -> loadValueOption.map(v -> v.getValue(time, powerProfileKey)); + return TimeSeriesOutputValue.from( + () -> loadValueOption.map(v -> v.getValue(time, powerProfileKey))); } @Override diff --git a/src/main/java/edu/ie3/datamodel/models/profile/PowerProfileKey.java b/src/main/java/edu/ie3/datamodel/models/profile/PowerProfileKey.java index 501fe22bc..aa9fa3cee 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/PowerProfileKey.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/PowerProfileKey.java @@ -12,10 +12,16 @@ /** Interface defining a power profile. */ public final class PowerProfileKey implements Serializable { + private final Type type; private final String value; public final boolean noKeyAssigned; public PowerProfileKey(String key) { + this(key, Type.TS); + } + + public PowerProfileKey(String key, Type type) { + this.type = type; if (key == null || key.isEmpty()) { this.value = "No profile assigned."; this.noKeyAssigned = true; @@ -29,6 +35,10 @@ public String getValue() { return value; } + public Type getType() { + return type; + } + public boolean equals(PowerProfile other) { return equals(other.getKey()); } @@ -41,24 +51,51 @@ public boolean equalsAny(PowerProfile... others) { public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; PowerProfileKey that = (PowerProfileKey) o; - return noKeyAssigned == that.noKeyAssigned && Objects.equals(value, that.value); + return noKeyAssigned == that.noKeyAssigned + && type == that.type + && Objects.equals(value, that.value); } @Override public int hashCode() { - return Objects.hash(value, noKeyAssigned); + return Objects.hash(type, value, noKeyAssigned); } @Override public String toString() { - return "PowerProfileKey{" + "value='" + value + '\'' + ", noKeyAssigned=" + noKeyAssigned + '}'; + return "PowerProfileKey{" + + "type=" + + type + + ", value='" + + value + + '\'' + + ", noKeyAssigned=" + + noKeyAssigned + + '}'; } // static - public static final PowerProfileKey NO_KEY_ASSIGNED = new PowerProfileKey(""); + public static final PowerProfileKey NO_KEY_ASSIGNED = new PowerProfileKey("", Type.TS); public static String toUniformKey(String key) { return key.toLowerCase().replaceAll("[-_]*", ""); } + + public enum Type { + TS, + MARKOV; + + public static Type parse(String type) { + return switch (type) { + case "lpts" -> TS; + case "markov" -> MARKOV; + default -> + throw new IllegalArgumentException( + "The given type '" + + type + + "' is not supported for load profile time series meta information."); + }; + } + } } diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java new file mode 100644 index 000000000..9f3f817b7 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -0,0 +1,592 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.models.profile.markov; + +import edu.ie3.datamodel.io.source.PowerValueSource; +import edu.ie3.datamodel.models.StandardUnits; +import edu.ie3.datamodel.models.value.PValue; +import java.time.DayOfWeek; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.SplittableRandom; +import java.util.function.Supplier; +import javax.measure.quantity.Energy; +import javax.measure.quantity.Power; +import tech.units.indriya.ComparableQuantity; +import tech.units.indriya.quantity.Quantities; + +/** + * Container for Markov-chain-based load models produced by simonaMarkovLoad. + * + *

The model keeps the trained transition and GMM data together with the helpers used to sample + * one power value per simulation step. + */ +public class MarkovLoadModel { + + private static final int QUARTER_HOURS_PER_DAY = 96; + private static final int WEEKEND_FACTOR = QUARTER_HOURS_PER_DAY; + private static final int MONTH_FACTOR = QUARTER_HOURS_PER_DAY * 2; + private static final double PROBABILITY_TOLERANCE = 1e-5; + + private final String schema; + private final ZonedDateTime generatedAt; + private final Generator generator; + private final TimeModel timeModel; + private final ValueModel valueModel; + private final Parameters parameters; + private final TransitionData transitionData; + private final Optional gmmBuckets; + + private final ZoneId zoneId; + private final double[][][] transitions; + private final int bucketCount; + private final int stateCount; + private final int samplingIntervalMinutes; + private final double[] discretizationThresholds; + private final GmmStateData[][] gmmStates; + private final ComparableQuantity maxPowerFromModel; + private final ComparableQuantity minPowerFromModel; + + /** + * Builds a model from the parsed JSON sections and prepares the runtime lookup arrays. + * + *

Semantic invariants such as transition row sums and GMM component consistency are checked + * here as well, so directly constructed models are protected just like JSON-parsed models. + * + * @throws IllegalArgumentException if GMM data is missing, the normalization range is + * non-positive, or the transition tensor does not match {@code bucketCount * stateCount * + * stateCount} + */ + public MarkovLoadModel( + String schema, + ZonedDateTime generatedAt, + Generator generator, + TimeModel timeModel, + ValueModel valueModel, + Parameters parameters, + TransitionData transitionData, + Optional gmmBuckets) { + this.schema = Objects.requireNonNull(schema, "schema"); + this.generatedAt = Objects.requireNonNull(generatedAt, "generatedAt"); + this.generator = Objects.requireNonNull(generator, "generator"); + this.timeModel = Objects.requireNonNull(timeModel, "timeModel"); + this.valueModel = Objects.requireNonNull(valueModel, "valueModel"); + this.parameters = Objects.requireNonNull(parameters, "parameters"); + this.transitionData = Objects.requireNonNull(transitionData, "transitionData"); + this.gmmBuckets = Objects.requireNonNull(gmmBuckets, "gmmBuckets"); + + this.zoneId = ZoneId.of(timeModel.timezone()); + this.transitions = transitionData.values(); + this.bucketCount = timeModel.bucketCount(); + this.stateCount = valueModel.discretization().states(); + this.samplingIntervalMinutes = timeModel.samplingIntervalMinutes(); + this.discretizationThresholds = + valueModel.discretization().thresholdsRight().stream() + .mapToDouble(Double::doubleValue) + .toArray(); + this.gmmStates = + buildGmmStates( + gmmBuckets.orElseThrow( + () -> new IllegalArgumentException("Markov model lacks GMM data."))); + this.maxPowerFromModel = + valueModel + .normalization() + .maxPower() + .map(this::convertPowerReference) + .orElseThrow( + () -> new IllegalArgumentException("Markov model lacks normalization.max_power")); + this.minPowerFromModel = + valueModel + .normalization() + .minPower() + .map(this::convertPowerReference) + .orElseThrow( + () -> new IllegalArgumentException("Markov model lacks normalization.min_power")); + + if (!this.maxPowerFromModel.isGreaterThan(this.minPowerFromModel)) { + throw new IllegalArgumentException( + "Markov model normalization has non-positive range (max <= min)."); + } + + validateTransitionDimensions(); + } + + private void validateTransitionDimensions() { + if (transitions.length != bucketCount) { + throw new IllegalArgumentException( + "Transition bucket count mismatch. Expected " + + bucketCount + + " but was " + + transitions.length); + } + for (int bucket = 0; bucket < transitions.length; bucket++) { + if (transitions[bucket].length != stateCount) { + throw new IllegalArgumentException( + "Transition state count mismatch in bucket " + + bucket + + ". Expected " + + stateCount + + " but was " + + transitions[bucket].length); + } + for (int state = 0; state < transitions[bucket].length; state++) { + double[] row = transitions[bucket][state]; + if (row.length != stateCount) { + throw new IllegalArgumentException( + "Transition next-state count mismatch in bucket " + + bucket + + ", state " + + state + + ". Expected " + + stateCount + + " but was " + + row.length); + } + validateProbabilityVector(row, "Transition row in bucket " + bucket + ", state " + state); + } + } + } + + private static void validateProbabilityVector(double[] weights, String context) { + if (weights.length == 0) { + throw new IllegalArgumentException(context + " must not be empty."); + } + double sum = 0d; + for (double weight : weights) { + if (!Double.isFinite(weight)) { + throw new IllegalArgumentException(context + " contains a non-finite probability."); + } + if (weight < 0d) { + throw new IllegalArgumentException(context + " contains a negative probability."); + } + sum += weight; + } + if (Math.abs(sum - 1d) > PROBABILITY_TOLERANCE) { + throw new IllegalArgumentException(context + " must sum to 1.0, but summed to " + sum + "."); + } + } + + private GmmStateData[][] buildGmmStates(GmmBuckets buckets) { + List bucketList = buckets.buckets(); + if (bucketList.size() != bucketCount) { + throw new IllegalArgumentException( + "GMM bucket count mismatch. Expected " + bucketCount + " but was " + bucketList.size()); + } + GmmStateData[][] lookup = new GmmStateData[bucketCount][stateCount]; + for (int bucket = 0; bucket < bucketCount; bucket++) { + List states = bucketList.get(bucket).states(); + if (states.size() != stateCount) { + throw new IllegalArgumentException( + "State count mismatch in bucket " + bucket + ". Expected " + stateCount); + } + for (int state = 0; state < stateCount; state++) { + GmmBuckets.GmmState s = states.get(state); + lookup[bucket][state] = s != null ? s.toStateData() : null; + } + } + return lookup; + } + + private ComparableQuantity convertPowerReference( + ValueModel.Normalization.PowerReference reference) { + if (!"kW".equalsIgnoreCase(reference.unit())) { + throw new IllegalArgumentException( + "Unsupported reference power unit '" + reference.unit() + "'. Only kW is supported."); + } + return Quantities.getQuantity(reference.value(), StandardUnits.ACTIVE_POWER_IN); + } + + public String schema() { + return schema; + } + + public ZonedDateTime generatedAt() { + return generatedAt; + } + + public Generator generator() { + return generator; + } + + public TimeModel timeModel() { + return timeModel; + } + + public ValueModel valueModel() { + return valueModel; + } + + public Parameters parameters() { + return parameters; + } + + public TransitionData transitionData() { + return transitionData; + } + + public Optional gmmBuckets() { + return gmmBuckets; + } + + /** + * Returns a supplier for a single Markov step. + * + *

Callers are expected to create a new supplier for each time step. + */ + public Supplier getValueSupplier( + PowerValueSource.MarkovIdentifier data) { + Objects.requireNonNull(data, "data"); + return () -> computeStep(data); + } + + /** Convenience helper to compute a single step immediately. */ + public PowerValueSource.MarkovOutputValue getPower(PowerValueSource.MarkovIdentifier data) { + return getValueSupplier(data).get(); + } + + /** Returns the next timestamp by advancing the model's sampling interval. */ + public Optional getNextTimeKey(ZonedDateTime time) { + return Optional.ofNullable(time).map(t -> t.plusMinutes(samplingIntervalMinutes)); + } + + /** Returns the maximum power from the model's normalization configuration. */ + public Optional> getMaxPower() { + return Optional.of(maxPowerFromModel); + } + + /** Not implemented yet. Future feature */ + public Optional> getProfileEnergyScaling() { + return Optional.empty(); + } + + private PowerValueSource.MarkovOutputValue computeStep(PowerValueSource.MarkovIdentifier data) { + int bucket = bucketId(data.time()); + int currentState = resolveState(data); + SplittableRandom rng = new SplittableRandom(deriveSeed(data, bucket, currentState)); + StepResult step = simulateStep(bucket, currentState, rng); + ComparableQuantity power = scale(step.normalizedValue()); + return new PowerValueSource.MarkovOutputValue(Optional.of(new PValue(power)), step.nextState()); + } + + /** Maps a timestamp into the trainer's bucket layout: month, weekday/weekend and quarter-hour. */ + private int bucketId(ZonedDateTime time) { + ZonedDateTime zoned = time.withZoneSameInstant(zoneId); + int month = zoned.getMonthValue() - 1; + int weekendFlag = isWeekend(zoned) ? 1 : 0; + int quarterHour = zoned.getHour() * 4 + zoned.getMinute() / 15; + return Math.floorMod( + month * MONTH_FACTOR + weekendFlag * WEEKEND_FACTOR + quarterHour, bucketCount); + } + + /** True for Saturdays and Sundays. */ + private boolean isWeekend(ZonedDateTime time) { + DayOfWeek day = time.getDayOfWeek(); + return day == DayOfWeek.SATURDAY || day == DayOfWeek.SUNDAY; + } + + /** + * Uses the previous state if present; otherwise discretizes the initial normalized value. + * + * @throws IllegalArgumentException if a supplied previous state is out of bounds + */ + private int resolveState(PowerValueSource.MarkovIdentifier input) { + if (input.previousState().isPresent()) { + int state = input.previousState().getAsInt(); + if (state < 0 || state >= stateCount) { + throw new IllegalArgumentException("Previous state out of bounds: " + state); + } + return state; + } + double normalized = input.initialNormalizedValue().orElseThrow(); + return discretize(normalized); + } + + /** + * Returns the state-bin index for a normalized value. Boundary values map to the upper bin, + * matching the trainer's {@code searchsorted(side="right")} assignment. + */ + private int discretize(double normalized) { + double value = Math.clamp(normalized, 0d, 1d); + for (int i = 0; i < discretizationThresholds.length; i++) { + if (value < discretizationThresholds[i]) { + return i; + } + } + return discretizationThresholds.length; + } + + /** Produces a deterministic RNG seed from request seed, bucket, state and time slot. */ + private long deriveSeed(PowerValueSource.MarkovIdentifier input, int bucket, int state) { + long seed = input.randomSeed(); + seed = 31 * seed + bucket; + seed = 31 * seed + state; + long slot = input.time().toInstant().toEpochMilli() / (samplingIntervalMinutes * 60_000L); + return 31 * seed + slot; + } + + /** Draws the next state from the transition row and samples the corresponding GMM. */ + private StepResult simulateStep(int bucket, int currentState, SplittableRandom rng) { + int nextStateIndex = drawWeighted(transitions[bucket][currentState], rng); + double normalized = sampleNormalizedValue(bucket, nextStateIndex, rng); + return new StepResult(nextStateIndex, normalized); + } + + /** Picks an index by weighted random sampling. */ + private static int drawWeighted(double[] weights, SplittableRandom rng) { + double sample = rng.nextDouble(); + double cumulative = 0d; + for (int i = 0; i < weights.length; i++) { + cumulative += weights[i]; + if (sample <= cumulative) { + return i; + } + } + return weights.length - 1; + } + + /** + * Draws a normalized power value from the GMM at {@code (bucket, state)} and clamps Gaussian + * tails to the model range. + */ + private double sampleNormalizedValue(int bucket, int state, SplittableRandom rng) { + GmmStateData gmm = gmmStates[bucket][state]; + if (gmm == null) { + return 0d; + } + return Math.clamp(gmm.sample(rng), 0d, 1d); + } + + /** Maps a normalized value to physical power using the model's min/max range. */ + private ComparableQuantity scale(double normalizedValue) { + ComparableQuantity range = maxPowerFromModel.subtract(minPowerFromModel); + return minPowerFromModel.add(range.multiply(normalizedValue)).asType(Power.class); + } + + private record StepResult(int nextState, double normalizedValue) {} + + /** Runtime representation of a single GMM state. */ + private static final class GmmStateData { + private final double[] weights; + private final double[] means; + private final double[] variances; + + private GmmStateData(double[] weights, double[] means, double[] variances) { + this.weights = weights; + this.means = means; + this.variances = variances; + } + + private double sample(SplittableRandom rng) { + int component = drawWeighted(weights, rng); + double mean = means[component]; + double variance = Math.max(0d, variances[component]); + if (variance == 0d) { + return mean; + } + return mean + Math.sqrt(variance) * rng.nextGaussian(); + } + } + + /** Provenance metadata for the trained model. */ + public record Generator(String name, String version, Map config) {} + + /** Temporal layout of the model. */ + public record TimeModel( + int bucketCount, + String bucketEncodingFormula, + int samplingIntervalMinutes, + String timezone) {} + + /** Value-space configuration: physical unit, normalization range and state discretization. */ + public record ValueModel( + String valueUnit, Normalization normalization, Discretization discretization) { + + /** + * Normalization range used to map between normalized values and physical power. The optional + * wrapping reflects the JSON shape, both bounds are required at construction time. + */ + public record Normalization( + String method, Optional maxPower, Optional minPower) { + + /** A value/unit pair from the JSON normalization block. Only {@code "kW"} is supported. */ + public record PowerReference(double value, String unit) {} + } + + /** State-bin layout and right-edge thresholds. */ + public record Discretization(int states, List thresholdsRight) {} + } + + /** Optional metadata describing how the trainer produced the transitions and GMMs. */ + public record Parameters( + Optional transitions, Optional gmm) { + + /** + * Strategy used by the trainer when a transition row had no observations. Carried through for + * traceability. Not used at simulation time. + */ + public record TransitionParameters(String emptyRowStrategy) {} + + /** + * GMM-fitting metadata. + * + * @param valueColumn name of the column the trainer fit on + * @param verbose scikit-learn verbosity used during fitting + * @param heartbeatSeconds trainer watchdog interval, carried through for JSON round-trip only + */ + public record GmmParameters( + String valueColumn, OptionalInt verbose, OptionalInt heartbeatSeconds) {} + } + + /** + * Transition tensor {@code values[bucket][currentState][nextState] = probability}. Equality uses + * array contents instead of array identity. + */ + public record TransitionData(String dtype, String encoding, double[][][] values) { + public int bucketCount() { + return values.length; + } + + public int stateCount() { + return values.length == 0 ? 0 : values[0].length; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + return o instanceof TransitionData(String d, String e, double[][][] v) + && Objects.equals(dtype, d) + && Objects.equals(encoding, e) + && Arrays.deepEquals(values, v); + } + + @Override + public int hashCode() { + return Objects.hash(dtype, encoding, Arrays.deepHashCode(values)); + } + + @Override + public String toString() { + return "TransitionData[dtype=" + + dtype + + ", encoding=" + + encoding + + ", values=" + + Arrays.deepToString(values) + + "]"; + } + } + + /** Per-bucket Gaussian Mixture Models. State entries may be {@code null}. */ + public record GmmBuckets(List buckets) { + public record GmmBucket(List states) {} + + /** Parameters of one Gaussian Mixture Model. */ + public record GmmState(List weights, List means, List variances) { + public GmmState { + validateGmmComponents(weights, means, variances); + } + + private GmmStateData toStateData() { + return new GmmStateData(toArray(weights), toArray(means), toArray(variances)); + } + + private static void validateGmmComponents( + List weights, List means, List variances) { + Objects.requireNonNull(weights, "weights"); + Objects.requireNonNull(means, "means"); + Objects.requireNonNull(variances, "variances"); + + if (weights.isEmpty()) { + throw new IllegalArgumentException("GMM state must contain at least one component."); + } + if (weights.size() != means.size() || weights.size() != variances.size()) { + throw new IllegalArgumentException( + "GMM weights, means and variances must have the same number of entries."); + } + + double[] weightArray = toArray(weights); + validateProbabilityVector(weightArray, "GMM component weights"); + + for (int i = 0; i < means.size(); i++) { + Double mean = means.get(i); + Double variance = variances.get(i); + if (mean == null || !Double.isFinite(mean)) { + throw new IllegalArgumentException("GMM means must be finite."); + } + if (variance == null || !Double.isFinite(variance)) { + throw new IllegalArgumentException("GMM variances must be finite."); + } + if (variance < 0d) { + throw new IllegalArgumentException("GMM variances must be non-negative."); + } + } + } + + private static double[] toArray(List values) { + double[] array = new double[values.size()]; + for (int i = 0; i < values.size(); i++) { + array[i] = values.get(i); + } + return array; + } + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof MarkovLoadModel that)) return false; + return Objects.equals(schema, that.schema) + && Objects.equals(generatedAt, that.generatedAt) + && Objects.equals(generator, that.generator) + && Objects.equals(timeModel, that.timeModel) + && Objects.equals(valueModel, that.valueModel) + && Objects.equals(parameters, that.parameters) + && Objects.equals(transitionData, that.transitionData) + && Objects.equals(gmmBuckets, that.gmmBuckets); + } + + @Override + public int hashCode() { + return Objects.hash( + schema, + generatedAt, + generator, + timeModel, + valueModel, + parameters, + transitionData, + gmmBuckets); + } + + @Override + public String toString() { + return "MarkovLoadModel[" + + "schema=" + + schema + + ", generatedAt=" + + generatedAt + + ", generator=" + + generator + + ", timeModel=" + + timeModel + + ", valueModel=" + + valueModel + + ", parameters=" + + parameters + + ", transitionData=" + + transitionData + + ", gmmBuckets=" + + gmmBuckets + + ']'; + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy index c5b2735d0..d11d78e98 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy @@ -15,6 +15,7 @@ import edu.ie3.util.io.FileIOUtils import spock.lang.Shared import spock.lang.Specification +import java.io.Reader import java.nio.file.Files import java.nio.file.Path @@ -50,12 +51,18 @@ class CsvFileConnectorTest extends Specification { } def "The csv file connector initializes a reader without Exception, if the foreseen file is apparent"() { + given: + Reader reader = null + when: def filePath = fileNamingStrategy.getFilePath(NodeInput).orElseThrow() - cfc.initReader(filePath) + reader = cfc.initReader(filePath) then: noExceptionThrown() + + cleanup: + reader?.close() } def "The csv file connector is able to init writers utilizing a directory hierarchy"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy new file mode 100644 index 000000000..c20466276 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy @@ -0,0 +1,41 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.connectors + +import spock.lang.Specification + +import java.nio.file.Files +import java.nio.file.Path + +class JsonFileConnectorTest extends Specification { + + Path tempDir + + def setup() { + tempDir = Files.createTempDirectory("jsonFileConnector") + } + + def cleanup() { + if (tempDir != null) { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .forEach { Files.deleteIfExists(it) } + } + } + + def "initInputStream resolves .json ending and reads content"() { + given: + def file = tempDir.resolve("model.json") + Files.writeString(file, """{"foo":"bar"}""") + def connector = new JsonFileConnector(tempDir) + + when: + def content = connector.initInputStream(Path.of("model")).text + + then: + content == """{"foo":"bar"}""" + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy new file mode 100644 index 000000000..ae678dde5 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy @@ -0,0 +1,190 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.factory.markov + +import edu.ie3.datamodel.exceptions.FactoryException +import edu.ie3.datamodel.models.profile.markov.MarkovModelJsonTestSupport +import tools.jackson.databind.node.ObjectNode + +class MarkovLoadModelFactoryTest extends MarkovModelJsonTestSupport { + private final MarkovLoadModelFactory factory = new MarkovLoadModelFactory() + + def "buildModel returns parsed Markov load model from valid JSON"() { + given: + def root = objectMapper.readTree(validModelJson()) + + when: + def model = factory.get(new MarkovModelData(root)).getOrThrow() + + then: + model.schema() == "markov.load.v1" + model.generator().name() == "simonaMarkovLoad" + model.timeModel().bucketCount() == 1 + model.valueModel().discretization().states() == 2 + model.transitionData().bucketCount() == 1 + model.transitionData().stateCount() == 2 + model.transitionData().values()[0][0][1] == 0.9d + model.gmmBuckets().isPresent() + def gmmState = model.gmmBuckets().get().buckets().first().states().first() + gmmState.weights() == [1.0d] + gmmState.means() == [1.0d] + gmmState.variances() == [0.2d] + model.valueModel().normalization().maxPower().isPresent() + with(model.valueModel().normalization().maxPower().get()) { + value() == 1.5d + unit() == "kW" + } + model.valueModel().normalization().minPower().isPresent() + with(model.valueModel().normalization().minPower().get()) { + value() == 0.1d + unit() == "kW" + } + } + + def "buildModel throws FactoryException on transition dimension mismatch"() { + given: + def invalidJson = objectMapper.readTree(validModelJson().replace("\"shape\": [1,2,2]", "\"shape\": [2,2,2]")) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + def "buildModel throws FactoryException when transition row does not sum to one"() { + given: + def invalidJson = objectMapper.readTree(validModelJson() + .replace('[0.1, 0.9]', '[0.1, 0.8]')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + def "buildModel throws FactoryException when threshold count does not match state count"() { + given: "states=2 requires exactly 1 threshold, but 2 are provided" + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"thresholds_right": [0.5]', '"thresholds_right": [0.3, 0.7]')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + def "buildModel throws FactoryException when schema field is missing"() { + given: + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"schema": "markov.load.v1",', '')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + def "buildModel throws FactoryException when generated_at timestamp is invalid"() { + given: + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"generated_at": "2025-01-01T00:00:00Z"', '"generated_at": "not-a-timestamp"')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + def "buildModel throws FactoryException when bucket_encoding formula is missing"() { + given: + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"bucket_encoding": { "formula": "hour_of_day" }', '"bucket_encoding": {}')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + def "buildModel throws FactoryException when gmms buckets array is missing"() { + given: + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"buckets":', '"not_buckets":')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + def "buildModel throws FactoryException when GMM component arrays differ in size"() { + given: + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"variances": [0.2]', '"variances": [0.2, 0.3]')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + def "buildModel tolerates a missing parameters block"() { + given: + def root = objectMapper.readTree(validModelJson()) + ((ObjectNode) root).remove("parameters") + + when: + def model = factory.get(new MarkovModelData(root)).getOrThrow() + + then: + model.parameters().transitions().isEmpty() + model.parameters().gmm().isEmpty() + } + + def "buildModel throws FactoryException when max_power is missing"() { + given: + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"max_power": { "value": 1.5, "unit": "kW" },', '')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + def "buildModel throws FactoryException when normalization range is non-positive"() { + given: "max_power below min_power" + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"max_power": { "value": 1.5, "unit": "kW" }', '"max_power": { "value": 0.05, "unit": "kW" }')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + def "buildModel throws FactoryException on unsupported power unit"() { + given: + def invalidJson = objectMapper.readTree(validModelJson() + .replace('"max_power": { "value": 1.5, "unit": "kW" }', '"max_power": { "value": 1500.0, "unit": "W" }')) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/file/FileTypeTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/file/FileTypeTest.groovy new file mode 100644 index 000000000..ecfb744d2 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/file/FileTypeTest.groovy @@ -0,0 +1,30 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.file + +import edu.ie3.datamodel.exceptions.ParsingException +import spock.lang.Specification + +class FileTypeTest extends Specification { + + def "getFileType resolves CSV and JSON endings"() { + expect: + FileType.getFileType(fileName) == expected + + where: + fileName || expected + "data.csv" || FileType.CSV + "model.json" || FileType.JSON + } + + def "getFileType throws ParsingException on unknown ending"() { + when: + FileType.getFileType("unknown.txt") + + then: + thrown(ParsingException) + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy index af95fcbe2..5a8a63859 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy @@ -21,6 +21,7 @@ import edu.ie3.datamodel.models.input.system.type.* import edu.ie3.datamodel.models.input.thermal.CylindricalStorageInput import edu.ie3.datamodel.models.input.thermal.ThermalHouseInput import edu.ie3.datamodel.models.profile.BdewStandardLoadProfile +import edu.ie3.datamodel.models.profile.PowerProfileKey import edu.ie3.datamodel.models.result.NodeResult import edu.ie3.datamodel.models.result.connector.LineResult import edu.ie3.datamodel.models.result.connector.SwitchResult @@ -87,11 +88,27 @@ class EntityPersistenceNamingStrategyTest extends Specification { matcher.matches() then: "it also has correct capturing groups" - matcher.groupCount() == 1 - matcher.group(1) == "g3" + matcher.groupCount() == 2 + matcher.group(1) == "lpts" + matcher.group("type") == "lpts" + matcher.group(2) == "g3" matcher.group("profile") == "g3" } + def "The pattern for a Markov load profile time series file name matches and extracts the correct profile"() { + given: + def ens = new EntityPersistenceNamingStrategy() + def validFileName = "markov_demo1" + + when: + def matcher = ens.loadProfileTimeSeriesPattern.matcher(validFileName) + + then: + matcher.matches() + matcher.group("type") == "markov" + matcher.group("profile") == "demo1" + } + def "Trying to extract individual time series meta information throws an Exception, if it is provided a malformed string"() { given: def ens = new EntityPersistenceNamingStrategy() @@ -118,6 +135,19 @@ class EntityPersistenceNamingStrategyTest extends Specification { ex.message == "Cannot extract meta information on load profile time series from 'foo'." } + def "loadProfileTimesSeriesMetaInformation extracts profile from Markov load profile file name"() { + given: + def ens = new EntityPersistenceNamingStrategy() + def fileName = "markov_demo2" + + when: + def meta = ens.loadProfileTimesSeriesMetaInformation(fileName) + + then: + meta.profileKey.getValue() == "demo2" + meta.profileKey.type == PowerProfileKey.Type.MARKOV + } + def "The EntityPersistenceNamingStrategy is able to prepare the prefix properly"() { when: String actual = EntityPersistenceNamingStrategy.preparePrefix(prefix) diff --git a/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy index f75845a62..ff20b7553 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy @@ -24,6 +24,7 @@ import edu.ie3.datamodel.models.input.system.type.* import edu.ie3.datamodel.models.input.thermal.CylindricalStorageInput import edu.ie3.datamodel.models.input.thermal.ThermalHouseInput import edu.ie3.datamodel.models.profile.BdewStandardLoadProfile +import edu.ie3.datamodel.models.profile.PowerProfileKey import edu.ie3.datamodel.models.result.NodeResult import edu.ie3.datamodel.models.result.connector.LineResult import edu.ie3.datamodel.models.result.connector.SwitchResult @@ -684,7 +685,7 @@ class FileNamingStrategyTest extends Specification { def actual = strategy.loadProfileTimeSeriesPattern.pattern() then: - actual == "test_grid" + escapedFileSeparator + "input" + escapedFileSeparator + "participants" + escapedFileSeparator + "time_series" + escapedFileSeparator + "lpts_(?[a-zA-Z]{1,11}[0-9]{0,3})" + actual == "test_grid" + escapedFileSeparator + "input" + escapedFileSeparator + "participants" + escapedFileSeparator + "time_series" + escapedFileSeparator + "(?lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})" } def "A FileNamingStrategy with FlatHierarchy returns correct individual time series file name pattern"() { @@ -706,7 +707,7 @@ class FileNamingStrategyTest extends Specification { def actual = strategy.loadProfileTimeSeriesPattern.pattern() then: - actual == "lpts_(?[a-zA-Z]{1,11}[0-9]{0,3})" + actual == "(?lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})" } def "Trying to extract time series meta information throws an Exception, if it is provided a malformed string"() { @@ -837,6 +838,7 @@ class FileNamingStrategyTest extends Specification { LoadProfileMetaInformation.isAssignableFrom(metaInformation.getClass()) (metaInformation as LoadProfileMetaInformation).with { profileKey.value == "g3" + profileKey.type == PowerProfileKey.Type.TS } } @@ -852,6 +854,7 @@ class FileNamingStrategyTest extends Specification { LoadProfileMetaInformation.isAssignableFrom(metaInformation.getClass()) (metaInformation as LoadProfileMetaInformation).with { profileKey.value == "g3" + profileKey.type == PowerProfileKey.Type.TS } } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/naming/ModelFieldsTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/naming/ModelFieldsTest.groovy new file mode 100644 index 000000000..c15ab6243 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/naming/ModelFieldsTest.groovy @@ -0,0 +1,43 @@ +/* + * © 2026. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.naming + +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel +import spock.lang.Specification + +class ModelFieldsTest extends Specification { + + def "getMandatoryFields returns registered fields for MarkovLoadModel"() { + when: + def fields = ModelFields.getMandatoryFields(MarkovLoadModel) + + then: + fields.size() == 1 + fields[0].containsAll([ + // top-level + FieldNamingStrategy.MARKOV_SCHEMA, + FieldNamingStrategy.MARKOV_GENERATED_AT, + FieldNamingStrategy.MARKOV_GENERATOR, + FieldNamingStrategy.MARKOV_TIME_MODEL, + FieldNamingStrategy.MARKOV_VALUE_MODEL, + FieldNamingStrategy.MARKOV_DATA, + // nested - required for simulation + FieldNamingStrategy.MARKOV_GENERATOR_NAME, + FieldNamingStrategy.MARKOV_GENERATOR_VERSION, + FieldNamingStrategy.MARKOV_BUCKET_COUNT, + FieldNamingStrategy.MARKOV_SAMPLING_INTERVAL, + FieldNamingStrategy.MARKOV_TIMEZONE, + FieldNamingStrategy.MARKOV_DISCRETIZATION_STATES, + FieldNamingStrategy.MARKOV_DISCRETIZATION_THRESHOLDS, + FieldNamingStrategy.MARKOV_MAX_POWER_VALUE, + FieldNamingStrategy.MARKOV_MAX_POWER_UNIT, + FieldNamingStrategy.MARKOV_MIN_POWER_VALUE, + FieldNamingStrategy.MARKOV_MIN_POWER_UNIT, + FieldNamingStrategy.MARKOV_TRANSITION_VALUES, + FieldNamingStrategy.MARKOV_GMM_BUCKETS + ]) + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy new file mode 100644 index 000000000..df102a87c --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy @@ -0,0 +1,94 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.json + +import edu.ie3.datamodel.exceptions.SourceException +import edu.ie3.datamodel.io.connectors.JsonFileConnector +import edu.ie3.datamodel.io.naming.FileNamingStrategy +import edu.ie3.datamodel.models.Entity +import spock.lang.Specification + +import java.nio.file.Files +import java.nio.file.Path + +class JsonDataSourceTest extends Specification { + + Path tempDir + JsonDataSource dataSource + + def setup() { + tempDir = Files.createTempDirectory("jsonDataSource") + dataSource = new JsonDataSource(tempDir, new FileNamingStrategy()) + } + + def cleanup() { + if (tempDir != null) { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .forEach { Files.deleteIfExists(it) } + } + } + + def "initInputStream opens JSON file via connector"() { + given: + def file = tempDir.resolve("sample.json") + Files.writeString(file, """{"key":42}""") + + when: + def content = dataSource.initInputStream(Path.of("sample")).text + + then: + content == """{"key":42}""" + } + + def "readTree parses JSON file into a JsonNode tree"() { + given: + def file = tempDir.resolve("data.json") + Files.writeString(file, """{"answer":42}""") + + when: + def node = dataSource.readTree(Path.of("data")) + + then: + node.get("answer").asInt() == 42 + } + + def "readTree throws SourceException when file does not exist"() { + when: + dataSource.readTree(Path.of("nonexistent")) + + then: + thrown(SourceException) + } + + def "getSourceFields returns field names from JSON file"() { + given: + def file = tempDir.resolve("fields.json") + Files.writeString(file, """{"foo":"bar","nested":{"a":1}}""") + + when: + def fields = dataSource.getSourceFields(Path.of("fields")) + + then: + fields.isPresent() + fields.get().contains("foo") + fields.get().contains("nested.a") + } + + def "tabular access methods are unsupported"() { + when: + dataSource.getSourceFields(Entity) + + then: + thrown(UnsupportedOperationException) + + when: + dataSource.getSourceData(Entity) + + then: + thrown(UnsupportedOperationException) + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy new file mode 100644 index 000000000..737bf09bf --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy @@ -0,0 +1,170 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.json + +import edu.ie3.datamodel.exceptions.SourceException +import edu.ie3.datamodel.io.connectors.JsonFileConnector +import edu.ie3.datamodel.io.file.FileType +import edu.ie3.datamodel.io.naming.FileNamingStrategy +import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation +import edu.ie3.datamodel.io.source.PowerValueSource +import edu.ie3.datamodel.models.StandardUnits +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel +import edu.ie3.datamodel.models.profile.markov.MarkovModelJsonTestSupport +import tools.jackson.databind.node.ObjectNode + +import java.nio.file.Files +import java.nio.file.Path +import java.time.ZonedDateTime +import java.util.concurrent.atomic.AtomicInteger +import java.util.function.Function + +class JsonMarkovProfileSourceTest extends MarkovModelJsonTestSupport { + + Path tempDir + Path jsonFile + + def setup() { + tempDir = Files.createTempDirectory("markovProfileSource") + jsonFile = tempDir.resolve("model.json") + } + + def cleanup() { + if (tempDir != null) { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .forEach { Files.deleteIfExists(it) } + } + } + + def "validate and getModel read Markov model from JSON file"() { + given: + Files.writeString(jsonFile, validModelJson()) + def source = new JsonMarkovProfileSource( + new JsonDataSource(tempDir, new FileNamingStrategy()), + new FileLoadProfileMetaInformation("profile1", jsonFile, FileType.JSON) + ) + + when: + source.validate() + + then: + noExceptionThrown() + + when: + MarkovLoadModel modelFirst = source.getModel() + MarkovLoadModel modelSecond = source.getModel() + + then: + modelFirst.is(modelSecond) // cached instance reused + modelFirst.schema() == "markov.load.v1" + noExceptionThrown() + } + + def "validate and getModel read the underlying file only once"() { + given: "a connector that counts how often the file is opened" + Files.writeString(jsonFile, validModelJson()) + def openCount = new AtomicInteger() + def countingStream = { String path -> + openCount.incrementAndGet() + new FileInputStream(path) + } as Function + def source = new JsonMarkovProfileSource( + new JsonDataSource(new JsonFileConnector(tempDir, countingStream), new FileNamingStrategy()), + new FileLoadProfileMetaInformation("profile1", jsonFile, FileType.JSON) + ) + + when: + source.validate() + source.getModel() + source.getModel() + + then: + openCount.get() == 1 + } + + def "validate tolerates a missing optional parameters block"() { + given: + def root = objectMapper.readTree(validModelJson()) + ((ObjectNode) root).remove("parameters") + Files.writeString(jsonFile, root.toString()) + def source = new JsonMarkovProfileSource( + new JsonDataSource(tempDir, new FileNamingStrategy()), + new FileLoadProfileMetaInformation("profile1", jsonFile, FileType.JSON) + ) + + when: + source.validate() + def model = source.getModel() + + then: + noExceptionThrown() + model.parameters().transitions().isEmpty() + model.parameters().gmm().isEmpty() + } + + def "getModel throws SourceException on invalid JSON file"() { + given: + Files.writeString(jsonFile, "{}") + def source = new JsonMarkovProfileSource( + new JsonDataSource(tempDir, new FileNamingStrategy()), + new FileLoadProfileMetaInformation("brokenProfile", jsonFile, FileType.JSON) + ) + + when: + source.getModel() + + then: + thrown(SourceException) + } + + def "source exposes Markov-based power value supplier"() { + given: + Files.writeString(jsonFile, validModelJson()) + def source = new JsonMarkovProfileSource( + new JsonDataSource(tempDir, new FileNamingStrategy()), + new FileLoadProfileMetaInformation("profile1", jsonFile, FileType.JSON) + ) + def input = new PowerValueSource.MarkovIdentifier( + ZonedDateTime.parse("2025-01-01T00:00:00Z"), + OptionalInt.of(0), + OptionalDouble.empty(), + 42L) + + when: + def supplier = source.getValueSupplier(input) + def output = supplier.get() + + then: + source.getProfileKey().getValue() == "profile1" + source.getMaxPower().isPresent() + source.getMaxPower().get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 10d + output.value().isPresent() + output.nextState() == 1 + output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 5.25d + } + + protected static String validModelJson() { + return markovModelJson(defaultTransitionValues(), sourceGmmStates(), "10.0", "0.5") + } + + private static String sourceGmmStates() { + return """ + [ + { + "weights": [1.0], + "means": [1.0], + "variances": [0.0] + }, + { + "weights": [1.0], + "means": [0.5], + "variances": [0.0] + } + ] + """.stripIndent() + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSourceIT.groovy index 623bd5b5e..99a646458 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSourceIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlLoadProfileSourceIT.groovy @@ -64,7 +64,7 @@ class SqlLoadProfileSourceIT extends Specification implements TestContainerHelpe def "A SqlTimeSeriesSource can read and correctly parse a single value for a specific date"() { when: def supplier = loadSource.getValueSupplier(new PowerValueSource.TimeSeriesInputValue(TIME_00MIN)) - def value = supplier.get() + def value = supplier.get().value() then: value.present diff --git a/src/test/groovy/edu/ie3/datamodel/models/profile/PowerProfileKeyTest.groovy b/src/test/groovy/edu/ie3/datamodel/models/profile/PowerProfileKeyTest.groovy index 78984dab2..6290645f4 100644 --- a/src/test/groovy/edu/ie3/datamodel/models/profile/PowerProfileKeyTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/models/profile/PowerProfileKeyTest.groovy @@ -121,4 +121,32 @@ class PowerProfileKeyTest extends Specification { "" || PowerProfileKey.NO_KEY_ASSIGNED null || PowerProfileKey.NO_KEY_ASSIGNED } + + def "Power profile keys with same value but different type are distinct"() { + given: + def timeSeriesKey = new PowerProfileKey("demo", PowerProfileKey.Type.TS) + def markovKey = new PowerProfileKey("demo", PowerProfileKey.Type.MARKOV) + + expect: + timeSeriesKey != markovKey + timeSeriesKey.hashCode() != markovKey.hashCode() + } + + def "Power profile key type is parsed correctly from file prefix"() { + expect: + PowerProfileKey.Type.parse(type) == expected + + where: + type || expected + "lpts" || PowerProfileKey.Type.TS + "markov" || PowerProfileKey.Type.MARKOV + } + + def "Power profile key type parsing rejects unsupported file prefix"() { + when: + PowerProfileKey.Type.parse("unsupported") + + then: + thrown(IllegalArgumentException) + } } diff --git a/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy b/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy new file mode 100644 index 000000000..70cd5742f --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovLoadModelTest.groovy @@ -0,0 +1,161 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.models.profile.markov + +import edu.ie3.datamodel.io.factory.markov.MarkovLoadModelFactory +import edu.ie3.datamodel.io.factory.markov.MarkovModelData +import edu.ie3.datamodel.io.source.PowerValueSource +import edu.ie3.datamodel.models.StandardUnits + +import java.time.ZonedDateTime + +class MarkovLoadModelTest extends MarkovModelJsonTestSupport { + + private final MarkovLoadModelFactory factory = new MarkovLoadModelFactory() + + def "supplier scales deterministic normalized values and exposes next state"() { + given: + def model = loadModel(deterministicTransitions(), deterministicStates()) + def input = new PowerValueSource.MarkovIdentifier( + ZonedDateTime.parse("2025-01-01T00:00:00Z"), + OptionalInt.of(0), + OptionalDouble.empty(), + 99L + ) + + when: + def supplier = model.getValueSupplier(input) + def output = supplier.get() + def outputAgain = supplier.get() + + then: + output.value().isPresent() + output.nextState() == 1 + output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 4.2d + outputAgain.value() == output.value() + outputAgain.nextState() == output.nextState() + model.getMaxPower().isPresent() + model.getMaxPower().get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 5d + } + + def "supplier denormalizes using model min and max power"() { + given: + def model = loadModel(deterministicTransitions(), deterministicStates()) + def input = new PowerValueSource.MarkovIdentifier( + ZonedDateTime.parse("2025-01-01T00:00:00Z"), + OptionalInt.of(0), + OptionalDouble.empty(), + 17L + ) + + when: + def supplier = model.getValueSupplier(input) + def output = supplier.get() + + then: + output.nextState() == 1 + output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 4.2d + } + + def "supplier uses initial normalized value when no previous state is present"() { + given: + def model = loadModel(selfLoopTransitions(), deterministicStates()) + def input = new PowerValueSource.MarkovIdentifier( + ZonedDateTime.parse("2025-01-01T00:00:00Z"), + OptionalInt.empty(), + OptionalDouble.of(0.25d), + 13L + ) + + when: + def supplier = model.getValueSupplier(input) + def output = supplier.get() + + then: + output.nextState() == 0 // discretized from initial normalized value + output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 2.6d + } + + def "initial normalized value exactly on a threshold maps to the upper state"() { + given: "trainer semantics: searchsorted(side='right') puts boundary values into the upper bin" + def model = loadModel(selfLoopTransitions(), deterministicStates()) + def input = new PowerValueSource.MarkovIdentifier( + ZonedDateTime.parse("2025-01-01T00:00:00Z"), + OptionalInt.empty(), + OptionalDouble.of(0.5d), + 13L + ) + + when: + def output = model.getValueSupplier(input).get() + + then: + output.nextState() == 1 + output.value().get().p.get().to(StandardUnits.ACTIVE_POWER_IN).value.doubleValue() == 4.2d + } + + def "supplier rejects an out-of-bounds previous state"() { + given: + def model = loadModel(deterministicTransitions(), deterministicStates()) + def input = new PowerValueSource.MarkovIdentifier( + ZonedDateTime.parse("2025-01-01T00:00:00Z"), + OptionalInt.of(5), + OptionalDouble.empty(), + 1L + ) + + when: + model.getValueSupplier(input).get() + + then: + thrown(IllegalArgumentException) + } + + private loadModel(String transitions, String states) { + def json = markovModelJson(transitions, states, "5.0", "1.0") + def root = objectMapper.readTree(json) + factory.get(new MarkovModelData(root)).getOrThrow() + } + + private static String deterministicTransitions() { + return """ + [ + [ + [0.0, 1.0], + [0.0, 1.0] + ] + ] + """.stripIndent() + } + + private static String deterministicStates() { + return """ + [ + { + "weights": [1.0], + "means": [0.4], + "variances": [0.0] + }, + { + "weights": [1.0], + "means": [0.8], + "variances": [0.0] + } + ] + """.stripIndent() + } + + private static String selfLoopTransitions() { + return """ + [ + [ + [1.0, 0.0], + [0.0, 1.0] + ] + ] + """.stripIndent() + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovModelJsonTestSupport.groovy b/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovModelJsonTestSupport.groovy new file mode 100644 index 000000000..aac14b253 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/models/profile/markov/MarkovModelJsonTestSupport.groovy @@ -0,0 +1,102 @@ +/* + * © 2026. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.models.profile.markov + +import spock.lang.Specification +import tools.jackson.databind.ObjectMapper + +abstract class MarkovModelJsonTestSupport extends Specification { + + protected final ObjectMapper objectMapper = new ObjectMapper() + + protected static String validModelJson() { + return markovModelJson(defaultTransitionValues(), defaultGmmStates()) + } + + protected static String markovModelJson( + String transitions, + String states, + String maxPower = "1.5", + String minPower = "0.1" + ) { + return """ + { + "schema": "markov.load.v1", + "generated_at": "2025-01-01T00:00:00Z", + "generator": { + "name": "simonaMarkovLoad", + "version": "1.0.0", + "config": { "foo": "bar" } + }, + "time_model": { + "bucket_count": 1, + "bucket_encoding": { "formula": "hour_of_day" }, + "sampling_interval_minutes": 60, + "timezone": "UTC" + }, + "value_model": { + "value_unit": "normalized", + "normalization": { + "method": "none", + "max_power": { "value": $maxPower, "unit": "kW" }, + "min_power": { "value": $minPower, "unit": "kW" } + }, + "discretization": { + "states": 2, + "thresholds_right": [0.5] + } + }, + "parameters": { + "transitions": { "empty_row_strategy": "fill" }, + "gmm": { + "value_col": "p", + "verbose": 1, + "heartbeat_seconds": 5 + } + }, + "data": { + "transitions": { + "dtype": "float32", + "encoding": "nested_lists", + "shape": [1,2,2], + "values": $transitions + }, + "gmms": { + "buckets": [ + { + "states": $states + } + ] + } + } + } + """.stripIndent() + } + + protected static String defaultTransitionValues() { + return """ + [ + [ + [0.1, 0.9], + [0.3, 0.7] + ] + ] + """.stripIndent() + } + + protected static String defaultGmmStates() { + return """ + [ + { + "weights": [1.0], + "means": [1.0], + "variances": [0.2] + }, + null + ] + """.stripIndent() + } +}