From 13557aa46820e11f698ddb754aeb4daa5c64ea40 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Wed, 12 Feb 2025 17:05:58 +0100 Subject: [PATCH 01/31] add test to test time slicing --- tests/test_selection.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_selection.py b/tests/test_selection.py index 044b66e..f5526c7 100644 --- a/tests/test_selection.py +++ b/tests/test_selection.py @@ -32,8 +32,6 @@ def test_range_slice_within_range(ds): assert ds.y.min() >= y_start assert ds.y.max() <= y_end - ds - @pytest.mark.parametrize("x_start, x_end", ([-50000, -51000], [0, 500000])) def test_error_on_empty_range(ds, x_start, x_end): @@ -49,3 +47,17 @@ def test_error_on_empty_range(ds, x_start, x_end): with pytest.raises(AssertionError): ds = mdp.ops.selection.select_by_kwargs(ds, **coord_ranges) + + +def test_can_slice_time(ds): + """ + test if the slice is within the specified range + """ + start = "1990-09-03T00:03" + end = "1990-09-09T00:00" + step = "PT3H" + coord_ranges = { + "time": mdp.config.Range(start=start, end=end, step=step), + } + + ds = mdp.ops.selection.select_by_kwargs(ds, **coord_ranges) From d3802f2ea237b0bf5cdc6d0c12ae55208092b2dc Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Wed, 12 Feb 2025 17:06:27 +0100 Subject: [PATCH 02/31] add warning if using step --- mllam_data_prep/ops/selection.py | 37 ++++++++++++-------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/mllam_data_prep/ops/selection.py b/mllam_data_prep/ops/selection.py index 37b91c1..2de5508 100644 --- a/mllam_data_prep/ops/selection.py +++ b/mllam_data_prep/ops/selection.py @@ -1,11 +1,12 @@ import datetime +import warnings import pandas as pd from ..config import Range -def _normalize_slice_startstop(s): +def normalize_slice_startstop(s): if isinstance(s, pd.Timestamp): return s elif isinstance(s, str): @@ -17,7 +18,7 @@ def _normalize_slice_startstop(s): return s -def _normalize_slice_step(s): +def normalize_slice_step(s): if isinstance(s, pd.Timedelta): return s elif isinstance(s, str): @@ -63,21 +64,21 @@ def select_by_kwargs(ds, **coord_ranges): raise ValueError( f"Selection for coordinate {coord} must have either 'start' and 'end' given" ) - sel_start = _normalize_slice_startstop(selection.start) - sel_end = _normalize_slice_startstop(selection.end) - sel_step = _normalize_slice_step(selection.step) + sel_start = normalize_slice_startstop(selection.start) + sel_end = normalize_slice_startstop(selection.end) + sel_step = normalize_slice_step(selection.step) assert sel_start != sel_end, "Start and end cannot be the same" - # we don't select with the step size for now, but simply check (below) that - # the step size in the data is the same as the requested step size - ds = ds.sel({coord: slice(sel_start, sel_end)}) + # TODO Implement handling of time step size. See issue #69 + if coord == "time" and sel_step is not None: + warnings.warn( + "Step size for time coordinate is not yet supported and is ignored" + ) + sel_step = None + ################ - if coord == "time": - check_point_in_dataset(coord, sel_start, ds) - check_point_in_dataset(coord, sel_end, ds) - if sel_step is not None: - check_step(sel_step, coord, ds) + ds = ds.sel({coord: slice(sel_start, sel_end, sel_step)}) assert ( len(ds[coord]) > 0 @@ -92,16 +93,6 @@ def select_by_kwargs(ds, **coord_ranges): return ds -def check_point_in_dataset(coord, point, ds): - """ - check that the requested point is in the data. - """ - if point is not None and point not in ds[coord].values: - raise ValueError( - f"Provided value for coordinate {coord} ({point}) is not in the data." - ) - - def check_step(sel_step, coord, ds): """ check that the step requested is exactly what the data has From d7924dcab7405ea60ff0eedac61f0cacc646f105 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Wed, 12 Feb 2025 17:21:14 +0100 Subject: [PATCH 03/31] warn about ignoring step_sie --- mllam_data_prep/ops/selection.py | 24 ++---------------------- tests/test_selection.py | 2 ++ 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/mllam_data_prep/ops/selection.py b/mllam_data_prep/ops/selection.py index 2de5508..423cf86 100644 --- a/mllam_data_prep/ops/selection.py +++ b/mllam_data_prep/ops/selection.py @@ -1,9 +1,8 @@ -import datetime import warnings import pandas as pd -from ..config import Range +from mllam_data_prep.config import Range def normalize_slice_startstop(s): @@ -70,13 +69,11 @@ def select_by_kwargs(ds, **coord_ranges): assert sel_start != sel_end, "Start and end cannot be the same" - # TODO Implement handling of time step size. See issue #69 if coord == "time" and sel_step is not None: warnings.warn( - "Step size for time coordinate is not yet supported and is ignored" + "Step size for time coordinate is not supported and is ignored" ) sel_step = None - ################ ds = ds.sel({coord: slice(sel_start, sel_end, sel_step)}) @@ -91,20 +88,3 @@ def select_by_kwargs(ds, **coord_ranges): f"Selection for coordinate {coord} must be a list or a dict" ) return ds - - -def check_step(sel_step, coord, ds): - """ - check that the step requested is exactly what the data has - """ - all_steps = ds[coord].diff(dim=coord).values - first_step = all_steps[0].astype("timedelta64[s]").astype(datetime.timedelta) - - if not all(all_steps[0] == all_steps): - raise ValueError( - f"Step size for coordinate {coord} is not constant: {all_steps}" - ) - if sel_step != first_step: - raise ValueError( - f"Step size for coordinate {coord} is not the same as requested: {first_step} != {sel_step}" - ) diff --git a/tests/test_selection.py b/tests/test_selection.py index f5526c7..9975bd5 100644 --- a/tests/test_selection.py +++ b/tests/test_selection.py @@ -1,3 +1,5 @@ +from datetime import datetime + import pytest import xarray as xr From 868558f7d550cdd97db1d45a4d6ecdd52f993464 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Wed, 12 Feb 2025 17:37:15 +0100 Subject: [PATCH 04/31] update naming --- mllam_data_prep/ops/selection.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mllam_data_prep/ops/selection.py b/mllam_data_prep/ops/selection.py index 423cf86..bb49fc2 100644 --- a/mllam_data_prep/ops/selection.py +++ b/mllam_data_prep/ops/selection.py @@ -5,7 +5,7 @@ from mllam_data_prep.config import Range -def normalize_slice_startstop(s): +def str_to_datetime(s): if isinstance(s, pd.Timestamp): return s elif isinstance(s, str): @@ -17,7 +17,7 @@ def normalize_slice_startstop(s): return s -def normalize_slice_step(s): +def str_to_timedelta(s): if isinstance(s, pd.Timedelta): return s elif isinstance(s, str): @@ -63,9 +63,9 @@ def select_by_kwargs(ds, **coord_ranges): raise ValueError( f"Selection for coordinate {coord} must have either 'start' and 'end' given" ) - sel_start = normalize_slice_startstop(selection.start) - sel_end = normalize_slice_startstop(selection.end) - sel_step = normalize_slice_step(selection.step) + sel_start = str_to_datetime(selection.start) + sel_end = str_to_datetime(selection.end) + sel_step = str_to_timedelta(selection.step) assert sel_start != sel_end, "Start and end cannot be the same" From 7d725426ceae9553c918068f29f60132fd7bca9b Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Wed, 12 Feb 2025 17:37:29 +0100 Subject: [PATCH 05/31] test allow timedelta --- tests/test_from_config.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_from_config.py b/tests/test_from_config.py index 1a89361..515ce83 100644 --- a/tests/test_from_config.py +++ b/tests/test_from_config.py @@ -1,3 +1,4 @@ +import datetime import shutil import tempfile from pathlib import Path @@ -8,6 +9,7 @@ import mllam_data_prep as mdp import tests.data as testdata +from mllam_data_prep.ops.selection import str_to_datetime, str_to_timedelta def test_gen_data(): @@ -376,3 +378,23 @@ def test_sliced_dataset_can_instantiate_with_right_dimensions(): ds = mdp.create_dataset(config) # We pick a 10x10km slice of the data which should result in 16 grid points. assert ds.state.shape == (2, 49, 16) + + +def test_allow_timedelta_in_config(): + """ + The sliced example has a 10x10 km slice, so there should be 4x4 = 16 points herekj. + """ + fp = "tests/resources/sliced_example.danra.yaml" + config = mdp.Config.from_yaml(open(fp)) + config.output.coord_ranges["time"].start = str_to_datetime( + config.output.coord_ranges["time"].start + ) + config.output.coord_ranges["time"].end = str_to_datetime( + config.output.coord_ranges["time"].end + ) + config.output.coord_ranges["time"].step = str_to_timedelta( + config.output.coord_ranges["time"].step + ) + ds = mdp.create_dataset(config) + # We pick a 10x10km slice of the data which should result in 16 grid points. + assert ds.state.shape == (2, 49, 16) From 824d45dc95a8d9fb6ce8110d54e9af9a16bf3a98 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Wed, 12 Feb 2025 17:40:59 +0100 Subject: [PATCH 06/31] remove tests for checking time is in data and timestep is correct --- tests/test_from_config.py | 166 +++++++++++++++++++------------------- 1 file changed, 83 insertions(+), 83 deletions(-) diff --git a/tests/test_from_config.py b/tests/test_from_config.py index 515ce83..4e835c3 100644 --- a/tests/test_from_config.py +++ b/tests/test_from_config.py @@ -114,89 +114,89 @@ def test_merging_static_and_surface_analysis(): mdp.create_dataset_zarr(fp_config=fp_config) -@pytest.mark.parametrize("source_data_contains_time_range", [True, False]) -@pytest.mark.parametrize( - "time_stepsize", - [testdata.DT_ANALYSIS, testdata.DT_ANALYSIS * 2, testdata.DT_ANALYSIS / 2], -) -def test_time_selection(source_data_contains_time_range, time_stepsize): - """ - Check that time selection works as expected, so that when source - data doesn't contain the time range specified in the config and exception - is raised, and otherwise that the correct timesteps are in the output - """ - - tmpdir = tempfile.TemporaryDirectory() - datasets = testdata.create_data_collection( - data_kinds=["surface_analysis", "static"], fp_root=tmpdir.name - ) - - t_start_dataset = testdata.T_START - t_end_dataset = t_start_dataset + (testdata.NT_ANALYSIS - 1) * testdata.DT_ANALYSIS - - if source_data_contains_time_range: - t_start_config = t_start_dataset - t_end_config = t_end_dataset - else: - t_start_config = t_start_dataset - testdata.DT_ANALYSIS - t_end_config = t_end_dataset + testdata.DT_ANALYSIS - - config = dict( - schema_version=testdata.SCHEMA_VERSION, - dataset_version="v0.1.0", - output=dict( - variables=dict( - static=["grid_index", "feature"], - state=["time", "grid_index", "feature"], - forcing=["time", "grid_index", "feature"], - ), - coord_ranges=dict( - time=dict( - start=t_start_config.isoformat(), - end=t_end_config.isoformat(), - step=isodate.duration_isoformat(time_stepsize), - ) - ), - ), - inputs=dict( - danra_surface=dict( - path=datasets["surface_analysis"], - dims=["analysis_time", "x", "y"], - variables=testdata.DEFAULT_SURFACE_ANALYSIS_VARS, - dim_mapping=dict( - time=dict( - method="rename", - dim="analysis_time", - ), - grid_index=dict( - method="stack", - dims=["x", "y"], - ), - feature=dict( - method="stack_variables_by_var_name", - name_format="{var_name}", - ), - ), - target_output_variable="forcing", - ), - ), - ) - - # write yaml config to file - fn_config = "config.yaml" - fp_config = Path(tmpdir.name) / fn_config - with open(fp_config, "w") as f: - yaml.dump(config, f) - - # run the main function - if source_data_contains_time_range and time_stepsize == testdata.DT_ANALYSIS: - mdp.create_dataset_zarr(fp_config=fp_config) - else: - print( - f"Expecting ValueError for source_data_contains_time_range={source_data_contains_time_range} and time_stepsize={time_stepsize}" - ) - with pytest.raises(ValueError): - mdp.create_dataset_zarr(fp_config=fp_config) +# @pytest.mark.parametrize("source_data_contains_time_range", [True, False]) +# @pytest.mark.parametrize( +# "time_stepsize", +# [testdata.DT_ANALYSIS, testdata.DT_ANALYSIS * 2, testdata.DT_ANALYSIS / 2], +# ) +# def test_time_selection(source_data_contains_time_range, time_stepsize): +# """ +# Check that time selection works as expected, so that when source +# data doesn't contain the time range specified in the config and exception +# is raised, and otherwise that the correct timesteps are in the output +# """ +# +# tmpdir = tempfile.TemporaryDirectory() +# datasets = testdata.create_data_collection( +# data_kinds=["surface_analysis", "static"], fp_root=tmpdir.name +# ) +# +# t_start_dataset = testdata.T_START +# t_end_dataset = t_start_dataset + (testdata.NT_ANALYSIS - 1) * testdata.DT_ANALYSIS +# +# if source_data_contains_time_range: +# t_start_config = t_start_dataset +# t_end_config = t_end_dataset +# else: +# t_start_config = t_start_dataset - testdata.DT_ANALYSIS +# t_end_config = t_end_dataset + testdata.DT_ANALYSIS +# +# config = dict( +# schema_version=testdata.SCHEMA_VERSION, +# dataset_version="v0.1.0", +# output=dict( +# variables=dict( +# static=["grid_index", "feature"], +# state=["time", "grid_index", "feature"], +# forcing=["time", "grid_index", "feature"], +# ), +# coord_ranges=dict( +# time=dict( +# start=t_start_config.isoformat(), +# end=t_end_config.isoformat(), +# step=isodate.duration_isoformat(time_stepsize), +# ) +# ), +# ), +# inputs=dict( +# danra_surface=dict( +# path=datasets["surface_analysis"], +# dims=["analysis_time", "x", "y"], +# variables=testdata.DEFAULT_SURFACE_ANALYSIS_VARS, +# dim_mapping=dict( +# time=dict( +# method="rename", +# dim="analysis_time", +# ), +# grid_index=dict( +# method="stack", +# dims=["x", "y"], +# ), +# feature=dict( +# method="stack_variables_by_var_name", +# name_format="{var_name}", +# ), +# ), +# target_output_variable="forcing", +# ), +# ), +# ) +# +# # write yaml config to file +# fn_config = "config.yaml" +# fp_config = Path(tmpdir.name) / fn_config +# with open(fp_config, "w") as f: +# yaml.dump(config, f) +# +# # run the main function +# if source_data_contains_time_range and time_stepsize == testdata.DT_ANALYSIS: +# mdp.create_dataset_zarr(fp_config=fp_config) +# else: +# print( +# f"Expecting ValueError for source_data_contains_time_range={source_data_contains_time_range} and time_stepsize={time_stepsize}" +# ) +# with pytest.raises(ValueError): +# mdp.create_dataset_zarr(fp_config=fp_config) @pytest.mark.parametrize("use_common_feature_var_name", [True, False]) From ff249e92d8176b44076f2d26c64605554400f451 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Thu, 13 Feb 2025 18:19:29 +0100 Subject: [PATCH 07/31] it is specified in the docstring that it has to be a dit --- mllam_data_prep/ops/selection.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mllam_data_prep/ops/selection.py b/mllam_data_prep/ops/selection.py index bb49fc2..1fe0f4e 100644 --- a/mllam_data_prep/ops/selection.py +++ b/mllam_data_prep/ops/selection.py @@ -81,10 +81,5 @@ def select_by_kwargs(ds, **coord_ranges): len(ds[coord]) > 0 ), f"You have selected an empty range {sel_start}:{sel_end} for coordinate {coord}" - elif isinstance(selection, list): - ds = ds.sel({coord: selection}) - else: - raise NotImplementedError( - f"Selection for coordinate {coord} must be a list or a dict" ) return ds From 01df7c35f5b925dc10a10ae0bee700af8496ef26 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Thu, 13 Feb 2025 18:19:56 +0100 Subject: [PATCH 08/31] check selection and warn --- mllam_data_prep/ops/selection.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mllam_data_prep/ops/selection.py b/mllam_data_prep/ops/selection.py index 1fe0f4e..e8274f4 100644 --- a/mllam_data_prep/ops/selection.py +++ b/mllam_data_prep/ops/selection.py @@ -82,4 +82,11 @@ def select_by_kwargs(ds, **coord_ranges): ), f"You have selected an empty range {sel_start}:{sel_end} for coordinate {coord}" ) + check_selection(ds, coord, sel_start, sel_end) return ds + +def check_selection(ds, coord, sel_start, sel_end): + if ds[coord].values.min() < sel_start or ds[coord].values.max() > sel_end: + warnings.warn( + f"Selection points is outside the range of the range of {coord}, the data spans from {ds[coord].values.min()} to {ds[coord].values.max()}" + ) From ede3b9a2fc037717fce6e93f8c5c3e53d740f46e Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Thu, 13 Feb 2025 18:20:49 +0100 Subject: [PATCH 09/31] xarray handles this check --- mllam_data_prep/ops/selection.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mllam_data_prep/ops/selection.py b/mllam_data_prep/ops/selection.py index e8274f4..544241b 100644 --- a/mllam_data_prep/ops/selection.py +++ b/mllam_data_prep/ops/selection.py @@ -56,8 +56,6 @@ def select_by_kwargs(ds, **coord_ranges): """ for coord, selection in coord_ranges.items(): - if coord not in ds.coords: - raise ValueError(f"Coordinate {coord} not found in dataset") if isinstance(selection, Range): if selection.start is None and selection.end is None: raise ValueError( From 570bc65cd702f7fa4d9621c5a8a1e7a831aa3b3b Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Thu, 13 Feb 2025 18:21:11 +0100 Subject: [PATCH 10/31] is always range --- mllam_data_prep/ops/selection.py | 34 +++++++++++++------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/mllam_data_prep/ops/selection.py b/mllam_data_prep/ops/selection.py index 544241b..6274538 100644 --- a/mllam_data_prep/ops/selection.py +++ b/mllam_data_prep/ops/selection.py @@ -56,31 +56,25 @@ def select_by_kwargs(ds, **coord_ranges): """ for coord, selection in coord_ranges.items(): - if isinstance(selection, Range): - if selection.start is None and selection.end is None: - raise ValueError( - f"Selection for coordinate {coord} must have either 'start' and 'end' given" - ) - sel_start = str_to_datetime(selection.start) - sel_end = str_to_datetime(selection.end) - sel_step = str_to_timedelta(selection.step) + sel_start = str_to_datetime(selection.start) + sel_end = str_to_datetime(selection.end) + sel_step = str_to_timedelta(selection.step) - assert sel_start != sel_end, "Start and end cannot be the same" + assert sel_start != sel_end, "Start and end cannot be the same" - if coord == "time" and sel_step is not None: - warnings.warn( - "Step size for time coordinate is not supported and is ignored" - ) - sel_step = None + if coord == "time" and sel_step is not None: + warnings.warn( + "Step size for time coordinate is not supported and is ignored" + ) + sel_step = None - ds = ds.sel({coord: slice(sel_start, sel_end, sel_step)}) + check_selection(ds, coord, sel_start, sel_end) + ds = ds.sel({coord: slice(sel_start, sel_end, sel_step)}) - assert ( - len(ds[coord]) > 0 - ), f"You have selected an empty range {sel_start}:{sel_end} for coordinate {coord}" + assert ( + len(ds[coord]) > 0 + ), f"You have selected an empty range {sel_start}:{sel_end} for coordinate {coord}" - ) - check_selection(ds, coord, sel_start, sel_end) return ds def check_selection(ds, coord, sel_start, sel_end): From 4dcba28e6b9872b5386d61f4836e8050093b8f2b Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Thu, 13 Feb 2025 18:24:00 +0100 Subject: [PATCH 11/31] better warning --- mllam_data_prep/ops/selection.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mllam_data_prep/ops/selection.py b/mllam_data_prep/ops/selection.py index 6274538..9becf20 100644 --- a/mllam_data_prep/ops/selection.py +++ b/mllam_data_prep/ops/selection.py @@ -2,8 +2,6 @@ import pandas as pd -from mllam_data_prep.config import Range - def str_to_datetime(s): if isinstance(s, pd.Timestamp): @@ -77,8 +75,9 @@ def select_by_kwargs(ds, **coord_ranges): return ds + def check_selection(ds, coord, sel_start, sel_end): if ds[coord].values.min() < sel_start or ds[coord].values.max() > sel_end: warnings.warn( - f"Selection points is outside the range of the range of {coord}, the data spans from {ds[coord].values.min()} to {ds[coord].values.max()}" + f"Endpoints are outside the range of {coord}, the data spans [ {ds[coord].values.min()} : {ds[coord].values.max()} ] and the selection is [ {sel_start} : {sel_end} ]" ) From b23e426b67eaa37feeb4923ffb010e9c0bf38cd9 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Thu, 13 Feb 2025 19:36:25 +0100 Subject: [PATCH 12/31] make range accept timedelta and datetime --- mllam_data_prep/config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mllam_data_prep/config.py b/mllam_data_prep/config.py index d248121..e1c4e19 100644 --- a/mllam_data_prep/config.py +++ b/mllam_data_prep/config.py @@ -1,4 +1,5 @@ from dataclasses import dataclass, field +from datetime import datetime, timedelta from typing import Any, Dict, List, Optional, Union import dataclass_wizard @@ -72,9 +73,9 @@ class Range: then the entire range will be selected. """ - start: Union[str, int, float] - end: Union[str, int, float] - step: Union[str, int, float] = None + start: Union[str, int, float, datetime] + end: Union[str, int, float, datetime] + step: Union[str, int, float, timedelta] = None @dataclass From d19b304ed3801c971c51fcc5b2078cd65cf81ef0 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Thu, 13 Feb 2025 19:40:00 +0100 Subject: [PATCH 13/31] add test to test range --- tests/test_config.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_config.py b/tests/test_config.py index 5459db6..d5985da 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,7 +1,10 @@ +import datetime + import pytest from dataclass_wizard.errors import MissingFields, UnknownJSONKey import mllam_data_prep as mdp +from mllam_data_prep import config INVALID_EXTRA_FIELDS_CONFIG_YAML = """ schema_version: v0.1.0 @@ -119,3 +122,11 @@ def test_get_config_nested(): assert input_config.target_output_variable is not None with pytest.raises(AttributeError): input_config.foobarfield + + +def test_that_range_accepts_datetime(): + start = datetime.datetime(1990, 9, 3, 0, 0) + end = datetime.datetime(1990, 9, 4, 0, 0) + step = "PT3H" + + range_ = config.Range(start=start, end=end, step=step) From b38191cf602f8197a62bbd8ee355d275b6f870e7 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Thu, 13 Feb 2025 20:46:40 +0100 Subject: [PATCH 14/31] simplify caasts --- mllam_data_prep/ops/selection.py | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/mllam_data_prep/ops/selection.py b/mllam_data_prep/ops/selection.py index 9becf20..705b55f 100644 --- a/mllam_data_prep/ops/selection.py +++ b/mllam_data_prep/ops/selection.py @@ -3,28 +3,16 @@ import pandas as pd -def str_to_datetime(s): - if isinstance(s, pd.Timestamp): - return s - elif isinstance(s, str): - try: - return pd.Timestamp(s) - except ValueError: - return s - else: - return s +def str_to_timestamp(s): + if isinstance(s, str): + return pd.Timestamp(s) + return s def str_to_timedelta(s): - if isinstance(s, pd.Timedelta): - return s - elif isinstance(s, str): - try: - return pd.to_timedelta(s) - except ValueError: - return s - else: - return s + if isinstance(s, str): + return pd.to_time_delta(s) + return s def select_by_kwargs(ds, **coord_ranges): From a615ae7addac51eb4b1f3cfc6c53d37cb00fa417 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Fri, 14 Feb 2025 13:17:30 +0100 Subject: [PATCH 15/31] parametrize step_sizes --- tests/test_selection.py | 42 ++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/tests/test_selection.py b/tests/test_selection.py index 9975bd5..216d2e4 100644 --- a/tests/test_selection.py +++ b/tests/test_selection.py @@ -1,5 +1,7 @@ -from datetime import datetime +import pickle as pkl +import isodate +import numpy as np import pytest import xarray as xr @@ -12,7 +14,8 @@ def ds(): Load the height_levels.zarr dataset """ fp = "https://mllam-test-data.s3.eu-north-1.amazonaws.com/height_levels.zarr" - return xr.open_zarr(fp) + ds = xr.open_zarr(fp) + return ds def test_range_slice_within_range(ds): @@ -52,14 +55,39 @@ def test_error_on_empty_range(ds, x_start, x_end): def test_can_slice_time(ds): - """ - test if the slice is within the specified range - """ - start = "1990-09-03T00:03" + start = "1990-09-01T00:00" + end = "1990-09-09T00:00" + coord_ranges = { + "time": mdp.config.Range(start=start, end=end), + } + + ds = mdp.ops.selection.select_by_kwargs(ds, **coord_ranges) + + +@pytest.mark.parametrize("step", ["PT6H", "PT3H"]) +def test_if_steps_time(ds, step): + start = "1990-09-01T00:00" end = "1990-09-09T00:00" - step = "PT3H" coord_ranges = { "time": mdp.config.Range(start=start, end=end, step=step), } ds = mdp.ops.selection.select_by_kwargs(ds, **coord_ranges) + + td = isodate.parse_duration(step) + timestep_in_slice = np.timedelta64(int(td.total_seconds()), "s") + timestep_in_dataset = np.diff(ds.time)[0] + + assert timestep_in_slice == timestep_in_dataset + + +def test_raises_if_time_step_is_not_multiple(ds): + step = "PT5H" + start = "1990-09-01T03:00" + end = "1990-09-09T00:00" + coord_ranges = { + "time": mdp.config.Range(start=start, end=end, step=step), + } + + with pytest.raises(ValueError): + ds = mdp.ops.selection.select_by_kwargs(ds, **coord_ranges) From 87760f9224d6f13eaeaf244ef8ca59b65efdc742 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Fri, 14 Feb 2025 13:18:02 +0100 Subject: [PATCH 16/31] to timestamp if coordinate is time --- mllam_data_prep/ops/selection.py | 41 ++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/mllam_data_prep/ops/selection.py b/mllam_data_prep/ops/selection.py index 705b55f..41cac87 100644 --- a/mllam_data_prep/ops/selection.py +++ b/mllam_data_prep/ops/selection.py @@ -1,17 +1,18 @@ import warnings +import numpy as np import pandas as pd -def str_to_timestamp(s): +def to_timestamp(s): if isinstance(s, str): return pd.Timestamp(s) return s -def str_to_timedelta(s): +def to_timedelta(s): if isinstance(s, str): - return pd.to_time_delta(s) + return np.timedelta64(pd.to_timedelta(s)) return s @@ -42,17 +43,16 @@ def select_by_kwargs(ds, **coord_ranges): """ for coord, selection in coord_ranges.items(): - sel_start = str_to_datetime(selection.start) - sel_end = str_to_datetime(selection.end) - sel_step = str_to_timedelta(selection.step) + sel_start = selection.start + sel_end = selection.end + sel_step = selection.step - assert sel_start != sel_end, "Start and end cannot be the same" + if coord == "time": + sel_start = to_timestamp(selection.start) + sel_end = to_timestamp(selection.end) + sel_step = get_time_step(sel_step, ds) - if coord == "time" and sel_step is not None: - warnings.warn( - "Step size for time coordinate is not supported and is ignored" - ) - sel_step = None + assert sel_start != sel_end, "Start and end cannot be the same" check_selection(ds, coord, sel_start, sel_end) ds = ds.sel({coord: slice(sel_start, sel_end, sel_step)}) @@ -64,8 +64,23 @@ def select_by_kwargs(ds, **coord_ranges): return ds +def get_time_step(sel_step, ds): + if sel_step is None: + return None + + dataset_timedelta = ds.time[1] - ds.time[0] + sel_timedelta = to_timedelta(sel_step) + step = sel_timedelta / dataset_timedelta + if step % 1 != 0: + raise ValueError( + f"The chosen stepsize {sel_step} is not multiple of the stepsize in the dataset {dataset_timedelta}" + ) + + return int(step) + + def check_selection(ds, coord, sel_start, sel_end): if ds[coord].values.min() < sel_start or ds[coord].values.max() > sel_end: warnings.warn( - f"Endpoints are outside the range of {coord}, the data spans [ {ds[coord].values.min()} : {ds[coord].values.max()} ] and the selection is [ {sel_start} : {sel_end} ]" + f"\nEndpoints are outside the range of {coord}, \nDataset span: [ {ds[coord].values.min()} : {ds[coord].values.max()} ] \nChosen slice: [ {sel_start} : {sel_end} ]\n" ) From 747986a792aa0b764dfada36e51d496fabd88c24 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Fri, 14 Feb 2025 13:33:53 +0100 Subject: [PATCH 17/31] add tests instantiating from string end_points and datetime endpoints --- tests/test_config.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_config.py b/tests/test_config.py index d5985da..e3f68e6 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -113,6 +113,16 @@ def test_get_config_issues(): """ +def test_can_load_config_with_datetime_object_in_time_range(): + fp = "tests/resources/sliced_example.danra.yaml" + mdp.Config.from_yaml_file(fp) + + +def test_can_load_config_with_datetime_string_in_time_range(): + fp = "tests/resources/sliced_example_with_datetime_strings.danra.yaml" + mdp.Config.from_yaml_file(fp) + + def test_get_config_nested(): config = mdp.Config.from_yaml(VALID_EXAMPLE_CONFIG_YAML) From da5c70cb2570b155afd7a333fc74340f956be25e Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Fri, 14 Feb 2025 13:34:20 +0100 Subject: [PATCH 18/31] move slicing to input data and create string yaml with string datetime --- tests/resources/sliced_example.danra.yaml | 9 ++- ...d_example_with_datetime_strings.danra.yaml | 61 +++++++++++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 tests/resources/sliced_example_with_datetime_strings.danra.yaml diff --git a/tests/resources/sliced_example.danra.yaml b/tests/resources/sliced_example.danra.yaml index 6d60d85..7015021 100644 --- a/tests/resources/sliced_example.danra.yaml +++ b/tests/resources/sliced_example.danra.yaml @@ -4,11 +4,6 @@ dataset_version: v0.1.0 output: variables: state: [time, grid_index, state_feature] - coord_ranges: - time: - start: 1990-09-03T00:00 - end: 1990-09-09T00:00 - step: PT3H chunking: time: 1 splitting: @@ -58,5 +53,9 @@ inputs: y: start: -50000 end: -40000 + time: + start: 1990-09-03T00:00 + end: 1990-09-09T00:00 + step: PT3H target_output_variable: state diff --git a/tests/resources/sliced_example_with_datetime_strings.danra.yaml b/tests/resources/sliced_example_with_datetime_strings.danra.yaml new file mode 100644 index 0000000..2225619 --- /dev/null +++ b/tests/resources/sliced_example_with_datetime_strings.danra.yaml @@ -0,0 +1,61 @@ +schema_version: v0.6.0 +dataset_version: v0.1.0 + +output: + variables: + state: [time, grid_index, state_feature] + chunking: + time: 1 + splitting: + dim: time + splits: + train: + start: 1990-09-03T00:00 + end: 1990-09-06T00:00 + compute_statistics: + ops: [mean, std, diff_mean, diff_std] + dims: [grid_index, time] + val: + start: 1990-09-06T00:00 + end: 1990-09-07T00:00 + test: + start: 1990-09-07T00:00 + end: 1990-09-09T00:00 + +inputs: + danra_height_levels: + path: https://mllam-test-data.s3.eu-north-1.amazonaws.com/height_levels.zarr + dims: [time, x, y, altitude] + variables: + u: + altitude: + values: [100,] + units: m + v: + altitude: + values: [100, ] + units: m + dim_mapping: + time: + method: rename + dim: time + state_feature: + method: stack_variables_by_var_name + dims: [altitude] + name_format: "{var_name}{altitude}m" + grid_index: + method: stack + dims: [x, y] + coord_ranges: + x: + start: -50000 + end: -40000 + y: + start: -50000 + end: -40000 + time: + start: "1990-09-03T00:00" + end: "1990-09-09T00:00" + step: "PT3H" + + target_output_variable: state From 0f1466fbeeeff1652539e2f49fc247f676101d75 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Fri, 14 Feb 2025 13:36:38 +0100 Subject: [PATCH 19/31] remove redundant test --- tests/test_from_config.py | 85 --------------------------------------- 1 file changed, 85 deletions(-) diff --git a/tests/test_from_config.py b/tests/test_from_config.py index 4e835c3..a86b582 100644 --- a/tests/test_from_config.py +++ b/tests/test_from_config.py @@ -114,91 +114,6 @@ def test_merging_static_and_surface_analysis(): mdp.create_dataset_zarr(fp_config=fp_config) -# @pytest.mark.parametrize("source_data_contains_time_range", [True, False]) -# @pytest.mark.parametrize( -# "time_stepsize", -# [testdata.DT_ANALYSIS, testdata.DT_ANALYSIS * 2, testdata.DT_ANALYSIS / 2], -# ) -# def test_time_selection(source_data_contains_time_range, time_stepsize): -# """ -# Check that time selection works as expected, so that when source -# data doesn't contain the time range specified in the config and exception -# is raised, and otherwise that the correct timesteps are in the output -# """ -# -# tmpdir = tempfile.TemporaryDirectory() -# datasets = testdata.create_data_collection( -# data_kinds=["surface_analysis", "static"], fp_root=tmpdir.name -# ) -# -# t_start_dataset = testdata.T_START -# t_end_dataset = t_start_dataset + (testdata.NT_ANALYSIS - 1) * testdata.DT_ANALYSIS -# -# if source_data_contains_time_range: -# t_start_config = t_start_dataset -# t_end_config = t_end_dataset -# else: -# t_start_config = t_start_dataset - testdata.DT_ANALYSIS -# t_end_config = t_end_dataset + testdata.DT_ANALYSIS -# -# config = dict( -# schema_version=testdata.SCHEMA_VERSION, -# dataset_version="v0.1.0", -# output=dict( -# variables=dict( -# static=["grid_index", "feature"], -# state=["time", "grid_index", "feature"], -# forcing=["time", "grid_index", "feature"], -# ), -# coord_ranges=dict( -# time=dict( -# start=t_start_config.isoformat(), -# end=t_end_config.isoformat(), -# step=isodate.duration_isoformat(time_stepsize), -# ) -# ), -# ), -# inputs=dict( -# danra_surface=dict( -# path=datasets["surface_analysis"], -# dims=["analysis_time", "x", "y"], -# variables=testdata.DEFAULT_SURFACE_ANALYSIS_VARS, -# dim_mapping=dict( -# time=dict( -# method="rename", -# dim="analysis_time", -# ), -# grid_index=dict( -# method="stack", -# dims=["x", "y"], -# ), -# feature=dict( -# method="stack_variables_by_var_name", -# name_format="{var_name}", -# ), -# ), -# target_output_variable="forcing", -# ), -# ), -# ) -# -# # write yaml config to file -# fn_config = "config.yaml" -# fp_config = Path(tmpdir.name) / fn_config -# with open(fp_config, "w") as f: -# yaml.dump(config, f) -# -# # run the main function -# if source_data_contains_time_range and time_stepsize == testdata.DT_ANALYSIS: -# mdp.create_dataset_zarr(fp_config=fp_config) -# else: -# print( -# f"Expecting ValueError for source_data_contains_time_range={source_data_contains_time_range} and time_stepsize={time_stepsize}" -# ) -# with pytest.raises(ValueError): -# mdp.create_dataset_zarr(fp_config=fp_config) - - @pytest.mark.parametrize("use_common_feature_var_name", [True, False]) def test_feature_collision(use_common_feature_var_name): """ From 3cd8933d6395b91e4f787264a6e54c346ae490d0 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Fri, 14 Feb 2025 13:39:22 +0100 Subject: [PATCH 20/31] better names in tests --- tests/test_selection.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_selection.py b/tests/test_selection.py index 216d2e4..2765353 100644 --- a/tests/test_selection.py +++ b/tests/test_selection.py @@ -65,7 +65,7 @@ def test_can_slice_time(ds): @pytest.mark.parametrize("step", ["PT6H", "PT3H"]) -def test_if_steps_time(ds, step): +def test_if_step_time_matches_output(ds, step): start = "1990-09-01T00:00" end = "1990-09-09T00:00" coord_ranges = { @@ -75,13 +75,13 @@ def test_if_steps_time(ds, step): ds = mdp.ops.selection.select_by_kwargs(ds, **coord_ranges) td = isodate.parse_duration(step) - timestep_in_slice = np.timedelta64(int(td.total_seconds()), "s") + timestep_chosen_in_slice = np.timedelta64(int(td.total_seconds()), "s") timestep_in_dataset = np.diff(ds.time)[0] - assert timestep_in_slice == timestep_in_dataset + assert timestep_chosen_in_slice == timestep_in_dataset -def test_raises_if_time_step_is_not_multiple(ds): +def test_raises_if_time_step_is_not_multiple_of_dataset_frequency(ds): step = "PT5H" start = "1990-09-01T03:00" end = "1990-09-09T00:00" From 381bc6451c2c38b1ed3b3d6da53991e8a55c0c2b Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Fri, 14 Feb 2025 13:42:44 +0100 Subject: [PATCH 21/31] better warning --- mllam_data_prep/ops/selection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllam_data_prep/ops/selection.py b/mllam_data_prep/ops/selection.py index 41cac87..2b68b98 100644 --- a/mllam_data_prep/ops/selection.py +++ b/mllam_data_prep/ops/selection.py @@ -82,5 +82,5 @@ def get_time_step(sel_step, ds): def check_selection(ds, coord, sel_start, sel_end): if ds[coord].values.min() < sel_start or ds[coord].values.max() > sel_end: warnings.warn( - f"\nEndpoints are outside the range of {coord}, \nDataset span: [ {ds[coord].values.min()} : {ds[coord].values.max()} ] \nChosen slice: [ {sel_start} : {sel_end} ]\n" + f"\nChosen slice exceeds the range of {coord} in the dataset.\n Dataset span: [ {ds[coord].values.min()} : {ds[coord].values.max()} ]\n Chosen slice: [ {sel_start} : {sel_end} ]\n" ) From 85b5c27dba496f969de05ba7742968aa6deaecf5 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Fri, 14 Feb 2025 13:44:30 +0100 Subject: [PATCH 22/31] remove test that doesn't test anything --- tests/test_from_config.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tests/test_from_config.py b/tests/test_from_config.py index a86b582..51ff140 100644 --- a/tests/test_from_config.py +++ b/tests/test_from_config.py @@ -293,23 +293,3 @@ def test_sliced_dataset_can_instantiate_with_right_dimensions(): ds = mdp.create_dataset(config) # We pick a 10x10km slice of the data which should result in 16 grid points. assert ds.state.shape == (2, 49, 16) - - -def test_allow_timedelta_in_config(): - """ - The sliced example has a 10x10 km slice, so there should be 4x4 = 16 points herekj. - """ - fp = "tests/resources/sliced_example.danra.yaml" - config = mdp.Config.from_yaml(open(fp)) - config.output.coord_ranges["time"].start = str_to_datetime( - config.output.coord_ranges["time"].start - ) - config.output.coord_ranges["time"].end = str_to_datetime( - config.output.coord_ranges["time"].end - ) - config.output.coord_ranges["time"].step = str_to_timedelta( - config.output.coord_ranges["time"].step - ) - ds = mdp.create_dataset(config) - # We pick a 10x10km slice of the data which should result in 16 grid points. - assert ds.state.shape == (2, 49, 16) From 5e217d1b752b432042751f96070ecafc840965a6 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Fri, 14 Feb 2025 13:44:44 +0100 Subject: [PATCH 23/31] remove obvious comment --- tests/test_from_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_from_config.py b/tests/test_from_config.py index 51ff140..b8444c1 100644 --- a/tests/test_from_config.py +++ b/tests/test_from_config.py @@ -277,7 +277,6 @@ def test_config_revision_examples(fp_example): """ tmpdir = tempfile.TemporaryDirectory() - # copy example to tempdir fp_config_copy = Path(tmpdir.name) / fp_example.name shutil.copy(fp_example, fp_config_copy) From 96f5d938ba059116118ce2abab7abd5c0068bee7 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Fri, 14 Feb 2025 14:30:07 +0100 Subject: [PATCH 24/31] remove import --- height_levels.pkl | Bin 0 -> 26919 bytes tests/test_from_config.py | 1 - 2 files changed, 1 deletion(-) create mode 100644 height_levels.pkl diff --git a/height_levels.pkl b/height_levels.pkl new file mode 100644 index 0000000000000000000000000000000000000000..1211b51846be6b9e2f0654ae8876bc970f75df36 GIT binary patch literal 26919 zcmdtq3tSXc|3C0Wxe6#M7*)W zUCr_mK51f#p^uv7r8O^^TGFGLrWP8O_fQeZOZ~p*e0Bu{ORcBZ^Z!3RuX*n|b7tnu znVIjI8JuHvpW9j|;s3KW&r)S(shD>dkMrBr+v}U`_E6(|Ajy+_$J!zU_1QJbkVXrzV=X{J_VQEN3)y>^mTZ?c6;o8|G;M~)6gGvbY`6K0BZix)-c-m{OJAkRXrw|)jV|4uWNWNctJH#4nXXMw z(Ng7eZ9&+W(!>^28Zw;Q<}(Z#CWBr_jZkAo9#wB*rom#g+ay%O+iPM?;;o|Enk^ct zdJ|SBG$LUfw0l~HR?aGOC}*O;^Ex6LENl5Whtou<}opo_NW<-TGcW|t*2 zw7K=QCnY5f{+<5ECkYa>J;mm!7AzTQCakKF+K_I{)S65NYV;L^;4qQZu6uvCEKhu*<+FEspiMtp5S5gG6|N{R6VP;&4<>gPBZDI zP)cG+v3VFWHCo!HTddvIoQ|h9!$c>X(xf%2GF5c=XigZNDXDbMuw$EKYeYZw%1J7{ zMaxp`{L!UY*oIm2G*!k78x}KG88;|#K+^C*Lz9NvC(seSeGFAL4@-v5OeddPy4_-H z#M^+5y-ri>U{q&Q$$(+ni59&oGnTd2Q>>`79c;cRheIOtG^NlEHNr_}RG9@jtNHO| zIjV`;FeNo|I=h;D_!+HWMXH;WFIQ)gRFlc5r6b@wkpJKdRxjkO)R~k9x)$i9Z=}rB zsx)-A>$541S`3jcIfnpR0yo>`TlX4jLgm2-Kf`6)6>;Rjc*DQ%x1b2 z8pn?qp%|#sYlml>%-VFeRd$Z^Q(VY2WEd=Tt(n+KW^>2x4rB|OdUx%buGg#5L+PNH zL)nQQW{M2cT0%1nnPwrhTbL?cHN}vjnrxzLDZQ(p)lC%4N<<3hW#3Gt*Bd5lHA-!k zTFXNa?M;oy(Uesy$zUF6uw-ZkWMWo^wE`zR`;OeXX&RK!|Rc4B1npk%EDO^jqHe+jsnqZ1c)E;7MiZs9c zsQR>Ib*$o7v696UuTc?vt+&q}VNj9~E%c=tHp8MdR$!vlHcDZ2QQ}&4v(U+HN}(%W zWwr}#C=^e#3sT`R{zE3T7i7*~of8uqBnCB{X%|9;a5@cZ1R^1-ZWt1JBud=q|5QF& z=;K_+69_DV9I-^`pWq?HCb$W4EM%~@7M>E~S@j3oTJYG%E-mWX(k&FLd1KgZo+h=5 zE?Rb8EROX?M_BcueI}i*LIQ;fA#no53pWazZdrD%%|~y@n5aZl^QL`thJ1f^v1nA8 znmV%th0R~1otUZBnw0#+wp*xfV})m&`=AumbZ+tHqWl+x>CSvs`y4tggc(AvGiz3e zU=!YOCip>FLi1;b2y+suo1Nmuy*B&ezu2h4@;_)Yp-5P*6+T3Rp_PO+wNeV3pQ~ZC zDwFvx?Lzx=|LwBt&USIj{Nq-UCAcMMa7NN;8!AxKP!>)(no{0_oOOTLKsDlt&D~5r znw>WpbruHd>vS1(=Rv{Di_S4AopW|uBOdvb=*pWS8pfm;CTX9dJCY2Io!YwTFfBVJ zCuyaEC0(b{ndz+cF$ok-lUO$_9JP5lp_Vpbt;pqJzsj@mp>QBPP|IU#TeyKh>!lim!>O-6F2#WJsx<#?jt*KBU zHguCngsZmOjW1lIcvV&J!qm(O^))kJ$9=c7=to<=v$37$-5-VN1kDU~K}1rla6Csp zALKenx*bWKXS&MfO;ML_%_;EOn$z=wp7q{py1~*;G|&?R#W01fgB3C@72(Lus$r{ZyTZie0<^W(upPncOV@uSAb~IwzZ3LfLJ+ zA%pJZq=r=KWE~416=LlDoSR#OvHa=g|{WtxP`kH>y|6ff%ss5(d{%uXq z)^z_(pHa8zh1Gw29}(1$2L^T%!GAtTV7N;M$?my?)hbiX3;lJfVSN|6u=@V@_KkY> zw&;)dHnG9&z2!gu?!7Ily|*jwwzoz1zqcRM-CK*zLzzww0CY8-%xogTiOjgr&So z7QLySVqZ4t#3n49it%j1u1Qg(iLhFn#V8WrCoDGiG!}Yop1KS?@ij@en9NGLo1=Fa z?16$k7t#U0<7q*72oG77ne=+b1Mkb&^9p;x!tNdUGl} z5r)=&P3W3s^Hs1XAk!dL6i1T3D#Y!q60e|yNLxepGLBZ_!55^5Tq9o$ugjb(3rnFl zk95bY%+O9&x>hJwP?Om_TuJs8$DO|&6PD4A)3ZDsHFKs#ZDx;hVKy&zbAgv`36dH1 zSho~A-gCLJcVa?hUaYqUPkRzOd(Gv}pXr35)TqPktYyV0+MalkCUmdCc165IzuB{D zV)e7Cd!1)hDj3^<-dfTF1^s8H2T5AHc$uw>ukZ@}vbCvir)g)j>^Xq$`kV1Rk8@_Y zc3BK;K+F2yna!$povpRGbFE6fF2xka_ny|(TB$T;)=& zMP+MUvocsJtVUgKZ;vlV3k-hL^OAcKYs?31jGL@6exnIHFrtKZ%5`Mez2*Zoa9)3D zLv}RdM(WG_d?cmmKR`O{Vd7lp?CDwPgK0=nR z=<=qFbR5{KjvxzeDSGrJtsQ^#9!MrXZ83Hsh zZ9iGCz@$7vT6cOx{75GM@|ODrQvPtKZ!1Y@?B*4cCbWEd($q#|>AcdUK+@sJZ2JgV zaMrz|BWVqMYgaci`H7IX`;c;BQ`!JhIy>=+p`>K<4*_Gz(o@guafeQhDhwB^u+i7Swu>FpIf$^ERAZIwT5&&J9qFVvLLr(o1LU} z?Z>}-MJ69h`0Ow#_x#>|l9YxUC!QxI;~Mm?Buihm`$+<5`JxU#_>cwPZd}uxw7Mna zv>}r_A0O3`l#euobR(rNy4~zeN>Hnw_6VN~+qa=a8jsKkB}SbR-P&UQQNRj(%4h;KM=@i<=(G)J*Mj z)GgCrTiBmOSipB@@4M+Q0u;JTmRBfbdD3;sL)PtxNWE8`aJZO^{>Igmtk2plp5hP zsc`-?@Zc~QFbEEL66SV?J32$jW3X>9%xVnRxx@17efj=$xderu;nL%<^Z;zV2aa(- z`vzFD3i`bV-h6+axeDi>g9lH*fWvUeS1@-6-0=yNtbu)( z!>mPc-5gk+2fJ9IFbOWz!_pMkdMq3>6xs*Cl0MKc0>;Z=&Lhwf2yZrm(QeRutry>~ z;tTNnk5Kj<)Re%2UGUgu82oQIVg<~97w(%2y=^e=C1}lpn=@dQ21bpC#;4(`r{L*+ zuw4{P?gSS~;o%lA&>t!qLffy=e1CTT0^NUt@?&u7f8fU5uwp9=Uk~*wVc`;3_7;@B z4xh<^^QXXrMi?*=4w(RRhr=C%phOP)_JCPk;JU|Pc`)qK1PUH->5ZOze@ZXG)~Df^ z@1gx5EZGD7w!!$1U``Qqya#W-4Wsj+c_u8L3eRUknGR}{uwW!SmH>nM!x7OiKMd|` z2fbUtxTetR1vlU7!S|=?3XD1njiqqaA$WQpY_}aIe+(CX2oEoVfeWExHnio!-7i3Q z3zVn9sVcZ}G^|L3;c-yk8y1GcvJOz%20qgq&i8=_fA7xsC!i7zIS+GB!X1a9WIyb? z6J~9K>(;>X<*>_QDA?iBS7E6Yww?^fq(i$JmW+jd3K%~S=JbV*2zXNlqaTIlAXw}R z&$~fcRTSSZ%|%#H29JFQgTIC&K8N{R;J$xD?+;*H0kqD8n{BY_B^Z?rjRv?%3r~-S z?S{eRc)0KhcsL3Mc7}?!(Do4A?GN4Eq5OIz-=C@FaN|#~;us7+0QFzM!cSq@dMI55 zpZOP@KOY`^9R|#RL!O7Z&%qrNq2yWEcLdBD4A;qFc@Nkn6bg^Sr4Pf>Ca|?99CI^* z?~na5EIAGRet_}cz?{9%u?^n*2u81l=B2Rs9e6$;%3gt*X|TWqkLh4=G8{1q<`04U z`a|ztFs>`KwuhTr!K$V(${QMg>&EwIRRuhK7PdP9lfQ)vzl4Xk!@y6Vq8QrVhr1U- z_c>6W2dBOWH(Fpt8VpZ?`Z2ID36{k{X&?AZH#ol|JlF;X1i~SWV6Fu2s0`=(CAk3m z{s^;C`F>E~-j+q7R(_zVE=$8TGH85u!bSU7>fiSusG)KbX z5P1GkC~E;Vez2fH_%_ANizyo~b>)VC4-5A|=|^z>doUm$=1zqY9n2aD%VVGr21{GQ zFLh$H8lFb(^o_Jc9KuR!(Tzg zCb)Y!l-uD(D-2JEg=3*~AeNUT43?KbVGE>Zem;N9JXrD)j5k2X zco-cIi=&{dEiCYd!Ph(U_2mBqy$?X^r?6@zG|q>ob71mw@bCnv7z}s!fbz%T#wIZQ zW+%R#!qZUt4V=FX2CRm;??A~bFv|qXl~5Q0OQYeK_OPTWjQ=f!Z^v;KMt=*7w?kPm zELaGG^I*OOdZ$2Z60GVCjUD0X<}g_T51*Iu?J17H-8-RtE!?;mhR=e9lc7`t=PO`9 zKbRW=B`si916Y2sBj1j26qbGt$E<@T1u*_K=*WiAsjzq$lsyRxI>X>#nC}j~FLmJC zu^xw2UqIsqc=}&3`3-pZd8iQJ?h#NP12=}k@RqR96H2eN=i8b80}R*)b2mcCGMM!y zET0Ai5tfdEV`5=RR~X+KI=o@@)pmS4#V4TbD_HOe3|-Pj`dK zkHEu?pyJx&d^@{;gz^%&aWf2G0So6s=}T~a1`HSvbDx5eD3~RM<$h53^)bGk(o=BE ze_+X07{3xa-h$CNu-FJ?6JWt07~BKqKL)*)3%in>eCRj2AI@-hH-&*kU1>eH_Vrb2S#uS*`2Py)g{6Y|4 zE_@f1z6%4gpyX*-9s)}nz>=eZd^yKDSo|6+NQL=NLTfNIUTV(EC+~rZ_n>?#3?B)l zVKBf8N=lpY<;pk0(l=p=2pzGoxHT-e+LV{i{|Z`{L!%WYkA;c|DEEcoWdVFS=@u9; z4@wNM{3%!}4cI1s_>);nt~B9JUj|j9VT=!KyQwjs@0JcvJOa0s_+$B(;Pfb{`pJ*y z$IORq2Sc};zC8a#5!|MPOPa#z+Z*xuDhrJ12-_a<;rVW};EBF)+eL4lzoY<89|l$K zFlK`npWjx1ZY|-7y`DUO+cdbOE1Z79gXgQ}z?cEB?KO9v@3sP-7zelc!6jQ8^7+$^ zQ1uv$IoN>bx19;yqTz`%Zajb60=Og*PXArP^HpnLj2gBLl5Dd^m>+8R`tJ_r_DjEU zUmgi#KfcA&4cox&)|;4Lc7q!m4;!w!&ePkQz?V(GV*a;P+=hMN_QltD`sD^NHnkG- z_gv*}4}~wkS;5n>SFd3HSh#)jWuAUn3S)CFVgBiI?)HRoheUfi%v1RbcV>r+NMAgU z^!js1Pdkfr+!>_(P9uHbC!`mgLOP`k=}tc)edz?!AD22LdrBtwx%~k1k2(0?a@ik7 z=wF?<{!UDP%)i@TMT+kKuOj>(m(X8Dgk}Ha7u4@xdRlN!;`;L?@HSl zcOL%dFdV!SezpeoUJTd13Oh}P|5C%23OKtjY!m`t3WC?&sQ13y;ZWHz>W#^R>*0>q z;kt=%sT|rLhB=q17pC&&y->3hj+h4HM!~4Au$?yyJVCuNE$99Tte6eU(%`{3xT6hR zS4q7vm0!9a+6!UMbZ8z2HIZL-i78YsCgQW=mg^$!l+}^8`E;_*2BQpq5DKw@gyt@h6gWEFHGfk z?1AgvgG;AE`$(7*2F+ejQ%b!tEjQw07&jY6rNMS_Ft81DucTg>%2#{^%a+4~R=8s< zTo(bC`a*jd^~SVZ&K78%2Q>ybA|A%Ig;Ce37pC&<4#2=w(0vB1m;lRqz=KWTj?>gD z({k&!!KLp&y9wqDf#&wqYoAJ7^~Fu4^3TkKUqr(dXQ`K_{Je!wmjnk% zV8~kPrD?v82A&CmUwlrzH07_zhIyT#?l|?*lt1VV7!m`0u23&c`Dd2GFGj%?-Z1YI z>aA(MP7en?LcR8$1!IO6{O({jcZUZi{>IaD_TJ(S?+TmGxyjQ9DsN!^I5>Rkb)Ihi z7(6iJSIj?C#T}jqo3FXX(+2|KoUBUBKX#SdTn-PsSHaVByx{P(E119kGWS3PoHOqd zPY=Idj``10uRXWtn)hF(UYqQYK)p6gH@krJchqaM^b+c|*?b-K+AJMTy*5i%QLoL? zpHi>Q(yvgj&C*HKYqNBVlSm(@Ub~YaC2=tI+RP)zpzs~M_AQ+G4GcN}mzFprBfj79 zPU=3G^Ev$2XAVi>=DhCHH^D7y91?b+mA&hbSi5VMubL{C6wf?wZ?jx38528c&#?8d z#iw$~s~Mb36h|5iD7Rpce^ z`Tekq{xar!U%~vP@VvZ&r@I`7`s}NiA5_UbzXo^z zo?dkgo}Y6A^TTg)_50u|;TBJycmECZmq2~L-+6k~k>BNxmTQMs_LjsjTjaqW-@;Mt z-FW^a6P&jVZfMeg=O3B?D_6o6*I|$NhJ600d2mu0oEPEF^EX)Gp|4@GtJ&>1A z>kOYC24`i%1qE>B=Wx?SxHl+>FL$&rJg0^?UWLAEVXGssv!n$t-=`x?NP^=nFl`}x zemk6X7A|P|5MOR(G~A?wduPIi*J?2A{2c5vA0~VX$NvP={2%7! zpO1pG#=`|K!IdlErV_aK8a(<)OTOH>0q}+%`r2WuO|bJx*vF?8FP{(&$B%|-FTm%Q z!CCv@f-7)k%hr6kO)+q<0FS-_&uxG=jzeFMHoSbRF0k`(*k=k%SOUlIhH1aR=UY6& zmz&iOF3`Z0v*4z+aPJX#RPrb_@xL(z5tKE3AM}Mg^kd%58nC#&bR{oz1s5a%xwwVgu!JoFmxnbFTj3N;m$W; z;(PF$4RFF9cQYKubS(O95wSpnn#eI}5gX7cN^1LwCXTM_|7TaHpgl-%es6{H7zE z&(HZdN4}lc+QOjja8W#L{|qcL!0s=@E%V@j58#(u;K;Ax@iM5bf){*cd^?s$ z;jIWbV<7Y&2j`~4Hq+rUI}9y^>o>uE`{B-$FtHMT;}gQSGocMU6%Nzm;MLJ^S{n3t z5xzDX2E7j#eGJ=w35!Z$_X@bhs}tYOfL8F!FgUV*rH$nlcA+nQZ!jo?K6Vl4(|sB>gnj60VXGDF;xn`bn|J-Y zB@;driiI`8TH3#VTf`JrvbEuNu^g--I`W##z$_DV@ElI=jgwluJq3Xy;RccRd zFReP7hV)2@N=;RzMns}Wx;Al6{a9-1+l4Y%R? zc*XfG4H@@g#5Y=~-#PM^)!t<~JQh^G<*xVQ$c;0p@=wt#VTPV&N|=~&Xo z-nGtLnnxr52y^+M0-c2g?Ci2Nq-M3Fl79VFCY>6atX|UQu4F}Ms1gqo?KvBTg~r5S zpUH?tw13XuGwmi?8(Sk^t1rK#w3#%Fk;%38p)}-8X7$=xIg=$te3>bK>hXWkI0ks2z)KKDfBpAUzl=fkx+Wj?UJ`%VP61Sj{M;oTIe3k6<(S&hoF-+*2(XYP*=ln#YrdokAS4 zxvMfKQj>novOQw-I+dx$GM;9d%LdA7uB5AzR(-ogmCA6t_&zNp{=k;>uYD!Sc{G)^Lo!k6*sENXy5ya-t z5B+dDGWQ zPjiiwq#=E2!p=S#mWBAbweowR_k`0ii8B+&P7bipjni{)P+n%Tw8dovn2 zg|=MXpB0Gdv?_M_711uzCz}#Ci)dhLU&8w@@j3cU-`FZ?v?Ki}KH`<|0gE^2%C?AR zn(sWhSf|W-ul>|}l~C{XHF_^Gn~KQ9sVGQA#24u&X}eqZW5{@r?&sc?l8KMg*L?XF zaR#r1_zLR6bH!Jy>sTF|lEgPqlr1REMiKteChrdB)yR08emjHyf*RS2>36NjE-tAt zcPafAwmM_@4i-gmsubQXBd(;MHG^}_-?am^xPcYjNE25$ZqI7H8S7$=U%g}OXw?e3 z;x;PA&vZ5x(jP=yaTgV|-6htFyV=Y;W30G`az$R5y=;Dsm@4k4j3lvy6*xc>SFAe3 zG7huJ5jMG(Fy@Mh!`MzEjIo~&5C}(CuYvt^1w!!*RfXcsIU-j)Ptzpv0-s*wQy!7| zURG~itX=^RH}@0d#H+Mg+MbeC=U4hojl+&x4{;#Nd5R{kAePKBZc&ju@i&^@OVAP< zuw~q7f;PvA$If$CjN;o(V5Nr81TjkVqn}BlKTW4m=wm^N?{Pe3+*42zn^Q^pUhwW= zY3E(T(ykg|No>WoxUV`K9jzOVE(w<@BwZ0I=v(@$LM6t;5q_ocbp6l5Q*`}z$C;s# zk>RN+DvhR_IyItaPfdy@vS+W<=!i&d_h@x=uWr%o-neV6d*gdrMkc;Q`%^bGvD`li zO}xm1hNfD>9^sa?PJBA%j87-;8lPk}gVMeHHrKkttQ)v;*lyoFaMh|cs~wMC$mPSi z(O^$(psg8pHp~#BTHQNGy61b)SjOweVwtQun9(m9=B`0lBt7^37sA@dtVs}9*v|(D zYg?<=z<%CWSmQffy>+pA1-HYRDk3^u9i>T$jL`JbMys@u-BO|>x~n5pk($&VYE3lV zJh8D0=mw6CzALuvVs)w++hpPjmh}OftYni_G@%CN_iCTBj9OtXfqE;t;nB>jacfrZ zjT(&umtv|v=CKRN74ukqWMVPf&KfpZ%O?M(3A@#!Xj?b@@s=r~5`Pr_C?MlMIx!dL z)ro;J;Q|e)LHB30qt*9kKi&?67id_Gi|iil&KtF|IuY?KyBul{_9m3`8#W9kQsAMyvTns zY>Muz+1QYOoyA71>F#!O_qPYni)`orhl3}t-#;5XA9trFLHJ@nA0T+js@K4N-dFJC zJ6yeWv3do6EqI2rI@J!I%~@6;n*^~*3!1osXE&Ddw+Byl8M%TdtB*`<$+pvqOtO4`P-(q7n;ZKWl`o*IFdm86MVAOoQ%!eFw z)|JnsaS=`Yaa;_e@-=Ga8bey_N%=KKJQRCSnktJ|%@`-Do^U^gmHJQ|%Ww_OCyrw& zg!}RAr$T&&c))OPd4O<#Ps_+e6`d<}!#&IWlW@<7wZnZb%gSSuSJ`A1 zO9K|lTJHvef zo8t=itQduRRq2nuXQ1?wDVv zbvmoa2OM?YF0)~H>(pkhvo@$h(S6lnOszT;)vCja8g*bJpE~RCL49>7u2~0Ok+A;* DaHL;= literal 0 HcmV?d00001 diff --git a/tests/test_from_config.py b/tests/test_from_config.py index b8444c1..fd21b78 100644 --- a/tests/test_from_config.py +++ b/tests/test_from_config.py @@ -9,7 +9,6 @@ import mllam_data_prep as mdp import tests.data as testdata -from mllam_data_prep.ops.selection import str_to_datetime, str_to_timedelta def test_gen_data(): From 7fc6f37062f660b0f2ce7e560d01fa91c847e7f1 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Fri, 14 Feb 2025 14:30:26 +0100 Subject: [PATCH 25/31] remove pickle --- tests/test_selection.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_selection.py b/tests/test_selection.py index 2765353..1052bf3 100644 --- a/tests/test_selection.py +++ b/tests/test_selection.py @@ -1,5 +1,3 @@ -import pickle as pkl - import isodate import numpy as np import pytest From f5fd87580dd37ca85e544ece40635fddba4dfe5d Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Fri, 14 Feb 2025 14:37:45 +0100 Subject: [PATCH 26/31] remove unused imports --- tests/test_config.py | 2 +- tests/test_from_config.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index d5f2477..d658ac3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -139,7 +139,7 @@ def test_that_range_accepts_datetime(): end = datetime.datetime(1990, 9, 4, 0, 0) step = "PT3H" - range_ = config.Range(start=start, end=end, step=step) + config.Range(start=start, end=end, step=step) def test_config_roundtrip(): diff --git a/tests/test_from_config.py b/tests/test_from_config.py index fd21b78..2e0ea45 100644 --- a/tests/test_from_config.py +++ b/tests/test_from_config.py @@ -1,4 +1,3 @@ -import datetime import shutil import tempfile from pathlib import Path From 218c738bb3d2cbe83df924e50a8adb8fe67bfd4b Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Fri, 14 Feb 2025 14:40:55 +0100 Subject: [PATCH 27/31] allow for none in step --- mllam_data_prep/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllam_data_prep/config.py b/mllam_data_prep/config.py index 60477ef..3bd8189 100644 --- a/mllam_data_prep/config.py +++ b/mllam_data_prep/config.py @@ -75,7 +75,7 @@ class Range: start: Union[str, int, float, datetime] end: Union[str, int, float, datetime] - step: Union[str, int, float, timedelta] = None + step: Union[str, int, float, timedelta, None] = None @dataclass From e9933345a7da4862310391dd7a2aed695899f6b4 Mon Sep 17 00:00:00 2001 From: Jacob Mathias Schreiner Date: Fri, 14 Feb 2025 18:46:11 +0100 Subject: [PATCH 28/31] remove height levels --- height_levels.pkl | Bin 26919 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 height_levels.pkl diff --git a/height_levels.pkl b/height_levels.pkl deleted file mode 100644 index 1211b51846be6b9e2f0654ae8876bc970f75df36..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 26919 zcmdtq3tSXc|3C0Wxe6#M7*)W zUCr_mK51f#p^uv7r8O^^TGFGLrWP8O_fQeZOZ~p*e0Bu{ORcBZ^Z!3RuX*n|b7tnu znVIjI8JuHvpW9j|;s3KW&r)S(shD>dkMrBr+v}U`_E6(|Ajy+_$J!zU_1QJbkVXrzV=X{J_VQEN3)y>^mTZ?c6;o8|G;M~)6gGvbY`6K0BZix)-c-m{OJAkRXrw|)jV|4uWNWNctJH#4nXXMw z(Ng7eZ9&+W(!>^28Zw;Q<}(Z#CWBr_jZkAo9#wB*rom#g+ay%O+iPM?;;o|Enk^ct zdJ|SBG$LUfw0l~HR?aGOC}*O;^Ex6LENl5Whtou<}opo_NW<-TGcW|t*2 zw7K=QCnY5f{+<5ECkYa>J;mm!7AzTQCakKF+K_I{)S65NYV;L^;4qQZu6uvCEKhu*<+FEspiMtp5S5gG6|N{R6VP;&4<>gPBZDI zP)cG+v3VFWHCo!HTddvIoQ|h9!$c>X(xf%2GF5c=XigZNDXDbMuw$EKYeYZw%1J7{ zMaxp`{L!UY*oIm2G*!k78x}KG88;|#K+^C*Lz9NvC(seSeGFAL4@-v5OeddPy4_-H z#M^+5y-ri>U{q&Q$$(+ni59&oGnTd2Q>>`79c;cRheIOtG^NlEHNr_}RG9@jtNHO| zIjV`;FeNo|I=h;D_!+HWMXH;WFIQ)gRFlc5r6b@wkpJKdRxjkO)R~k9x)$i9Z=}rB zsx)-A>$541S`3jcIfnpR0yo>`TlX4jLgm2-Kf`6)6>;Rjc*DQ%x1b2 z8pn?qp%|#sYlml>%-VFeRd$Z^Q(VY2WEd=Tt(n+KW^>2x4rB|OdUx%buGg#5L+PNH zL)nQQW{M2cT0%1nnPwrhTbL?cHN}vjnrxzLDZQ(p)lC%4N<<3hW#3Gt*Bd5lHA-!k zTFXNa?M;oy(Uesy$zUF6uw-ZkWMWo^wE`zR`;OeXX&RK!|Rc4B1npk%EDO^jqHe+jsnqZ1c)E;7MiZs9c zsQR>Ib*$o7v696UuTc?vt+&q}VNj9~E%c=tHp8MdR$!vlHcDZ2QQ}&4v(U+HN}(%W zWwr}#C=^e#3sT`R{zE3T7i7*~of8uqBnCB{X%|9;a5@cZ1R^1-ZWt1JBud=q|5QF& z=;K_+69_DV9I-^`pWq?HCb$W4EM%~@7M>E~S@j3oTJYG%E-mWX(k&FLd1KgZo+h=5 zE?Rb8EROX?M_BcueI}i*LIQ;fA#no53pWazZdrD%%|~y@n5aZl^QL`thJ1f^v1nA8 znmV%th0R~1otUZBnw0#+wp*xfV})m&`=AumbZ+tHqWl+x>CSvs`y4tggc(AvGiz3e zU=!YOCip>FLi1;b2y+suo1Nmuy*B&ezu2h4@;_)Yp-5P*6+T3Rp_PO+wNeV3pQ~ZC zDwFvx?Lzx=|LwBt&USIj{Nq-UCAcMMa7NN;8!AxKP!>)(no{0_oOOTLKsDlt&D~5r znw>WpbruHd>vS1(=Rv{Di_S4AopW|uBOdvb=*pWS8pfm;CTX9dJCY2Io!YwTFfBVJ zCuyaEC0(b{ndz+cF$ok-lUO$_9JP5lp_Vpbt;pqJzsj@mp>QBPP|IU#TeyKh>!lim!>O-6F2#WJsx<#?jt*KBU zHguCngsZmOjW1lIcvV&J!qm(O^))kJ$9=c7=to<=v$37$-5-VN1kDU~K}1rla6Csp zALKenx*bWKXS&MfO;ML_%_;EOn$z=wp7q{py1~*;G|&?R#W01fgB3C@72(Lus$r{ZyTZie0<^W(upPncOV@uSAb~IwzZ3LfLJ+ zA%pJZq=r=KWE~416=LlDoSR#OvHa=g|{WtxP`kH>y|6ff%ss5(d{%uXq z)^z_(pHa8zh1Gw29}(1$2L^T%!GAtTV7N;M$?my?)hbiX3;lJfVSN|6u=@V@_KkY> zw&;)dHnG9&z2!gu?!7Ily|*jwwzoz1zqcRM-CK*zLzzww0CY8-%xogTiOjgr&So z7QLySVqZ4t#3n49it%j1u1Qg(iLhFn#V8WrCoDGiG!}Yop1KS?@ij@en9NGLo1=Fa z?16$k7t#U0<7q*72oG77ne=+b1Mkb&^9p;x!tNdUGl} z5r)=&P3W3s^Hs1XAk!dL6i1T3D#Y!q60e|yNLxepGLBZ_!55^5Tq9o$ugjb(3rnFl zk95bY%+O9&x>hJwP?Om_TuJs8$DO|&6PD4A)3ZDsHFKs#ZDx;hVKy&zbAgv`36dH1 zSho~A-gCLJcVa?hUaYqUPkRzOd(Gv}pXr35)TqPktYyV0+MalkCUmdCc165IzuB{D zV)e7Cd!1)hDj3^<-dfTF1^s8H2T5AHc$uw>ukZ@}vbCvir)g)j>^Xq$`kV1Rk8@_Y zc3BK;K+F2yna!$povpRGbFE6fF2xka_ny|(TB$T;)=& zMP+MUvocsJtVUgKZ;vlV3k-hL^OAcKYs?31jGL@6exnIHFrtKZ%5`Mez2*Zoa9)3D zLv}RdM(WG_d?cmmKR`O{Vd7lp?CDwPgK0=nR z=<=qFbR5{KjvxzeDSGrJtsQ^#9!MrXZ83Hsh zZ9iGCz@$7vT6cOx{75GM@|ODrQvPtKZ!1Y@?B*4cCbWEd($q#|>AcdUK+@sJZ2JgV zaMrz|BWVqMYgaci`H7IX`;c;BQ`!JhIy>=+p`>K<4*_Gz(o@guafeQhDhwB^u+i7Swu>FpIf$^ERAZIwT5&&J9qFVvLLr(o1LU} z?Z>}-MJ69h`0Ow#_x#>|l9YxUC!QxI;~Mm?Buihm`$+<5`JxU#_>cwPZd}uxw7Mna zv>}r_A0O3`l#euobR(rNy4~zeN>Hnw_6VN~+qa=a8jsKkB}SbR-P&UQQNRj(%4h;KM=@i<=(G)J*Mj z)GgCrTiBmOSipB@@4M+Q0u;JTmRBfbdD3;sL)PtxNWE8`aJZO^{>Igmtk2plp5hP zsc`-?@Zc~QFbEEL66SV?J32$jW3X>9%xVnRxx@17efj=$xderu;nL%<^Z;zV2aa(- z`vzFD3i`bV-h6+axeDi>g9lH*fWvUeS1@-6-0=yNtbu)( z!>mPc-5gk+2fJ9IFbOWz!_pMkdMq3>6xs*Cl0MKc0>;Z=&Lhwf2yZrm(QeRutry>~ z;tTNnk5Kj<)Re%2UGUgu82oQIVg<~97w(%2y=^e=C1}lpn=@dQ21bpC#;4(`r{L*+ zuw4{P?gSS~;o%lA&>t!qLffy=e1CTT0^NUt@?&u7f8fU5uwp9=Uk~*wVc`;3_7;@B z4xh<^^QXXrMi?*=4w(RRhr=C%phOP)_JCPk;JU|Pc`)qK1PUH->5ZOze@ZXG)~Df^ z@1gx5EZGD7w!!$1U``Qqya#W-4Wsj+c_u8L3eRUknGR}{uwW!SmH>nM!x7OiKMd|` z2fbUtxTetR1vlU7!S|=?3XD1njiqqaA$WQpY_}aIe+(CX2oEoVfeWExHnio!-7i3Q z3zVn9sVcZ}G^|L3;c-yk8y1GcvJOz%20qgq&i8=_fA7xsC!i7zIS+GB!X1a9WIyb? z6J~9K>(;>X<*>_QDA?iBS7E6Yww?^fq(i$JmW+jd3K%~S=JbV*2zXNlqaTIlAXw}R z&$~fcRTSSZ%|%#H29JFQgTIC&K8N{R;J$xD?+;*H0kqD8n{BY_B^Z?rjRv?%3r~-S z?S{eRc)0KhcsL3Mc7}?!(Do4A?GN4Eq5OIz-=C@FaN|#~;us7+0QFzM!cSq@dMI55 zpZOP@KOY`^9R|#RL!O7Z&%qrNq2yWEcLdBD4A;qFc@Nkn6bg^Sr4Pf>Ca|?99CI^* z?~na5EIAGRet_}cz?{9%u?^n*2u81l=B2Rs9e6$;%3gt*X|TWqkLh4=G8{1q<`04U z`a|ztFs>`KwuhTr!K$V(${QMg>&EwIRRuhK7PdP9lfQ)vzl4Xk!@y6Vq8QrVhr1U- z_c>6W2dBOWH(Fpt8VpZ?`Z2ID36{k{X&?AZH#ol|JlF;X1i~SWV6Fu2s0`=(CAk3m z{s^;C`F>E~-j+q7R(_zVE=$8TGH85u!bSU7>fiSusG)KbX z5P1GkC~E;Vez2fH_%_ANizyo~b>)VC4-5A|=|^z>doUm$=1zqY9n2aD%VVGr21{GQ zFLh$H8lFb(^o_Jc9KuR!(Tzg zCb)Y!l-uD(D-2JEg=3*~AeNUT43?KbVGE>Zem;N9JXrD)j5k2X zco-cIi=&{dEiCYd!Ph(U_2mBqy$?X^r?6@zG|q>ob71mw@bCnv7z}s!fbz%T#wIZQ zW+%R#!qZUt4V=FX2CRm;??A~bFv|qXl~5Q0OQYeK_OPTWjQ=f!Z^v;KMt=*7w?kPm zELaGG^I*OOdZ$2Z60GVCjUD0X<}g_T51*Iu?J17H-8-RtE!?;mhR=e9lc7`t=PO`9 zKbRW=B`si916Y2sBj1j26qbGt$E<@T1u*_K=*WiAsjzq$lsyRxI>X>#nC}j~FLmJC zu^xw2UqIsqc=}&3`3-pZd8iQJ?h#NP12=}k@RqR96H2eN=i8b80}R*)b2mcCGMM!y zET0Ai5tfdEV`5=RR~X+KI=o@@)pmS4#V4TbD_HOe3|-Pj`dK zkHEu?pyJx&d^@{;gz^%&aWf2G0So6s=}T~a1`HSvbDx5eD3~RM<$h53^)bGk(o=BE ze_+X07{3xa-h$CNu-FJ?6JWt07~BKqKL)*)3%in>eCRj2AI@-hH-&*kU1>eH_Vrb2S#uS*`2Py)g{6Y|4 zE_@f1z6%4gpyX*-9s)}nz>=eZd^yKDSo|6+NQL=NLTfNIUTV(EC+~rZ_n>?#3?B)l zVKBf8N=lpY<;pk0(l=p=2pzGoxHT-e+LV{i{|Z`{L!%WYkA;c|DEEcoWdVFS=@u9; z4@wNM{3%!}4cI1s_>);nt~B9JUj|j9VT=!KyQwjs@0JcvJOa0s_+$B(;Pfb{`pJ*y z$IORq2Sc};zC8a#5!|MPOPa#z+Z*xuDhrJ12-_a<;rVW};EBF)+eL4lzoY<89|l$K zFlK`npWjx1ZY|-7y`DUO+cdbOE1Z79gXgQ}z?cEB?KO9v@3sP-7zelc!6jQ8^7+$^ zQ1uv$IoN>bx19;yqTz`%Zajb60=Og*PXArP^HpnLj2gBLl5Dd^m>+8R`tJ_r_DjEU zUmgi#KfcA&4cox&)|;4Lc7q!m4;!w!&ePkQz?V(GV*a;P+=hMN_QltD`sD^NHnkG- z_gv*}4}~wkS;5n>SFd3HSh#)jWuAUn3S)CFVgBiI?)HRoheUfi%v1RbcV>r+NMAgU z^!js1Pdkfr+!>_(P9uHbC!`mgLOP`k=}tc)edz?!AD22LdrBtwx%~k1k2(0?a@ik7 z=wF?<{!UDP%)i@TMT+kKuOj>(m(X8Dgk}Ha7u4@xdRlN!;`;L?@HSl zcOL%dFdV!SezpeoUJTd13Oh}P|5C%23OKtjY!m`t3WC?&sQ13y;ZWHz>W#^R>*0>q z;kt=%sT|rLhB=q17pC&&y->3hj+h4HM!~4Au$?yyJVCuNE$99Tte6eU(%`{3xT6hR zS4q7vm0!9a+6!UMbZ8z2HIZL-i78YsCgQW=mg^$!l+}^8`E;_*2BQpq5DKw@gyt@h6gWEFHGfk z?1AgvgG;AE`$(7*2F+ejQ%b!tEjQw07&jY6rNMS_Ft81DucTg>%2#{^%a+4~R=8s< zTo(bC`a*jd^~SVZ&K78%2Q>ybA|A%Ig;Ce37pC&<4#2=w(0vB1m;lRqz=KWTj?>gD z({k&!!KLp&y9wqDf#&wqYoAJ7^~Fu4^3TkKUqr(dXQ`K_{Je!wmjnk% zV8~kPrD?v82A&CmUwlrzH07_zhIyT#?l|?*lt1VV7!m`0u23&c`Dd2GFGj%?-Z1YI z>aA(MP7en?LcR8$1!IO6{O({jcZUZi{>IaD_TJ(S?+TmGxyjQ9DsN!^I5>Rkb)Ihi z7(6iJSIj?C#T}jqo3FXX(+2|KoUBUBKX#SdTn-PsSHaVByx{P(E119kGWS3PoHOqd zPY=Idj``10uRXWtn)hF(UYqQYK)p6gH@krJchqaM^b+c|*?b-K+AJMTy*5i%QLoL? zpHi>Q(yvgj&C*HKYqNBVlSm(@Ub~YaC2=tI+RP)zpzs~M_AQ+G4GcN}mzFprBfj79 zPU=3G^Ev$2XAVi>=DhCHH^D7y91?b+mA&hbSi5VMubL{C6wf?wZ?jx38528c&#?8d z#iw$~s~Mb36h|5iD7Rpce^ z`Tekq{xar!U%~vP@VvZ&r@I`7`s}NiA5_UbzXo^z zo?dkgo}Y6A^TTg)_50u|;TBJycmECZmq2~L-+6k~k>BNxmTQMs_LjsjTjaqW-@;Mt z-FW^a6P&jVZfMeg=O3B?D_6o6*I|$NhJ600d2mu0oEPEF^EX)Gp|4@GtJ&>1A z>kOYC24`i%1qE>B=Wx?SxHl+>FL$&rJg0^?UWLAEVXGssv!n$t-=`x?NP^=nFl`}x zemk6X7A|P|5MOR(G~A?wduPIi*J?2A{2c5vA0~VX$NvP={2%7! zpO1pG#=`|K!IdlErV_aK8a(<)OTOH>0q}+%`r2WuO|bJx*vF?8FP{(&$B%|-FTm%Q z!CCv@f-7)k%hr6kO)+q<0FS-_&uxG=jzeFMHoSbRF0k`(*k=k%SOUlIhH1aR=UY6& zmz&iOF3`Z0v*4z+aPJX#RPrb_@xL(z5tKE3AM}Mg^kd%58nC#&bR{oz1s5a%xwwVgu!JoFmxnbFTj3N;m$W; z;(PF$4RFF9cQYKubS(O95wSpnn#eI}5gX7cN^1LwCXTM_|7TaHpgl-%es6{H7zE z&(HZdN4}lc+QOjja8W#L{|qcL!0s=@E%V@j58#(u;K;Ax@iM5bf){*cd^?s$ z;jIWbV<7Y&2j`~4Hq+rUI}9y^>o>uE`{B-$FtHMT;}gQSGocMU6%Nzm;MLJ^S{n3t z5xzDX2E7j#eGJ=w35!Z$_X@bhs}tYOfL8F!FgUV*rH$nlcA+nQZ!jo?K6Vl4(|sB>gnj60VXGDF;xn`bn|J-Y zB@;driiI`8TH3#VTf`JrvbEuNu^g--I`W##z$_DV@ElI=jgwluJq3Xy;RccRd zFReP7hV)2@N=;RzMns}Wx;Al6{a9-1+l4Y%R? zc*XfG4H@@g#5Y=~-#PM^)!t<~JQh^G<*xVQ$c;0p@=wt#VTPV&N|=~&Xo z-nGtLnnxr52y^+M0-c2g?Ci2Nq-M3Fl79VFCY>6atX|UQu4F}Ms1gqo?KvBTg~r5S zpUH?tw13XuGwmi?8(Sk^t1rK#w3#%Fk;%38p)}-8X7$=xIg=$te3>bK>hXWkI0ks2z)KKDfBpAUzl=fkx+Wj?UJ`%VP61Sj{M;oTIe3k6<(S&hoF-+*2(XYP*=ln#YrdokAS4 zxvMfKQj>novOQw-I+dx$GM;9d%LdA7uB5AzR(-ogmCA6t_&zNp{=k;>uYD!Sc{G)^Lo!k6*sENXy5ya-t z5B+dDGWQ zPjiiwq#=E2!p=S#mWBAbweowR_k`0ii8B+&P7bipjni{)P+n%Tw8dovn2 zg|=MXpB0Gdv?_M_711uzCz}#Ci)dhLU&8w@@j3cU-`FZ?v?Ki}KH`<|0gE^2%C?AR zn(sWhSf|W-ul>|}l~C{XHF_^Gn~KQ9sVGQA#24u&X}eqZW5{@r?&sc?l8KMg*L?XF zaR#r1_zLR6bH!Jy>sTF|lEgPqlr1REMiKteChrdB)yR08emjHyf*RS2>36NjE-tAt zcPafAwmM_@4i-gmsubQXBd(;MHG^}_-?am^xPcYjNE25$ZqI7H8S7$=U%g}OXw?e3 z;x;PA&vZ5x(jP=yaTgV|-6htFyV=Y;W30G`az$R5y=;Dsm@4k4j3lvy6*xc>SFAe3 zG7huJ5jMG(Fy@Mh!`MzEjIo~&5C}(CuYvt^1w!!*RfXcsIU-j)Ptzpv0-s*wQy!7| zURG~itX=^RH}@0d#H+Mg+MbeC=U4hojl+&x4{;#Nd5R{kAePKBZc&ju@i&^@OVAP< zuw~q7f;PvA$If$CjN;o(V5Nr81TjkVqn}BlKTW4m=wm^N?{Pe3+*42zn^Q^pUhwW= zY3E(T(ykg|No>WoxUV`K9jzOVE(w<@BwZ0I=v(@$LM6t;5q_ocbp6l5Q*`}z$C;s# zk>RN+DvhR_IyItaPfdy@vS+W<=!i&d_h@x=uWr%o-neV6d*gdrMkc;Q`%^bGvD`li zO}xm1hNfD>9^sa?PJBA%j87-;8lPk}gVMeHHrKkttQ)v;*lyoFaMh|cs~wMC$mPSi z(O^$(psg8pHp~#BTHQNGy61b)SjOweVwtQun9(m9=B`0lBt7^37sA@dtVs}9*v|(D zYg?<=z<%CWSmQffy>+pA1-HYRDk3^u9i>T$jL`JbMys@u-BO|>x~n5pk($&VYE3lV zJh8D0=mw6CzALuvVs)w++hpPjmh}OftYni_G@%CN_iCTBj9OtXfqE;t;nB>jacfrZ zjT(&umtv|v=CKRN74ukqWMVPf&KfpZ%O?M(3A@#!Xj?b@@s=r~5`Pr_C?MlMIx!dL z)ro;J;Q|e)LHB30qt*9kKi&?67id_Gi|iil&KtF|IuY?KyBul{_9m3`8#W9kQsAMyvTns zY>Muz+1QYOoyA71>F#!O_qPYni)`orhl3}t-#;5XA9trFLHJ@nA0T+js@K4N-dFJC zJ6yeWv3do6EqI2rI@J!I%~@6;n*^~*3!1osXE&Ddw+Byl8M%TdtB*`<$+pvqOtO4`P-(q7n;ZKWl`o*IFdm86MVAOoQ%!eFw z)|JnsaS=`Yaa;_e@-=Ga8bey_N%=KKJQRCSnktJ|%@`-Do^U^gmHJQ|%Ww_OCyrw& zg!}RAr$T&&c))OPd4O<#Ps_+e6`d<}!#&IWlW@<7wZnZb%gSSuSJ`A1 zO9K|lTJHvef zo8t=itQduRRq2nuXQ1?wDVv zbvmoa2OM?YF0)~H>(pkhvo@$h(S6lnOszT;)vCja8g*bJpE~RCL49>7u2~0Ok+A;* DaHL;= From 779530cf1b7599aa8ab77cf4bf76742d46da9c4a Mon Sep 17 00:00:00 2001 From: matschreiner Date: Sun, 16 Feb 2025 11:44:24 +0100 Subject: [PATCH 29/31] improve test name Co-authored-by: Hauke Schulz <43613877+observingClouds@users.noreply.github.com> --- tests/test_selection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_selection.py b/tests/test_selection.py index 1052bf3..44795b0 100644 --- a/tests/test_selection.py +++ b/tests/test_selection.py @@ -63,7 +63,7 @@ def test_can_slice_time(ds): @pytest.mark.parametrize("step", ["PT6H", "PT3H"]) -def test_if_step_time_matches_output(ds, step): +def test_timestep_matches_output(ds, step): start = "1990-09-01T00:00" end = "1990-09-09T00:00" coord_ranges = { From 4d52059620cb0e882906266127dd824eb14a1b8b Mon Sep 17 00:00:00 2001 From: matschreiner Date: Sun, 16 Feb 2025 11:44:37 +0100 Subject: [PATCH 30/31] improve testname Co-authored-by: Hauke Schulz <43613877+observingClouds@users.noreply.github.com> --- tests/test_selection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_selection.py b/tests/test_selection.py index 44795b0..f287617 100644 --- a/tests/test_selection.py +++ b/tests/test_selection.py @@ -52,7 +52,7 @@ def test_error_on_empty_range(ds, x_start, x_end): ds = mdp.ops.selection.select_by_kwargs(ds, **coord_ranges) -def test_can_slice_time(ds): +def test_slice_time(ds): start = "1990-09-01T00:00" end = "1990-09-09T00:00" coord_ranges = { From 3542f8bcc24dad58375a83ec62f5c39ef945a094 Mon Sep 17 00:00:00 2001 From: matschreiner Date: Sun, 16 Feb 2025 11:44:51 +0100 Subject: [PATCH 31/31] improve testname Co-authored-by: Hauke Schulz <43613877+observingClouds@users.noreply.github.com> --- tests/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index d658ac3..633e1a1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -134,7 +134,7 @@ def test_get_config_nested(): input_config.foobarfield -def test_that_range_accepts_datetime(): +def test_range_accepts_datetime(): start = datetime.datetime(1990, 9, 3, 0, 0) end = datetime.datetime(1990, 9, 4, 0, 0) step = "PT3H"