From 89ea32cc99175b6b663e8d2bbcaa470ff1cefeef Mon Sep 17 00:00:00 2001 From: Jose Osorio Date: Wed, 25 Jun 2025 13:51:55 -0500 Subject: [PATCH 1/2] EPD-2072: update dataset parser --- autoblocks/_impl/datasets/models/dataset.py | 49 +++++ autoblocks/_impl/datasets/models/schema.py | 2 + tests/autoblocks/test_app_client_datasets.py | 203 +++++++++++++++++++ 3 files changed, 254 insertions(+) create mode 100644 tests/autoblocks/test_app_client_datasets.py diff --git a/autoblocks/_impl/datasets/models/dataset.py b/autoblocks/_impl/datasets/models/dataset.py index cfcfca5a..ccb7b807 100644 --- a/autoblocks/_impl/datasets/models/dataset.py +++ b/autoblocks/_impl/datasets/models/dataset.py @@ -13,6 +13,7 @@ from pydantic import model_validator from autoblocks._impl.datasets.models.schema import SchemaProperty +from autoblocks._impl.datasets.models.schema import create_schema_property from autoblocks._impl.util import cuid_generator # Type alias for schema property lists @@ -35,6 +36,30 @@ class Dataset(BaseModel): extra="allow", ) + @field_validator("schema_properties", mode="before") + @classmethod + def validate_schema_properties(cls, v: Optional[List[Dict[str, Any]]]) -> Optional[List[SchemaProperty]]: + """Validate and convert schema properties using factory function""" + if v is None: + return None + + if not isinstance(v, list): + return v + + result = [] + for item in v: + if isinstance(item, dict): + try: + result.append(create_schema_property(item)) + except (ValueError, TypeError): + # If we can't parse a specific property, skip it rather than failing entirely + # This provides better resilience against API changes + continue + else: + result.append(item) + + return result + class DatasetSchema(BaseModel): """Dataset schema V2""" @@ -51,6 +76,30 @@ class DatasetSchema(BaseModel): extra="allow", ) + @field_validator("schema_properties", mode="before") + @classmethod + def validate_schema_properties(cls, v: Optional[List[Dict[str, Any]]]) -> Optional[List[SchemaProperty]]: + """Validate and convert schema properties using factory function""" + if v is None: + return None + + if not isinstance(v, list): + return v + + result = [] + for item in v: + if isinstance(item, dict): + try: + result.append(create_schema_property(item)) + except (ValueError, TypeError): + # If we can't parse a specific property, skip it rather than failing entirely + # This provides better resilience against API changes + continue + else: + result.append(item) + + return result + class DatasetItem(BaseModel): """Dataset item V2""" diff --git a/autoblocks/_impl/datasets/models/schema.py b/autoblocks/_impl/datasets/models/schema.py index 5ba56561..6f5ea8b9 100644 --- a/autoblocks/_impl/datasets/models/schema.py +++ b/autoblocks/_impl/datasets/models/schema.py @@ -11,6 +11,7 @@ from pydantic import BaseModel from pydantic import ConfigDict +from pydantic import Field from pydantic import field_validator @@ -33,6 +34,7 @@ class BaseSchemaProperty(BaseModel): id: str name: str required: bool + default_value: Optional[Any] = Field(default=None, alias="defaultValue") model_config = ConfigDict( populate_by_name=True, diff --git a/tests/autoblocks/test_app_client_datasets.py b/tests/autoblocks/test_app_client_datasets.py new file mode 100644 index 00000000..e83f1bc0 --- /dev/null +++ b/tests/autoblocks/test_app_client_datasets.py @@ -0,0 +1,203 @@ +"""Tests for AutoblocksAppClient dataset deserialization with defaultValue fields.""" + +from autoblocks._impl.config.constants import API_ENDPOINT_V2 +from autoblocks.api.app_client import AutoblocksAppClient + + +def test_dataset_deserialization_with_default_value(httpx_mock): + """Test that datasets with defaultValue field in schema properties can be deserialized correctly.""" + # This test reproduces the exact error scenario from the user's issue + httpx_mock.add_response( + url=f"{API_ENDPOINT_V2}/apps/test-app/datasets", + method="GET", + status_code=200, + json=[ + { + "id": "dataset-id-1", + "externalId": "test-dataset", + "name": "Test Dataset", + "createdAt": "2023-01-01T00:00:00Z", + "latestRevisionId": "revision-1", + "schemaVersion": 1, + "schema": [ + { + "id": "string-prop-id", + "name": "String Property", + "type": "String", + "required": True, + }, + { + "id": "s23p00ugn0gb5zhmg123", + "name": "Number Property", + "type": "Number", + "required": False, + "defaultValue": 30, # This was causing the validation error + }, + { + "id": "select-prop-id", + "name": "Select Property", + "type": "Select", + "required": False, + "options": ["option1", "option2"], + "defaultValue": "option1", + }, + ], + } + ], + match_headers={"Authorization": "Bearer mock-api-key"}, + ) + + client = AutoblocksAppClient(app_slug="test-app", api_key="mock-api-key") + datasets = client.datasets.list() + + # Verify the dataset was deserialized successfully + assert len(datasets) == 1 + dataset = datasets[0] + + assert dataset.id == "dataset-id-1" + assert dataset.external_id == "test-dataset" + assert dataset.name == "Test Dataset" + assert dataset.schema_version == 1 + + # Verify schema properties were parsed correctly + assert dataset.schema_properties is not None + assert len(dataset.schema_properties) == 3 + + # Check string property + string_prop = dataset.schema_properties[0] + assert string_prop.id == "string-prop-id" + assert string_prop.name == "String Property" + assert string_prop.type.value == "String" + assert string_prop.required is True + assert string_prop.default_value is None + + # Check number property with defaultValue + number_prop = dataset.schema_properties[1] + assert number_prop.id == "s23p00ugn0gb5zhmg123" + assert number_prop.name == "Number Property" + assert number_prop.type.value == "Number" + assert number_prop.required is False + assert number_prop.default_value == 30 + + # Check select property with defaultValue + select_prop = dataset.schema_properties[2] + assert select_prop.id == "select-prop-id" + assert select_prop.name == "Select Property" + assert select_prop.type.value == "Select" + assert select_prop.required is False + assert select_prop.default_value == "option1" + # Type check to ensure we have a SelectProperty before accessing options + from autoblocks._impl.datasets.models.schema import SelectProperty + + assert isinstance(select_prop, SelectProperty) + assert select_prop.options == ["option1", "option2"] + + +def test_dataset_deserialization_with_invalid_schema_property(): + """Test that datasets with invalid schema properties are handled gracefully.""" + from autoblocks._impl.api.utils.serialization import deserialize_model + from autoblocks._impl.datasets.models.dataset import Dataset + + # Mock data with an invalid schema property that should be skipped + dataset_data = { + "id": "dataset-id-1", + "externalId": "test-dataset", + "name": "Test Dataset", + "schema": [ + { + "id": "valid-prop-id", + "name": "Valid Property", + "type": "String", + "required": True, + }, + { + "id": "invalid-prop-id", + "name": "Invalid Property", + "type": "UnknownType", # This should cause the property to be skipped + "required": False, + }, + { + "id": "another-valid-prop-id", + "name": "Another Valid Property", + "type": "Number", + "required": False, + "defaultValue": 42, + }, + ], + } + + # This should not raise an exception + dataset = deserialize_model(Dataset, dataset_data) + + # Verify the dataset was created successfully + assert dataset.id == "dataset-id-1" + assert dataset.external_id == "test-dataset" + assert dataset.name == "Test Dataset" + + # Should have only 2 valid properties (invalid one skipped) + assert dataset.schema_properties is not None + assert len(dataset.schema_properties) == 2 + + # Check that valid properties are present + prop_names = [prop.name for prop in dataset.schema_properties] + assert "Valid Property" in prop_names + assert "Another Valid Property" in prop_names + assert "Invalid Property" not in prop_names + + +def test_dataset_schema_property_factory_function(): + """Test the schema property factory function directly.""" + from autoblocks._impl.datasets.models.schema import SchemaPropertyType + from autoblocks._impl.datasets.models.schema import SelectProperty + from autoblocks._impl.datasets.models.schema import create_schema_property + + # Test creating a number property with defaultValue + number_prop_data = { + "id": "test-number-prop", + "name": "Test Number", + "type": "Number", + "required": False, + "defaultValue": 42, + } + + number_prop = create_schema_property(number_prop_data) + assert number_prop.id == "test-number-prop" + assert number_prop.name == "Test Number" + assert number_prop.type == SchemaPropertyType.NUMBER + assert number_prop.required is False + assert number_prop.default_value == 42 + + # Test creating a select property with defaultValue + select_prop_data = { + "id": "test-select-prop", + "name": "Test Select", + "type": "Select", + "required": True, + "options": ["option1", "option2", "option3"], + "defaultValue": "option2", + } + + select_prop = create_schema_property(select_prop_data) + assert select_prop.id == "test-select-prop" + assert select_prop.name == "Test Select" + assert select_prop.type == SchemaPropertyType.SELECT + assert select_prop.required is True + assert select_prop.default_value == "option2" + # Type check before accessing options + assert isinstance(select_prop, SelectProperty) + assert select_prop.options == ["option1", "option2", "option3"] + + # Test creating a string property without defaultValue + string_prop_data = { + "id": "test-string-prop", + "name": "Test String", + "type": "String", + "required": True, + } + + string_prop = create_schema_property(string_prop_data) + assert string_prop.id == "test-string-prop" + assert string_prop.name == "Test String" + assert string_prop.type == SchemaPropertyType.STRING + assert string_prop.required is True + assert string_prop.default_value is None From 87eb86c74ec8f9c25b2d0eb561356b78700152b0 Mon Sep 17 00:00:00 2001 From: Jose Osorio Date: Wed, 25 Jun 2025 13:58:12 -0500 Subject: [PATCH 2/2] Feedback --- autoblocks/_impl/datasets/models/dataset.py | 75 ++++++++++---------- tests/autoblocks/test_app_client_datasets.py | 9 +-- 2 files changed, 40 insertions(+), 44 deletions(-) diff --git a/autoblocks/_impl/datasets/models/dataset.py b/autoblocks/_impl/datasets/models/dataset.py index ccb7b807..5e791fbf 100644 --- a/autoblocks/_impl/datasets/models/dataset.py +++ b/autoblocks/_impl/datasets/models/dataset.py @@ -20,6 +20,41 @@ SchemaPropertyList = List[Union[SchemaProperty, Dict[str, Any]]] +def _validate_schema_properties(v: Optional[List[Dict[str, Any]]]) -> Optional[List[SchemaProperty]]: + """ + Shared validator function for schema properties. + + Validates and converts schema properties using factory function. + If a property can't be parsed, it's skipped rather than failing entirely + to provide better resilience against API changes. + + Args: + v: Raw schema properties data from API + + Returns: + List of validated SchemaProperty instances or None + """ + if v is None: + return None + + if not isinstance(v, list): + return v + + result = [] + for item in v: + if isinstance(item, dict): + try: + result.append(create_schema_property(item)) + except (ValueError, TypeError): + # If we can't parse a specific property, skip it rather than failing entirely + # This provides better resilience against API changes + continue + else: + result.append(item) + + return result + + class Dataset(BaseModel): """Dataset V2""" @@ -40,25 +75,7 @@ class Dataset(BaseModel): @classmethod def validate_schema_properties(cls, v: Optional[List[Dict[str, Any]]]) -> Optional[List[SchemaProperty]]: """Validate and convert schema properties using factory function""" - if v is None: - return None - - if not isinstance(v, list): - return v - - result = [] - for item in v: - if isinstance(item, dict): - try: - result.append(create_schema_property(item)) - except (ValueError, TypeError): - # If we can't parse a specific property, skip it rather than failing entirely - # This provides better resilience against API changes - continue - else: - result.append(item) - - return result + return _validate_schema_properties(v) class DatasetSchema(BaseModel): @@ -80,25 +97,7 @@ class DatasetSchema(BaseModel): @classmethod def validate_schema_properties(cls, v: Optional[List[Dict[str, Any]]]) -> Optional[List[SchemaProperty]]: """Validate and convert schema properties using factory function""" - if v is None: - return None - - if not isinstance(v, list): - return v - - result = [] - for item in v: - if isinstance(item, dict): - try: - result.append(create_schema_property(item)) - except (ValueError, TypeError): - # If we can't parse a specific property, skip it rather than failing entirely - # This provides better resilience against API changes - continue - else: - result.append(item) - - return result + return _validate_schema_properties(v) class DatasetItem(BaseModel): diff --git a/tests/autoblocks/test_app_client_datasets.py b/tests/autoblocks/test_app_client_datasets.py index e83f1bc0..2fe60930 100644 --- a/tests/autoblocks/test_app_client_datasets.py +++ b/tests/autoblocks/test_app_client_datasets.py @@ -1,6 +1,9 @@ """Tests for AutoblocksAppClient dataset deserialization with defaultValue fields.""" from autoblocks._impl.config.constants import API_ENDPOINT_V2 +from autoblocks._impl.datasets.models.schema import SchemaPropertyType +from autoblocks._impl.datasets.models.schema import SelectProperty +from autoblocks._impl.datasets.models.schema import create_schema_property from autoblocks.api.app_client import AutoblocksAppClient @@ -86,8 +89,6 @@ def test_dataset_deserialization_with_default_value(httpx_mock): assert select_prop.type.value == "Select" assert select_prop.required is False assert select_prop.default_value == "option1" - # Type check to ensure we have a SelectProperty before accessing options - from autoblocks._impl.datasets.models.schema import SelectProperty assert isinstance(select_prop, SelectProperty) assert select_prop.options == ["option1", "option2"] @@ -147,10 +148,6 @@ def test_dataset_deserialization_with_invalid_schema_property(): def test_dataset_schema_property_factory_function(): """Test the schema property factory function directly.""" - from autoblocks._impl.datasets.models.schema import SchemaPropertyType - from autoblocks._impl.datasets.models.schema import SelectProperty - from autoblocks._impl.datasets.models.schema import create_schema_property - # Test creating a number property with defaultValue number_prop_data = { "id": "test-number-prop",