diff --git a/autoblocks/_impl/datasets/models/dataset.py b/autoblocks/_impl/datasets/models/dataset.py index cfcfca5a..5e791fbf 100644 --- a/autoblocks/_impl/datasets/models/dataset.py +++ b/autoblocks/_impl/datasets/models/dataset.py @@ -13,12 +13,48 @@ from pydantic import model_validator from autoblocks._impl.datasets.models.schema import SchemaProperty +from autoblocks._impl.datasets.models.schema import create_schema_property from autoblocks._impl.util import cuid_generator # Type alias for schema property lists SchemaPropertyList = List[Union[SchemaProperty, Dict[str, Any]]] +def _validate_schema_properties(v: Optional[List[Dict[str, Any]]]) -> Optional[List[SchemaProperty]]: + """ + Shared validator function for schema properties. + + Validates and converts schema properties using factory function. + If a property can't be parsed, it's skipped rather than failing entirely + to provide better resilience against API changes. + + Args: + v: Raw schema properties data from API + + Returns: + List of validated SchemaProperty instances or None + """ + if v is None: + return None + + if not isinstance(v, list): + return v + + result = [] + for item in v: + if isinstance(item, dict): + try: + result.append(create_schema_property(item)) + except (ValueError, TypeError): + # If we can't parse a specific property, skip it rather than failing entirely + # This provides better resilience against API changes + continue + else: + result.append(item) + + return result + + class Dataset(BaseModel): """Dataset V2""" @@ -35,6 +71,12 @@ class Dataset(BaseModel): extra="allow", ) + @field_validator("schema_properties", mode="before") + @classmethod + def validate_schema_properties(cls, v: Optional[List[Dict[str, Any]]]) -> Optional[List[SchemaProperty]]: + """Validate and convert schema properties using factory function""" + return _validate_schema_properties(v) + class DatasetSchema(BaseModel): """Dataset schema V2""" @@ -51,6 +93,12 @@ class DatasetSchema(BaseModel): extra="allow", ) + @field_validator("schema_properties", mode="before") + @classmethod + def validate_schema_properties(cls, v: Optional[List[Dict[str, Any]]]) -> Optional[List[SchemaProperty]]: + """Validate and convert schema properties using factory function""" + return _validate_schema_properties(v) + class DatasetItem(BaseModel): """Dataset item V2""" diff --git a/autoblocks/_impl/datasets/models/schema.py b/autoblocks/_impl/datasets/models/schema.py index 5ba56561..6f5ea8b9 100644 --- a/autoblocks/_impl/datasets/models/schema.py +++ b/autoblocks/_impl/datasets/models/schema.py @@ -11,6 +11,7 @@ from pydantic import BaseModel from pydantic import ConfigDict +from pydantic import Field from pydantic import field_validator @@ -33,6 +34,7 @@ class BaseSchemaProperty(BaseModel): id: str name: str required: bool + default_value: Optional[Any] = Field(default=None, alias="defaultValue") model_config = ConfigDict( populate_by_name=True, diff --git a/tests/autoblocks/test_app_client_datasets.py b/tests/autoblocks/test_app_client_datasets.py new file mode 100644 index 00000000..2fe60930 --- /dev/null +++ b/tests/autoblocks/test_app_client_datasets.py @@ -0,0 +1,200 @@ +"""Tests for AutoblocksAppClient dataset deserialization with defaultValue fields.""" + +from autoblocks._impl.config.constants import API_ENDPOINT_V2 +from autoblocks._impl.datasets.models.schema import SchemaPropertyType +from autoblocks._impl.datasets.models.schema import SelectProperty +from autoblocks._impl.datasets.models.schema import create_schema_property +from autoblocks.api.app_client import AutoblocksAppClient + + +def test_dataset_deserialization_with_default_value(httpx_mock): + """Test that datasets with defaultValue field in schema properties can be deserialized correctly.""" + # This test reproduces the exact error scenario from the user's issue + httpx_mock.add_response( + url=f"{API_ENDPOINT_V2}/apps/test-app/datasets", + method="GET", + status_code=200, + json=[ + { + "id": "dataset-id-1", + "externalId": "test-dataset", + "name": "Test Dataset", + "createdAt": "2023-01-01T00:00:00Z", + "latestRevisionId": "revision-1", + "schemaVersion": 1, + "schema": [ + { + "id": "string-prop-id", + "name": "String Property", + "type": "String", + "required": True, + }, + { + "id": "s23p00ugn0gb5zhmg123", + "name": "Number Property", + "type": "Number", + "required": False, + "defaultValue": 30, # This was causing the validation error + }, + { + "id": "select-prop-id", + "name": "Select Property", + "type": "Select", + "required": False, + "options": ["option1", "option2"], + "defaultValue": "option1", + }, + ], + } + ], + match_headers={"Authorization": "Bearer mock-api-key"}, + ) + + client = AutoblocksAppClient(app_slug="test-app", api_key="mock-api-key") + datasets = client.datasets.list() + + # Verify the dataset was deserialized successfully + assert len(datasets) == 1 + dataset = datasets[0] + + assert dataset.id == "dataset-id-1" + assert dataset.external_id == "test-dataset" + assert dataset.name == "Test Dataset" + assert dataset.schema_version == 1 + + # Verify schema properties were parsed correctly + assert dataset.schema_properties is not None + assert len(dataset.schema_properties) == 3 + + # Check string property + string_prop = dataset.schema_properties[0] + assert string_prop.id == "string-prop-id" + assert string_prop.name == "String Property" + assert string_prop.type.value == "String" + assert string_prop.required is True + assert string_prop.default_value is None + + # Check number property with defaultValue + number_prop = dataset.schema_properties[1] + assert number_prop.id == "s23p00ugn0gb5zhmg123" + assert number_prop.name == "Number Property" + assert number_prop.type.value == "Number" + assert number_prop.required is False + assert number_prop.default_value == 30 + + # Check select property with defaultValue + select_prop = dataset.schema_properties[2] + assert select_prop.id == "select-prop-id" + assert select_prop.name == "Select Property" + assert select_prop.type.value == "Select" + assert select_prop.required is False + assert select_prop.default_value == "option1" + + assert isinstance(select_prop, SelectProperty) + assert select_prop.options == ["option1", "option2"] + + +def test_dataset_deserialization_with_invalid_schema_property(): + """Test that datasets with invalid schema properties are handled gracefully.""" + from autoblocks._impl.api.utils.serialization import deserialize_model + from autoblocks._impl.datasets.models.dataset import Dataset + + # Mock data with an invalid schema property that should be skipped + dataset_data = { + "id": "dataset-id-1", + "externalId": "test-dataset", + "name": "Test Dataset", + "schema": [ + { + "id": "valid-prop-id", + "name": "Valid Property", + "type": "String", + "required": True, + }, + { + "id": "invalid-prop-id", + "name": "Invalid Property", + "type": "UnknownType", # This should cause the property to be skipped + "required": False, + }, + { + "id": "another-valid-prop-id", + "name": "Another Valid Property", + "type": "Number", + "required": False, + "defaultValue": 42, + }, + ], + } + + # This should not raise an exception + dataset = deserialize_model(Dataset, dataset_data) + + # Verify the dataset was created successfully + assert dataset.id == "dataset-id-1" + assert dataset.external_id == "test-dataset" + assert dataset.name == "Test Dataset" + + # Should have only 2 valid properties (invalid one skipped) + assert dataset.schema_properties is not None + assert len(dataset.schema_properties) == 2 + + # Check that valid properties are present + prop_names = [prop.name for prop in dataset.schema_properties] + assert "Valid Property" in prop_names + assert "Another Valid Property" in prop_names + assert "Invalid Property" not in prop_names + + +def test_dataset_schema_property_factory_function(): + """Test the schema property factory function directly.""" + # Test creating a number property with defaultValue + number_prop_data = { + "id": "test-number-prop", + "name": "Test Number", + "type": "Number", + "required": False, + "defaultValue": 42, + } + + number_prop = create_schema_property(number_prop_data) + assert number_prop.id == "test-number-prop" + assert number_prop.name == "Test Number" + assert number_prop.type == SchemaPropertyType.NUMBER + assert number_prop.required is False + assert number_prop.default_value == 42 + + # Test creating a select property with defaultValue + select_prop_data = { + "id": "test-select-prop", + "name": "Test Select", + "type": "Select", + "required": True, + "options": ["option1", "option2", "option3"], + "defaultValue": "option2", + } + + select_prop = create_schema_property(select_prop_data) + assert select_prop.id == "test-select-prop" + assert select_prop.name == "Test Select" + assert select_prop.type == SchemaPropertyType.SELECT + assert select_prop.required is True + assert select_prop.default_value == "option2" + # Type check before accessing options + assert isinstance(select_prop, SelectProperty) + assert select_prop.options == ["option1", "option2", "option3"] + + # Test creating a string property without defaultValue + string_prop_data = { + "id": "test-string-prop", + "name": "Test String", + "type": "String", + "required": True, + } + + string_prop = create_schema_property(string_prop_data) + assert string_prop.id == "test-string-prop" + assert string_prop.name == "Test String" + assert string_prop.type == SchemaPropertyType.STRING + assert string_prop.required is True + assert string_prop.default_value is None