diff --git a/.autoblocks.yml b/.autoblocks.yml index 6cecd61a..74ddaa5a 100644 --- a/.autoblocks.yml +++ b/.autoblocks.yml @@ -4,7 +4,7 @@ autogenerate: prompts: - id: used-by-ci-dont-delete - major_version: 6 + major_version: 7 - id: used-by-ci-dont-delete-with-tools major_version: 1 diff --git a/autoblocks/_impl/prompts/autogenerate.py b/autoblocks/_impl/prompts/autogenerate.py index 05cb9057..7347cf48 100644 --- a/autoblocks/_impl/prompts/autogenerate.py +++ b/autoblocks/_impl/prompts/autogenerate.py @@ -18,6 +18,11 @@ from autoblocks._impl.util import AutoblocksEnvVar from autoblocks._impl.util import encode_uri_component +try: + from pydantic import AliasChoices +except ImportError: # pragma: no cover - older pydantic + AliasChoices = None # type: ignore + class Template(FrozenModel): id: str @@ -99,8 +104,8 @@ def generate_params_class_code(prompt: PromptCodegen) -> str: if type_hint is None: continue snake_case_key = to_snake_case(key) - auto += f'{indent()}{snake_case_key}: {type_hint} = pydantic.Field(..., alias="{key}")\n' + auto += f'{indent()}{snake_case_key}: {type_hint} = pydantic.Field(..., alias="{key}")\n' return auto diff --git a/autoblocks/_impl/prompts/v2/discovery/prompts.py b/autoblocks/_impl/prompts/v2/discovery/prompts.py index 7df3218b..290a8670 100644 --- a/autoblocks/_impl/prompts/v2/discovery/prompts.py +++ b/autoblocks/_impl/prompts/v2/discovery/prompts.py @@ -9,6 +9,12 @@ from autoblocks._impl.prompts.utils import infer_type from autoblocks._impl.prompts.utils import to_snake_case from autoblocks._impl.prompts.utils import to_title_case + +try: + from pydantic import AliasChoices +except ImportError: # pragma: no cover - older pydantic + AliasChoices = None # type: ignore + from autoblocks._impl.prompts.v2.client import PromptsAPIClient from autoblocks._impl.prompts.v2.discovery.managers import generate_execution_context_class_code from autoblocks._impl.prompts.v2.discovery.managers import generate_factory_class_code @@ -42,7 +48,6 @@ def generate_params_class_code(title_case_id: str, version: str, params: Dict[st continue snake_case_key = to_snake_case(key) auto += f'{indent()}{snake_case_key}: {type_hint} = pydantic.Field(..., alias="{key}")\n' - return auto diff --git a/poetry.lock b/poetry.lock index 1a7df2c7..dfc7e98d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -156,7 +156,7 @@ version = "0.7.0" description = "Reusable constraint types to use with typing.Annotated" optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -2917,14 +2917,14 @@ files = [ [[package]] name = "pydantic" -version = "2.11.4" +version = "2.11.7" description = "Data validation using Python type hints" optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main", "dev"] files = [ - {file = "pydantic-2.11.4-py3-none-any.whl", hash = "sha256:d9615eaa9ac5a063471da949c8fc16376a84afb5024688b3ff885693506764eb"}, - {file = "pydantic-2.11.4.tar.gz", hash = "sha256:32738d19d63a226a52eed76645a98ee07c1f410ee41d93b4afbfa85ed8111c2d"}, + {file = "pydantic-2.11.7-py3-none-any.whl", hash = "sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b"}, + {file = "pydantic-2.11.7.tar.gz", hash = "sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db"}, ] [package.dependencies] @@ -2943,7 +2943,7 @@ version = "2.33.2" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "pydantic_core-2.33.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8"}, {file = "pydantic_core-2.33.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d"}, @@ -3878,7 +3878,7 @@ version = "0.4.0" description = "Runtime typing introspection tools" optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "typing_inspection-0.4.0-py3-none-any.whl", hash = "sha256:50e72559fcd2a6367a19f7a7e610e6afcb9fac940c650290eed893d61386832f"}, {file = "typing_inspection-0.4.0.tar.gz", hash = "sha256:9765c87de36671694a67904bf2c96e395be9c6439bb6c87b5142569dcdd65122"}, @@ -4417,4 +4417,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.1" python-versions = "^3.9.0" -content-hash = "a3e1998454835c109cfc333cc3b590ec21c744cac704bb45fd41207133dca3a0" +content-hash = "da192192783484f4ddad5a351f88770a5a4e81dd6c22634144812de64c870237" diff --git a/pyproject.toml b/pyproject.toml index ce6fcb78..cce89468 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ cuid2 = ">=2.0.0" opentelemetry-exporter-otlp-proto-http = ">=1.0.0" opentelemetry-api = ">=1.0.0" opentelemetry-sdk = ">=1.0.0" +pydantic = ">=2.11.7" [tool.poetry.group.dev.dependencies] pre-commit = "^4.0.0" diff --git a/tests/autoblocks/__snapshots__/test_prompts_autogenerate.ambr b/tests/autoblocks/__snapshots__/test_prompts_autogenerate.ambr index f9270f1b..a7245e58 100644 --- a/tests/autoblocks/__snapshots__/test_prompts_autogenerate.ambr +++ b/tests/autoblocks/__snapshots__/test_prompts_autogenerate.ambr @@ -20,7 +20,7 @@ class PromptAParams(FrozenModel): frequency_penalty: Union[float, int] = pydantic.Field(..., alias="frequencyPenalty") - max_tokens: Union[float, int] = pydantic.Field(..., alias="maxTokens") + max_completion_tokens: Union[float, int] = pydantic.Field(..., alias="maxCompletionTokens") model: str = pydantic.Field(..., alias="model") presence_penalty: Union[float, int] = pydantic.Field(..., alias="presencePenalty") temperature: Union[float, int] = pydantic.Field(..., alias="temperature") @@ -89,7 +89,7 @@ class PromptBParams(FrozenModel): frequency_penalty: Union[float, int] = pydantic.Field(..., alias="frequencyPenalty") - max_tokens: Union[float, int] = pydantic.Field(..., alias="maxTokens") + max_completion_tokens: Union[float, int] = pydantic.Field(..., alias="maxCompletionTokens") model: str = pydantic.Field(..., alias="model") presence_penalty: Union[float, int] = pydantic.Field(..., alias="presencePenalty") temperature: Union[float, int] = pydantic.Field(..., alias="temperature") diff --git a/tests/autoblocks/discovery/test_prompts.py b/tests/autoblocks/discovery/test_prompts.py index 9b86f47d..a18007aa 100644 --- a/tests/autoblocks/discovery/test_prompts.py +++ b/tests/autoblocks/discovery/test_prompts.py @@ -6,6 +6,11 @@ from autoblocks._impl.prompts.v2.discovery.prompts import generate_prompt_implementations from autoblocks._impl.prompts.v2.discovery.prompts import generate_version_implementations +try: + from pydantic import AliasChoices +except ImportError: # pragma: no cover - older pydantic + AliasChoices = None # type: ignore + class TestPrompts: def test_generate_params_class_code_empty(self): @@ -26,7 +31,7 @@ def test_generate_params_class_code(self): version = "1" params = { "temperature": 0.7, - "maxTokens": 100, + "maxCompletionTokens": 100, "model": "gpt-4", "isEnabled": True, } @@ -36,7 +41,7 @@ def test_generate_params_class_code(self): # Check class definition and parameters assert "class _TestPromptV1Params(FrozenModel):" in result assert 'temperature: Union[float, int] = pydantic.Field(..., alias="temperature")' in result - assert 'max_tokens: Union[float, int] = pydantic.Field(..., alias="maxTokens")' in result + assert 'max_completion_tokens: Union[float, int] = pydantic.Field(..., alias="maxCompletionTokens")' in result assert ' model: str = pydantic.Field(..., alias="model")' in result assert ' is_enabled: bool = pydantic.Field(..., alias="isEnabled")' in result @@ -47,7 +52,7 @@ def test_generate_params_class_code_nested(self): params = { "params": { "temperature": 0.7, - "maxTokens": 100, + "maxCompletionTokens": 100, } } @@ -56,7 +61,7 @@ def test_generate_params_class_code_nested(self): # Check class definition and extracted parameters assert "class _TestPromptV1Params(FrozenModel):" in result assert 'temperature: Union[float, int] = pydantic.Field(..., alias="temperature")' in result - assert 'max_tokens: Union[float, int] = pydantic.Field(..., alias="maxTokens")' in result + assert 'max_completion_tokens: Union[float, int] = pydantic.Field(..., alias="maxCompletionTokens")' in result @patch("autoblocks._impl.prompts.v2.discovery.prompts.generate_params_class_code") @patch("autoblocks._impl.prompts.v2.discovery.prompts.generate_template_renderer_class_code") diff --git a/tests/autoblocks/test_prompts_autogenerate.py b/tests/autoblocks/test_prompts_autogenerate.py index ba870634..5ea0be23 100644 --- a/tests/autoblocks/test_prompts_autogenerate.py +++ b/tests/autoblocks/test_prompts_autogenerate.py @@ -171,7 +171,7 @@ def test_write(httpx_mock, snapshot): "params": { "params": { "frequencyPenalty": 0, - "maxTokens": 256, + "maxCompletionTokens": 256, "model": "gpt-4", "presencePenalty": 0.3, "stopSequences": [], @@ -209,7 +209,7 @@ def test_write(httpx_mock, snapshot): "params": { "params": { "frequencyPenalty": 0, - "maxTokens": 256, + "maxCompletionTokens": 256, "model": "gpt-4", "presencePenalty": -0.3, "stopSequences": [], diff --git a/tests/e2e/prompts.py b/tests/e2e/prompts.py index fbee22e8..2e0c62c8 100644 --- a/tests/e2e/prompts.py +++ b/tests/e2e/prompts.py @@ -14,13 +14,21 @@ from autoblocks.prompts.renderer import TemplateRenderer from autoblocks.prompts.renderer import ToolRenderer +try: + from pydantic import AliasChoices +except ImportError: # pragma: no cover - older pydantic + AliasChoices = None # type: ignore + class QuestionAnswererParams(FrozenModel): temperature: Union[float, int] = pydantic.Field(..., alias="temperature") top_p: Union[float, int] = pydantic.Field(..., alias="topP") frequency_penalty: Union[float, int] = pydantic.Field(..., alias="frequencyPenalty") presence_penalty: Union[float, int] = pydantic.Field(..., alias="presencePenalty") - max_tokens: Union[float, int] = pydantic.Field(..., alias="maxTokens") + max_completion_tokens: Union[float, int] = pydantic.Field( + ..., + alias="maxCompletionTokens", + ) model: str = pydantic.Field(..., alias="model") @@ -93,7 +101,10 @@ class TextSummarizationParams(FrozenModel): top_p: Union[float, int] = pydantic.Field(..., alias="topP") frequency_penalty: Union[float, int] = pydantic.Field(..., alias="frequencyPenalty") presence_penalty: Union[float, int] = pydantic.Field(..., alias="presencePenalty") - max_tokens: Union[float, int] = pydantic.Field(..., alias="maxTokens") + max_completion_tokens: Union[float, int] = pydantic.Field( + ..., + alias="maxCompletionTokens", + ) model: str = pydantic.Field(..., alias="model") @@ -178,7 +189,11 @@ class UsedByCiDontDeleteParams(FrozenModel): top_p: Union[float, int] = pydantic.Field(..., alias="topP") frequency_penalty: Union[float, int] = pydantic.Field(..., alias="frequencyPenalty") presence_penalty: Union[float, int] = pydantic.Field(..., alias="presencePenalty") - max_tokens: Union[float, int] = pydantic.Field(..., alias="maxTokens") + max_completion_tokens: Union[float, int] = pydantic.Field( + ..., + alias="maxCompletionTokens", + ) + seed: Union[float, int] = pydantic.Field(..., alias="seed") model: str = pydantic.Field(..., alias="model") response_format: Dict[str, Any] = pydantic.Field(..., alias="responseFormat") @@ -240,7 +255,7 @@ class UsedByCiDontDeletePromptManager( AutoblocksPromptManager[UsedByCiDontDeleteExecutionContext], ): __prompt_id__ = "used-by-ci-dont-delete" - __prompt_major_version__ = "6" + __prompt_major_version__ = "7" __execution_context_class__ = UsedByCiDontDeleteExecutionContext diff --git a/tests/e2e/prompts_v2/apps/ci_app/__init__.py b/tests/e2e/prompts_v2/apps/ci_app/__init__.py index 7409bbdc..4da31845 100644 --- a/tests/e2e/prompts_v2/apps/ci_app/__init__.py +++ b/tests/e2e/prompts_v2/apps/ci_app/__init__.py @@ -12,7 +12,12 @@ def prompt_basic_prompt_manager( init_timeout: Optional[float] = None, refresh_timeout: Optional[float] = None, refresh_interval: Optional[float] = None, -) -> Union[prompts._PromptBasicV1PromptManager, prompts._PromptBasicV2PromptManager]: +) -> Union[ + prompts._PromptBasicV1PromptManager, + prompts._PromptBasicV2PromptManager, + prompts._PromptBasicV3PromptManager, + prompts._PromptBasicV4PromptManager, +]: return prompts.PromptBasicFactory.create( major_version=major_version, minor_version=minor_version, diff --git a/tests/e2e/prompts_v2/apps/ci_app/prompts.py b/tests/e2e/prompts_v2/apps/ci_app/prompts.py index 42e12f0b..54f471c1 100644 --- a/tests/e2e/prompts_v2/apps/ci_app/prompts.py +++ b/tests/e2e/prompts_v2/apps/ci_app/prompts.py @@ -12,8 +12,88 @@ from autoblocks.prompts.v2.renderer import ToolRenderer +class _PromptBasicV4Params(FrozenModel): + max_completion_tokens: Union[float, int] = pydantic.Field( + ..., + alias="maxCompletionTokens", + ) + model: str = pydantic.Field(..., alias="model") + + +class _PromptBasicV4TemplateRenderer(TemplateRenderer): + __name_mapper__ = { + "first_name": "first_name", + } + + def template_c( + self, + *, + first_name: str, + ) -> str: + return self._render( + "template-c", + first_name=first_name, + ) + + +class _PromptBasicV4ToolRenderer(ToolRenderer): + __name_mapper__ = {} + + +class _PromptBasicV4ExecutionContext( + PromptExecutionContext[_PromptBasicV4Params, _PromptBasicV4TemplateRenderer, _PromptBasicV4ToolRenderer] +): + __params_class__ = _PromptBasicV4Params + __template_renderer_class__ = _PromptBasicV4TemplateRenderer + __tool_renderer_class__ = _PromptBasicV4ToolRenderer + + +class _PromptBasicV4PromptManager(AutoblocksPromptManager[_PromptBasicV4ExecutionContext]): + __app_id__ = "b5sz3k5d61w9f8325fhxuxkr" + __prompt_id__ = "prompt-basic" + __prompt_major_version__ = "4" + __execution_context_class__ = _PromptBasicV4ExecutionContext + + +class _PromptBasicV3Params(FrozenModel): + max_completion_tokens: Union[float, int] = pydantic.Field( + ..., + alias="maxCompletionTokens", + ) + model: str = pydantic.Field(..., alias="model") + + +class _PromptBasicV3TemplateRenderer(TemplateRenderer): + __name_mapper__ = {} + + def template_c( + self, + ) -> str: + return self._render( + "template-c", + ) + + +class _PromptBasicV3ToolRenderer(ToolRenderer): + __name_mapper__ = {} + + +class _PromptBasicV3ExecutionContext( + PromptExecutionContext[_PromptBasicV3Params, _PromptBasicV3TemplateRenderer, _PromptBasicV3ToolRenderer] +): + __params_class__ = _PromptBasicV3Params + __template_renderer_class__ = _PromptBasicV3TemplateRenderer + __tool_renderer_class__ = _PromptBasicV3ToolRenderer + + +class _PromptBasicV3PromptManager(AutoblocksPromptManager[_PromptBasicV3ExecutionContext]): + __app_id__ = "b5sz3k5d61w9f8325fhxuxkr" + __prompt_id__ = "prompt-basic" + __prompt_major_version__ = "3" + __execution_context_class__ = _PromptBasicV3ExecutionContext + + class _PromptBasicV2Params(FrozenModel): - max_tokens: Union[float, int] = pydantic.Field(..., alias="maxTokens") model: str = pydantic.Field(..., alias="model") @@ -103,7 +183,12 @@ def create( init_timeout: Optional[float] = None, refresh_timeout: Optional[float] = None, refresh_interval: Optional[float] = None, - ) -> Union[_PromptBasicV1PromptManager, _PromptBasicV2PromptManager]: + ) -> Union[ + _PromptBasicV1PromptManager, + _PromptBasicV2PromptManager, + _PromptBasicV3PromptManager, + _PromptBasicV4PromptManager, + ]: kwargs: Dict[str, Any] = {} if api_key is not None: kwargs["api_key"] = api_key @@ -115,11 +200,15 @@ def create( kwargs["refresh_interval"] = refresh_interval if major_version is None: - major_version = "2" # Latest version + major_version = "4" # Latest version if major_version == "1": return _PromptBasicV1PromptManager(minor_version=minor_version, **kwargs) if major_version == "2": return _PromptBasicV2PromptManager(minor_version=minor_version, **kwargs) + if major_version == "3": + return _PromptBasicV3PromptManager(minor_version=minor_version, **kwargs) + if major_version == "4": + return _PromptBasicV4PromptManager(minor_version=minor_version, **kwargs) - raise ValueError("Unsupported major version. Available versions: 1, 2") + raise ValueError("Unsupported major version. Available versions: 1, 2, 3, 4") diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index b23e3a07..d4182ea5 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -225,7 +225,7 @@ def test_prompt_manager(): ) with mgr.exec() as ctx: - assert ctx.params.max_tokens == 256 + assert ctx.params.max_completion_tokens == 256 assert ctx.params.model == "gpt-4" assert ctx.params.temperature == 0.3 assert ctx.params.top_p == 1 @@ -246,18 +246,18 @@ def test_prompt_manager(): ctx.render_template.template_b( name="Alice", ) - == "My name is Alice." + == "My name is Alice!" ) assert ctx.render_template.template_c() == "I am template c and I have no params" assert ctx.track() == { "id": "used-by-ci-dont-delete", - "version": "6.0", - "revisionId": "cm6gsq0t60003nbscwcqkdgat", + "version": "7.0", + "revisionId": "cmdn4jziw0003lb99ckzpddix", "params": { "params": { - "maxTokens": 256, + "maxCompletionTokens": 256, "model": "gpt-4", "stopSequences": [], "temperature": 0.3, @@ -275,7 +275,7 @@ def test_prompt_manager(): }, { "id": "template-b", - "template": "My name is {{ name }}.", + "template": "My name is {{ name }}!", }, { "id": "template-c", @@ -292,7 +292,7 @@ def test_prompt_manager_latest(): ) with mgr.exec() as ctx: - assert ctx.params.max_tokens == 256 + assert ctx.params.max_completion_tokens == 256 assert ctx.params.model == "gpt-4" assert ctx.params.temperature == 0.3 assert ctx.params.top_p == 1 @@ -320,11 +320,11 @@ def test_prompt_manager_latest(): assert ctx.track() == { "id": "used-by-ci-dont-delete", - "version": "6.1", - "revisionId": "cm6gswg4z000b11nw5dyqmvqw", + "version": "7.0", + "revisionId": "cmdn4jziw0003lb99ckzpddix", "params": { "params": { - "maxTokens": 256, + "maxCompletionTokens": 256, "model": "gpt-4", "stopSequences": [], "temperature": 0.3, @@ -369,7 +369,7 @@ def test_prompt_manager_weighted(): with mgr.exec() as ctx: assert ctx.params.model == "gpt-4" - assert ctx.track()["version"] in ("6.0", "6.1") + assert ctx.track()["version"] in ("7.0", "7.1") def test_prompt_manager_no_model_params(): @@ -505,7 +505,7 @@ def test_init_prompt_manager_inside_test_suite(httpx_mock): dict( entityExternalId="used-by-ci-dont-delete", entityType="prompt", - revisionId="cm6gswg4z000b11nw5dyqmvqw", + revisionId="cmdn4jziw0003lb99ckzpddix", usedAt=mock.ANY, ), ], @@ -555,13 +555,15 @@ def hash(self): def test_fn(test_case: MyTestCase) -> str: mgr = UsedByCiDontDeletePromptManager( - minor_version="1", + minor_version="0", ) with mgr.exec() as prompt: return prompt.params.model class MyEvaluator(BaseTestEvaluator): - id = "my-evaluator" + @property + def id(self): + return "my-evaluator" mgr = UsedByCiDontDeleteNoParamsPromptManager( minor_version="0", @@ -675,7 +677,9 @@ async def test_fn(test_case: MyTestCase) -> str: return f"{test_case.x}" class MyEvaluator(BaseTestEvaluator): - id = "my-evaluator" + @property + def id(self): + return "my-evaluator" async def evaluate_test_case(self, test_case: MyTestCase, output: str) -> Evaluation: return Evaluation(score=0.97) diff --git a/tests/e2e/test_prompts_v2.py b/tests/e2e/test_prompts_v2.py index 9de036cc..87f0415e 100644 --- a/tests/e2e/test_prompts_v2.py +++ b/tests/e2e/test_prompts_v2.py @@ -36,18 +36,18 @@ def test_app_sdk_test_prompt_basic_v1(): def test_app_sdk_test_prompt_basic_v2(): """Test the app_sdk_test.prompt_basic in major version 2.""" mgr = ci_app.prompt_basic_prompt_manager( - major_version="2", + major_version="4", minor_version="0", ) with mgr.exec() as ctx: # Assert parameters assert ctx.params.model == "gpt-4o" - assert ctx.params.max_tokens == 256 # type: ignore[union-attr] + assert ctx.params.max_completion_tokens == 256 # type: ignore[union-attr] # Test template rendering assert ( - ctx.render_template.template_c( # type: ignore[union-attr] + ctx.render_template.template_c( # type: ignore[union-attr,call-arg] first_name="Alice", ) == "Hello, Alice!" @@ -56,7 +56,7 @@ def test_app_sdk_test_prompt_basic_v2(): # Check tracking info track_info = ctx.track() assert track_info["id"] == "prompt-basic" - assert track_info["version"].startswith("2.") + assert track_info["version"].startswith("4.") assert track_info["appId"] == "b5sz3k5d61w9f8325fhxuxkr" assert "templates" in track_info assert len(track_info["templates"]) == 1 @@ -68,7 +68,7 @@ def test_prompt_revision_override_structure(): """Test that V2 prompts have the same structure as V1 for revision overrides.""" # This test verifies that the manager has the necessary attributes for overrides mgr = ci_app.prompt_basic_prompt_manager( - major_version="2", + major_version="4", minor_version="0", )