diff --git a/autoblocks/_impl/testing/models.py b/autoblocks/_impl/testing/models.py index 32ae2c0e..aa36e0fc 100644 --- a/autoblocks/_impl/testing/models.py +++ b/autoblocks/_impl/testing/models.py @@ -265,6 +265,7 @@ class CreateHumanReviewJob: assignee_email_address: Union[str, list[str]] name: str + rubric_id: Optional[str] = None def get_assignee_email_addresses(self) -> list[str]: if isinstance(self.assignee_email_address, str): diff --git a/autoblocks/_impl/testing/v2/api.py b/autoblocks/_impl/testing/v2/api.py new file mode 100644 index 00000000..3260efb9 --- /dev/null +++ b/autoblocks/_impl/testing/v2/api.py @@ -0,0 +1,80 @@ +import logging +from typing import Any +from typing import List +from typing import Optional + +from httpx import Response +from tenacity import retry +from tenacity import stop_after_attempt +from tenacity import wait_random_exponential + +from autoblocks._impl import global_state +from autoblocks._impl.config.constants import API_ENDPOINT_V2 +from autoblocks._impl.util import AutoblocksEnvVar + +log = logging.getLogger(__name__) + +TIMEOUT_SECONDS = 30 + + +@retry(stop=stop_after_attempt(3), wait=wait_random_exponential(multiplier=1, max=30), reraise=True) +async def post_to_api_with_retry( + url: str, + api_key: str, + json: dict[str, Any], +) -> Response: + resp = await global_state.http_client().post( + url, + json=json, + timeout=TIMEOUT_SECONDS, + headers={"Authorization": f"Bearer {api_key}"}, + ) + if not resp.is_success: + try: + error_body = resp.text + log.error(f"API request failed with status {resp.status_code} for {url}. Response body: {error_body}") + except Exception: + log.error(f"API request failed with status {resp.status_code} for {url}. Could not read response body.") + + resp.raise_for_status() + return resp + + +async def post_to_api( + path: str, + json: dict[str, Any], +) -> Response: + api_key = AutoblocksEnvVar.V2_API_KEY.get() + if not api_key: + raise ValueError(f"You must set the {AutoblocksEnvVar.V2_API_KEY} environment variable.") + + url = f"{API_ENDPOINT_V2}{path}" + async with global_state.test_run_api_semaphore(): + return await post_to_api_with_retry( + url, + api_key, + json, + ) + + +async def send_create_human_review_job( + run_id: str, + start_timestamp: str, + end_timestamp: str, + assignee_email_addresses: List[str], + name: str, + app_slug: str, + rubric_id: Optional[str] = None, +) -> None: + + await post_to_api( + f"/apps/{app_slug}/human-review/jobs", + json=dict( + runId=run_id, + startTimestamp=start_timestamp, + endTimestamp=end_timestamp, + rubricId=rubric_id, + assigneeEmailAddresses=assignee_email_addresses, + name=name, + ), + ) diff --git a/autoblocks/_impl/testing/v2/run.py b/autoblocks/_impl/testing/v2/run.py index 19bdb768..c14cb1a9 100644 --- a/autoblocks/_impl/testing/v2/run.py +++ b/autoblocks/_impl/testing/v2/run.py @@ -28,6 +28,7 @@ from autoblocks._impl.context_vars import test_run_context_var from autoblocks._impl.testing.models import BaseTestCase from autoblocks._impl.testing.models import BaseTestEvaluator +from autoblocks._impl.testing.models import CreateHumanReviewJob from autoblocks._impl.testing.models import Evaluation from autoblocks._impl.testing.models import EvaluationWithId from autoblocks._impl.testing.models import TestCaseContext @@ -38,10 +39,12 @@ from autoblocks._impl.testing.util import serialize_test_case from autoblocks._impl.testing.util import yield_grid_search_param_combos from autoblocks._impl.testing.util import yield_test_case_contexts_from_test_cases +from autoblocks._impl.testing.v2.api import send_create_human_review_job from autoblocks._impl.tracer.util import SpanAttribute from autoblocks._impl.util import AutoblocksEnvVar from autoblocks._impl.util import all_settled from autoblocks._impl.util import cuid_generator +from autoblocks._impl.util import now_rfc3339 from autoblocks._impl.util import parse_autoblocks_overrides from autoblocks._impl.util import serialize_to_string @@ -351,8 +354,10 @@ async def run_test_suite_for_grid_combo( fn: Union[Callable[[TestCaseType], Any], Callable[[TestCaseType], Awaitable[Any]]], before_evaluators_hook: Optional[Callable[[TestCaseType, Any], Any]], grid_search_params_combo: Optional[GridSearchParamsCombo], + human_review_job: Optional[CreateHumanReviewJob], ) -> None: run_id = cuid_generator() + start_timestamp = now_rfc3339() log.info(f"Running test suite '{test_id}' with {len(test_cases)} test cases") # Determine message with priority: unified overrides > legacy env var @@ -391,6 +396,21 @@ async def run_test_suite_for_grid_combo( if test_run_reset_token: test_run_context_var.reset(test_run_reset_token) + end_timestamp = now_rfc3339() + if human_review_job is not None: + try: + await send_create_human_review_job( + run_id=run_id, + app_slug=app_slug, + start_timestamp=start_timestamp, + end_timestamp=end_timestamp, + assignee_email_addresses=human_review_job.get_assignee_email_addresses(), + name=human_review_job.name, + rubric_id=human_review_job.rubric_id, + ) + except Exception as err: + log.warn(f"Failed to create human review job for test run '{run_id}'", exc_info=err) + async def async_run_test_suite( test_id: str, @@ -401,6 +421,7 @@ async def async_run_test_suite( before_evaluators_hook: Optional[Callable[[TestCaseType, Any], Any]], max_test_case_concurrency: int, grid_search_params: Optional[GridSearchParams], + human_review_job: Optional[CreateHumanReviewJob], ) -> None: # This will be set if the user passed filters to the CLI @@ -453,6 +474,7 @@ async def async_run_test_suite( fn=fn, before_evaluators_hook=before_evaluators_hook, grid_search_params_combo=None, + human_review_job=human_review_job, ) except Exception as err: log.error(f"Error running test suite '{test_id}'", exc_info=err) @@ -469,6 +491,7 @@ async def async_run_test_suite( fn=fn, before_evaluators_hook=before_evaluators_hook, grid_search_params_combo=grid_params_combo, + human_review_job=human_review_job, ) for grid_params_combo in yield_grid_search_param_combos(grid_search_params) ], @@ -488,6 +511,7 @@ def run_test_suite( max_test_case_concurrency: int = DEFAULT_MAX_TEST_CASE_CONCURRENCY, before_evaluators_hook: Optional[Callable[[TestCaseType, Any], Any]] = None, grid_search_params: Optional[GridSearchParams] = None, + human_review_job: Optional[CreateHumanReviewJob] = None, ) -> None: ... @@ -502,6 +526,7 @@ def run_test_suite( max_test_case_concurrency: int = DEFAULT_MAX_TEST_CASE_CONCURRENCY, before_evaluators_hook: Optional[Callable[[TestCaseType, Any], Any]] = None, grid_search_params: Optional[GridSearchParams] = None, + human_review_job: Optional[CreateHumanReviewJob] = None, ) -> None: ... @@ -515,6 +540,7 @@ def run_test_suite( max_test_case_concurrency: int = DEFAULT_MAX_TEST_CASE_CONCURRENCY, before_evaluators_hook: Optional[Callable[[TestCaseType, Any], Any]] = None, grid_search_params: Optional[GridSearchParams] = None, + human_review_job: Optional[CreateHumanReviewJob] = None, ) -> None: if not global_state.is_auto_tracer_initialized(): log.error( @@ -534,6 +560,7 @@ def run_test_suite( before_evaluators_hook=before_evaluators_hook, max_test_case_concurrency=max_test_case_concurrency, grid_search_params=grid_search_params, + human_review_job=human_review_job, ), global_state.event_loop(), ).result() diff --git a/autoblocks/_impl/util.py b/autoblocks/_impl/util.py index 69d6fc6b..67df56c5 100644 --- a/autoblocks/_impl/util.py +++ b/autoblocks/_impl/util.py @@ -145,6 +145,15 @@ def now_iso_8601() -> str: return datetime.now(timezone.utc).isoformat() +def now_rfc3339() -> str: + """ + Returns the current UTC timestamp in RFC 3339 format. + RFC 3339 is a profile of ISO 8601 commonly used in APIs. + Format: 2025-06-25T20:47:46.429Z + """ + return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + def is_cli_running() -> bool: return AutoblocksEnvVar.CLI_SERVER_ADDRESS.get() is not None