Adam/epd 2036 update python sdk to support creating human review jobs in v2

autoblocks/_impl/testing/models.py

@@ -265,6 +265,7 @@ class CreateHumanReviewJob:
 
     assignee_email_address: Union[str, list[str]]
     name: str
+    rubric_id: Optional[str] = None
 
     def get_assignee_email_addresses(self) -> list[str]:
         if isinstance(self.assignee_email_address, str):

autoblocks/_impl/testing/v2/api.py

@@ -0,0 +1,80 @@
+import logging
+from typing import Any
+from typing import List
+from typing import Optional
+
+from httpx import Response
+from tenacity import retry
+from tenacity import stop_after_attempt
+from tenacity import wait_random_exponential
+
+from autoblocks._impl import global_state
+from autoblocks._impl.config.constants import API_ENDPOINT_V2
+from autoblocks._impl.util import AutoblocksEnvVar
+
+log = logging.getLogger(__name__)
+
+TIMEOUT_SECONDS = 30
+
+
+@retry(stop=stop_after_attempt(3), wait=wait_random_exponential(multiplier=1, max=30), reraise=True)
+async def post_to_api_with_retry(
+    url: str,
+    api_key: str,
+    json: dict[str, Any],
+) -> Response:
+    resp = await global_state.http_client().post(
+        url,
+        json=json,
+        timeout=TIMEOUT_SECONDS,
+        headers={"Authorization": f"Bearer {api_key}"},
+    )
+    if not resp.is_success:
+        try:
+            error_body = resp.text
+            log.error(f"API request failed with status {resp.status_code} for {url}. Response body: {error_body}")
+        except Exception:
+            log.error(f"API request failed with status {resp.status_code} for {url}. Could not read response body.")
+
+    resp.raise_for_status()
+    return resp
+
+
+async def post_to_api(
+    path: str,
+    json: dict[str, Any],
+) -> Response:
+    api_key = AutoblocksEnvVar.V2_API_KEY.get()
+    if not api_key:
+        raise ValueError(f"You must set the {AutoblocksEnvVar.V2_API_KEY} environment variable.")
+
+    url = f"{API_ENDPOINT_V2}{path}"
+    async with global_state.test_run_api_semaphore():
+        return await post_to_api_with_retry(
+            url,
+            api_key,
+            json,
+        )
+
+
+async def send_create_human_review_job(
+    run_id: str,
+    start_timestamp: str,
+    end_timestamp: str,
+    assignee_email_addresses: List[str],
+    name: str,
+    app_slug: str,
+    rubric_id: Optional[str] = None,
+) -> None:
+
+    await post_to_api(
+        f"/apps/{app_slug}/human-review/jobs",
+        json=dict(
+            runId=run_id,
+            startTimestamp=start_timestamp,
+            endTimestamp=end_timestamp,
+            rubricId=rubric_id,
+            assigneeEmailAddresses=assignee_email_addresses,
+            name=name,
+        ),
+    )

autoblocks/_impl/testing/v2/run.py

@@ -28,6 +28,7 @@
 from autoblocks._impl.context_vars import test_run_context_var
 from autoblocks._impl.testing.models import BaseTestCase
 from autoblocks._impl.testing.models import BaseTestEvaluator
+from autoblocks._impl.testing.models import CreateHumanReviewJob
 from autoblocks._impl.testing.models import Evaluation
 from autoblocks._impl.testing.models import EvaluationWithId
 from autoblocks._impl.testing.models import TestCaseContext
@@ -38,10 +39,12 @@
 from autoblocks._impl.testing.util import serialize_test_case
 from autoblocks._impl.testing.util import yield_grid_search_param_combos
 from autoblocks._impl.testing.util import yield_test_case_contexts_from_test_cases
+from autoblocks._impl.testing.v2.api import send_create_human_review_job
 from autoblocks._impl.tracer.util import SpanAttribute
 from autoblocks._impl.util import AutoblocksEnvVar
 from autoblocks._impl.util import all_settled
 from autoblocks._impl.util import cuid_generator
+from autoblocks._impl.util import now_rfc3339
 from autoblocks._impl.util import parse_autoblocks_overrides
 from autoblocks._impl.util import serialize_to_string
 
@@ -351,8 +354,10 @@ async def run_test_suite_for_grid_combo(
     fn: Union[Callable[[TestCaseType], Any], Callable[[TestCaseType], Awaitable[Any]]],
     before_evaluators_hook: Optional[Callable[[TestCaseType, Any], Any]],
     grid_search_params_combo: Optional[GridSearchParamsCombo],
+    human_review_job: Optional[CreateHumanReviewJob],
 ) -> None:
     run_id = cuid_generator()
+    start_timestamp = now_rfc3339()
 
     log.info(f"Running test suite '{test_id}' with {len(test_cases)} test cases")
     # Determine message with priority: unified overrides > legacy env var
@@ -391,6 +396,21 @@ async def run_test_suite_for_grid_combo(
         if test_run_reset_token:
             test_run_context_var.reset(test_run_reset_token)
 
+    end_timestamp = now_rfc3339()
+    if human_review_job is not None:
+        try:
+            await send_create_human_review_job(
+                run_id=run_id,
+                app_slug=app_slug,
+                start_timestamp=start_timestamp,
+                end_timestamp=end_timestamp,
+                assignee_email_addresses=human_review_job.get_assignee_email_addresses(),
+                name=human_review_job.name,
+                rubric_id=human_review_job.rubric_id,
+            )
+        except Exception as err:
+            log.warn(f"Failed to create human review job for test run '{run_id}'", exc_info=err)
+
 
 async def async_run_test_suite(
     test_id: str,
@@ -401,6 +421,7 @@ async def async_run_test_suite(
     before_evaluators_hook: Optional[Callable[[TestCaseType, Any], Any]],
     max_test_case_concurrency: int,
     grid_search_params: Optional[GridSearchParams],
+    human_review_job: Optional[CreateHumanReviewJob],
 ) -> None:
 
     # This will be set if the user passed filters to the CLI
@@ -453,6 +474,7 @@ async def async_run_test_suite(
                 fn=fn,
                 before_evaluators_hook=before_evaluators_hook,
                 grid_search_params_combo=None,
+                human_review_job=human_review_job,
             )
         except Exception as err:
             log.error(f"Error running test suite '{test_id}'", exc_info=err)
@@ -469,6 +491,7 @@ async def async_run_test_suite(
                     fn=fn,
                     before_evaluators_hook=before_evaluators_hook,
                     grid_search_params_combo=grid_params_combo,
+                    human_review_job=human_review_job,
                 )
                 for grid_params_combo in yield_grid_search_param_combos(grid_search_params)
             ],
@@ -488,6 +511,7 @@ def run_test_suite(
     max_test_case_concurrency: int = DEFAULT_MAX_TEST_CASE_CONCURRENCY,
     before_evaluators_hook: Optional[Callable[[TestCaseType, Any], Any]] = None,
     grid_search_params: Optional[GridSearchParams] = None,
+    human_review_job: Optional[CreateHumanReviewJob] = None,
 ) -> None: ...
 
 
@@ -502,6 +526,7 @@ def run_test_suite(
     max_test_case_concurrency: int = DEFAULT_MAX_TEST_CASE_CONCURRENCY,
     before_evaluators_hook: Optional[Callable[[TestCaseType, Any], Any]] = None,
     grid_search_params: Optional[GridSearchParams] = None,
+    human_review_job: Optional[CreateHumanReviewJob] = None,
 ) -> None: ...
 
 
@@ -515,6 +540,7 @@ def run_test_suite(
     max_test_case_concurrency: int = DEFAULT_MAX_TEST_CASE_CONCURRENCY,
     before_evaluators_hook: Optional[Callable[[TestCaseType, Any], Any]] = None,
     grid_search_params: Optional[GridSearchParams] = None,
+    human_review_job: Optional[CreateHumanReviewJob] = None,
 ) -> None:
     if not global_state.is_auto_tracer_initialized():
         log.error(
@@ -534,6 +560,7 @@ def run_test_suite(
             before_evaluators_hook=before_evaluators_hook,
             max_test_case_concurrency=max_test_case_concurrency,
             grid_search_params=grid_search_params,
+            human_review_job=human_review_job,
         ),
         global_state.event_loop(),
     ).result()

autoblocks/_impl/util.py

@@ -145,6 +145,15 @@ def now_iso_8601() -> str:
     return datetime.now(timezone.utc).isoformat()
 
 
+def now_rfc3339() -> str:
+    """
+    Returns the current UTC timestamp in RFC 3339 format.
+    RFC 3339 is a profile of ISO 8601 commonly used in APIs.
+    Format: 2025-06-25T20:47:46.429Z
+    """
+    return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
 def is_cli_running() -> bool:
     return AutoblocksEnvVar.CLI_SERVER_ADDRESS.get() is not None