From e5bba25b4277853fcf7a4fcdbab5607d046d5da4 Mon Sep 17 00:00:00 2001
From: Adam Nolte <ajnolte12@gmail.com>
Date: Wed, 25 Jun 2025 15:20:08 -0500
Subject: [PATCH 1/9] Add create human review job paramerter

---
 autoblocks/_impl/testing/v2/api.py | 70 ++++++++++++++++++++++++++++++
 autoblocks/_impl/testing/v2/run.py | 33 ++++++++++++++
 2 files changed, 103 insertions(+)
 create mode 100644 autoblocks/_impl/testing/v2/api.py

diff --git a/autoblocks/_impl/testing/v2/api.py b/autoblocks/_impl/testing/v2/api.py
new file mode 100644
index 00000000..8967a757
--- /dev/null
+++ b/autoblocks/_impl/testing/v2/api.py
@@ -0,0 +1,70 @@
+import logging
+from typing import Any
+from typing import Optional
+
+from httpx import Response
+from tenacity import retry
+from tenacity import stop_after_attempt
+from tenacity import wait_random_exponential
+
+from autoblocks._impl import global_state
+from autoblocks._impl.config.constants import API_ENDPOINT_V2
+from autoblocks._impl.util import AutoblocksEnvVar
+
+log = logging.getLogger(__name__)
+
+TIMEOUT_SECONDS = 30
+
+
+@retry(stop=stop_after_attempt(3), wait=wait_random_exponential(multiplier=1, max=30), reraise=True)
+async def post_to_api_with_retry(
+    url: str,
+    api_key: str,
+    json: dict[str, Any],
+) -> Response:
+    resp = await global_state.http_client().post(
+        url,
+        json=json,
+        timeout=TIMEOUT_SECONDS,
+        headers={"Authorization": f"Bearer {api_key}"},
+    )
+    resp.raise_for_status()
+    return resp
+
+
+async def post_to_api(
+    path: str,
+    json: dict[str, Any],
+) -> Response:
+    api_key = AutoblocksEnvVar.V2_API_KEY.get()
+    if not api_key:
+        raise ValueError(f"You must set the {AutoblocksEnvVar.V2_API_KEY} environment variable.")
+
+    async with global_state.test_run_api_semaphore():
+        return await post_to_api_with_retry(
+            f"{API_ENDPOINT_V2}{path}",
+            api_key,
+            json,
+        )
+
+
+async def send_create_human_review_job(
+    run_id: str,
+    start_timestamp: str,
+    end_timestamp: str,
+    assignee_email_address: str,
+    name: str,
+    app_slug: str,
+    rubric_id: Optional[str] = None,
+) -> None:
+    await post_to_api(
+        f"/apps/{app_slug}/human-review/jobs",
+        json=dict(
+            runId=run_id,
+            startTimestamp=start_timestamp,
+            endTimestamp=end_timestamp,
+            rubricId=rubric_id,
+            assigneeEmailAddress=assignee_email_address,
+            name=name,
+        ),
+    )
diff --git a/autoblocks/_impl/testing/v2/run.py b/autoblocks/_impl/testing/v2/run.py
index 19bdb768..698191bb 100644
--- a/autoblocks/_impl/testing/v2/run.py
+++ b/autoblocks/_impl/testing/v2/run.py
@@ -4,6 +4,8 @@
 import inspect
 import json
 import logging
+from datetime import datetime
+from datetime import timezone
 from typing import Any
 from typing import Awaitable
 from typing import Callable
@@ -28,6 +30,7 @@
 from autoblocks._impl.context_vars import test_run_context_var
 from autoblocks._impl.testing.models import BaseTestCase
 from autoblocks._impl.testing.models import BaseTestEvaluator
+from autoblocks._impl.testing.models import CreateHumanReviewJob
 from autoblocks._impl.testing.models import Evaluation
 from autoblocks._impl.testing.models import EvaluationWithId
 from autoblocks._impl.testing.models import TestCaseContext
@@ -38,6 +41,7 @@
 from autoblocks._impl.testing.util import serialize_test_case
 from autoblocks._impl.testing.util import yield_grid_search_param_combos
 from autoblocks._impl.testing.util import yield_test_case_contexts_from_test_cases
+from autoblocks._impl.testing.v2.api import send_create_human_review_job
 from autoblocks._impl.tracer.util import SpanAttribute
 from autoblocks._impl.util import AutoblocksEnvVar
 from autoblocks._impl.util import all_settled
@@ -351,8 +355,10 @@ async def run_test_suite_for_grid_combo(
     fn: Union[Callable[[TestCaseType], Any], Callable[[TestCaseType], Awaitable[Any]]],
     before_evaluators_hook: Optional[Callable[[TestCaseType, Any], Any]],
     grid_search_params_combo: Optional[GridSearchParamsCombo],
+    human_review_job: Optional[CreateHumanReviewJob],
 ) -> None:
     run_id = cuid_generator()
+    start_timestamp = datetime.now(timezone.utc).isoformat()
 
     log.info(f"Running test suite '{test_id}' with {len(test_cases)} test cases")
     # Determine message with priority: unified overrides > legacy env var
@@ -391,6 +397,26 @@ async def run_test_suite_for_grid_combo(
         if test_run_reset_token:
             test_run_context_var.reset(test_run_reset_token)
 
+    end_timestamp = datetime.now(timezone.utc).isoformat()
+    if human_review_job is not None:
+        try:
+            assignee_email_addresses = human_review_job.get_assignee_email_addresses()
+            await all_settled(
+                [
+                    send_create_human_review_job(
+                        run_id=run_id,
+                        app_slug=app_slug,
+                        start_timestamp=start_timestamp,
+                        end_timestamp=end_timestamp,
+                        assignee_email_address=assignee_email_address,
+                        name=human_review_job.name,
+                    )
+                    for assignee_email_address in assignee_email_addresses
+                ]
+            )
+        except Exception as err:
+            log.warn(f"Failed to create human review job for test run '{run_id}'", exc_info=err)
+
 
 async def async_run_test_suite(
     test_id: str,
@@ -401,6 +427,7 @@ async def async_run_test_suite(
     before_evaluators_hook: Optional[Callable[[TestCaseType, Any], Any]],
     max_test_case_concurrency: int,
     grid_search_params: Optional[GridSearchParams],
+    human_review_job: Optional[CreateHumanReviewJob],
 ) -> None:
 
     # This will be set if the user passed filters to the CLI
@@ -453,6 +480,7 @@ async def async_run_test_suite(
                 fn=fn,
                 before_evaluators_hook=before_evaluators_hook,
                 grid_search_params_combo=None,
+                human_review_job=human_review_job,
             )
         except Exception as err:
             log.error(f"Error running test suite '{test_id}'", exc_info=err)
@@ -469,6 +497,7 @@ async def async_run_test_suite(
                     fn=fn,
                     before_evaluators_hook=before_evaluators_hook,
                     grid_search_params_combo=grid_params_combo,
+                    human_review_job=human_review_job,
                 )
                 for grid_params_combo in yield_grid_search_param_combos(grid_search_params)
             ],
@@ -488,6 +517,7 @@ def run_test_suite(
     max_test_case_concurrency: int = DEFAULT_MAX_TEST_CASE_CONCURRENCY,
     before_evaluators_hook: Optional[Callable[[TestCaseType, Any], Any]] = None,
     grid_search_params: Optional[GridSearchParams] = None,
+    human_review_job: Optional[CreateHumanReviewJob] = None,
 ) -> None: ...
 
 
@@ -502,6 +532,7 @@ def run_test_suite(
     max_test_case_concurrency: int = DEFAULT_MAX_TEST_CASE_CONCURRENCY,
     before_evaluators_hook: Optional[Callable[[TestCaseType, Any], Any]] = None,
     grid_search_params: Optional[GridSearchParams] = None,
+    human_review_job: Optional[CreateHumanReviewJob] = None,
 ) -> None: ...
 
 
@@ -515,6 +546,7 @@ def run_test_suite(
     max_test_case_concurrency: int = DEFAULT_MAX_TEST_CASE_CONCURRENCY,
     before_evaluators_hook: Optional[Callable[[TestCaseType, Any], Any]] = None,
     grid_search_params: Optional[GridSearchParams] = None,
+    human_review_job: Optional[CreateHumanReviewJob] = None,
 ) -> None:
     if not global_state.is_auto_tracer_initialized():
         log.error(
@@ -534,6 +566,7 @@ def run_test_suite(
             before_evaluators_hook=before_evaluators_hook,
             max_test_case_concurrency=max_test_case_concurrency,
             grid_search_params=grid_search_params,
+            human_review_job=human_review_job,
         ),
         global_state.event_loop(),
     ).result()

From d8a64fe9d92f3de9a9680c8e0a457a0bc66b9ef4 Mon Sep 17 00:00:00 2001
From: Adam Nolte <ajnolte12@gmail.com>
Date: Wed, 25 Jun 2025 15:27:21 -0500
Subject: [PATCH 2/9] Add rubric id

---
 autoblocks/_impl/testing/models.py | 1 +
 autoblocks/_impl/testing/v2/run.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/autoblocks/_impl/testing/models.py b/autoblocks/_impl/testing/models.py
index 32ae2c0e..aa36e0fc 100644
--- a/autoblocks/_impl/testing/models.py
+++ b/autoblocks/_impl/testing/models.py
@@ -265,6 +265,7 @@ class CreateHumanReviewJob:
 
     assignee_email_address: Union[str, list[str]]
     name: str
+    rubric_id: Optional[str] = None
 
     def get_assignee_email_addresses(self) -> list[str]:
         if isinstance(self.assignee_email_address, str):
diff --git a/autoblocks/_impl/testing/v2/run.py b/autoblocks/_impl/testing/v2/run.py
index 698191bb..78db52d7 100644
--- a/autoblocks/_impl/testing/v2/run.py
+++ b/autoblocks/_impl/testing/v2/run.py
@@ -410,6 +410,7 @@ async def run_test_suite_for_grid_combo(
                         end_timestamp=end_timestamp,
                         assignee_email_address=assignee_email_address,
                         name=human_review_job.name,
+                        rubric_id=human_review_job.rubric_id,
                     )
                     for assignee_email_address in assignee_email_addresses
                 ]

From 688af32bdae3f4c67e822949173a8ffcd9783557 Mon Sep 17 00:00:00 2001
From: Adam Nolte <ajnolte12@gmail.com>
Date: Wed, 25 Jun 2025 15:37:04 -0500
Subject: [PATCH 3/9] Add some logs

---
 autoblocks/_impl/testing/v2/run.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/autoblocks/_impl/testing/v2/run.py b/autoblocks/_impl/testing/v2/run.py
index 78db52d7..94128ec1 100644
--- a/autoblocks/_impl/testing/v2/run.py
+++ b/autoblocks/_impl/testing/v2/run.py
@@ -401,7 +401,7 @@ async def run_test_suite_for_grid_combo(
     if human_review_job is not None:
         try:
             assignee_email_addresses = human_review_job.get_assignee_email_addresses()
-            await all_settled(
+            results = await all_settled(
                 [
                     send_create_human_review_job(
                         run_id=run_id,
@@ -415,6 +415,9 @@ async def run_test_suite_for_grid_combo(
                     for assignee_email_address in assignee_email_addresses
                 ]
             )
+            for result in results:
+                if isinstance(result, Exception):
+                    log.warn(f"Failed to create human review job for test run '{run_id}'", exc_info=result)
         except Exception as err:
             log.warn(f"Failed to create human review job for test run '{run_id}'", exc_info=err)
 

From edfca11311fe666e279a750baf646d34aa215947 Mon Sep 17 00:00:00 2001
From: Adam Nolte <ajnolte12@gmail.com>
Date: Wed, 25 Jun 2025 15:39:05 -0500
Subject: [PATCH 4/9] Add some logs

---
 autoblocks/_impl/testing/v2/api.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/autoblocks/_impl/testing/v2/api.py b/autoblocks/_impl/testing/v2/api.py
index 8967a757..b11080a6 100644
--- a/autoblocks/_impl/testing/v2/api.py
+++ b/autoblocks/_impl/testing/v2/api.py
@@ -28,6 +28,12 @@ async def post_to_api_with_retry(
         timeout=TIMEOUT_SECONDS,
         headers={"Authorization": f"Bearer {api_key}"},
     )
+    if not resp.is_success:
+        try:
+            error_body = resp.text
+            log.error(f"API request failed with status {resp.status_code} for {url}. Response body: {error_body}")
+        except Exception:
+            log.error(f"API request failed with status {resp.status_code} for {url}. Could not read response body.")
     resp.raise_for_status()
     return resp
 

From 7acc7be6bf6b255a11bce02de65f40267f0583c9 Mon Sep 17 00:00:00 2001
From: Adam Nolte <ajnolte12@gmail.com>
Date: Wed, 25 Jun 2025 15:43:19 -0500
Subject: [PATCH 5/9] Add z

---
 autoblocks/_impl/testing/v2/api.py | 9 ++-------
 autoblocks/_impl/testing/v2/run.py | 7 +++----
 2 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/autoblocks/_impl/testing/v2/api.py b/autoblocks/_impl/testing/v2/api.py
index b11080a6..c64e660f 100644
--- a/autoblocks/_impl/testing/v2/api.py
+++ b/autoblocks/_impl/testing/v2/api.py
@@ -28,12 +28,6 @@ async def post_to_api_with_retry(
         timeout=TIMEOUT_SECONDS,
         headers={"Authorization": f"Bearer {api_key}"},
     )
-    if not resp.is_success:
-        try:
-            error_body = resp.text
-            log.error(f"API request failed with status {resp.status_code} for {url}. Response body: {error_body}")
-        except Exception:
-            log.error(f"API request failed with status {resp.status_code} for {url}. Could not read response body.")
     resp.raise_for_status()
     return resp
 
@@ -46,9 +40,10 @@ async def post_to_api(
     if not api_key:
         raise ValueError(f"You must set the {AutoblocksEnvVar.V2_API_KEY} environment variable.")
 
+    url = f"{API_ENDPOINT_V2}{path}"
     async with global_state.test_run_api_semaphore():
         return await post_to_api_with_retry(
-            f"{API_ENDPOINT_V2}{path}",
+            url,
             api_key,
             json,
         )
diff --git a/autoblocks/_impl/testing/v2/run.py b/autoblocks/_impl/testing/v2/run.py
index 94128ec1..cd2abcd1 100644
--- a/autoblocks/_impl/testing/v2/run.py
+++ b/autoblocks/_impl/testing/v2/run.py
@@ -4,8 +4,6 @@
 import inspect
 import json
 import logging
-from datetime import datetime
-from datetime import timezone
 from typing import Any
 from typing import Awaitable
 from typing import Callable
@@ -46,6 +44,7 @@
 from autoblocks._impl.util import AutoblocksEnvVar
 from autoblocks._impl.util import all_settled
 from autoblocks._impl.util import cuid_generator
+from autoblocks._impl.util import now_iso_8601
 from autoblocks._impl.util import parse_autoblocks_overrides
 from autoblocks._impl.util import serialize_to_string
 
@@ -358,7 +357,7 @@ async def run_test_suite_for_grid_combo(
     human_review_job: Optional[CreateHumanReviewJob],
 ) -> None:
     run_id = cuid_generator()
-    start_timestamp = datetime.now(timezone.utc).isoformat()
+    start_timestamp = now_iso_8601() + "Z"  # add Z to be compatible with the API
 
     log.info(f"Running test suite '{test_id}' with {len(test_cases)} test cases")
     # Determine message with priority: unified overrides > legacy env var
@@ -397,7 +396,7 @@ async def run_test_suite_for_grid_combo(
         if test_run_reset_token:
             test_run_context_var.reset(test_run_reset_token)
 
-    end_timestamp = datetime.now(timezone.utc).isoformat()
+    end_timestamp = now_iso_8601() + "Z"  # add Z to be compatible with the API
     if human_review_job is not None:
         try:
             assignee_email_addresses = human_review_job.get_assignee_email_addresses()

From f12aae266cc415e2eac0d7f803977029158e8877 Mon Sep 17 00:00:00 2001
From: Adam Nolte <ajnolte12@gmail.com>
Date: Wed, 25 Jun 2025 15:45:46 -0500
Subject: [PATCH 6/9] Add log back

---
 autoblocks/_impl/testing/v2/api.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/autoblocks/_impl/testing/v2/api.py b/autoblocks/_impl/testing/v2/api.py
index c64e660f..1a68ea9d 100644
--- a/autoblocks/_impl/testing/v2/api.py
+++ b/autoblocks/_impl/testing/v2/api.py
@@ -28,6 +28,13 @@ async def post_to_api_with_retry(
         timeout=TIMEOUT_SECONDS,
         headers={"Authorization": f"Bearer {api_key}"},
     )
+    if not resp.is_success:
+        try:
+            error_body = resp.text
+            log.error(f"API request failed with status {resp.status_code} for {url}. Response body: {error_body}")
+        except Exception:
+            log.error(f"API request failed with status {resp.status_code} for {url}. Could not read response body.")
+
     resp.raise_for_status()
     return resp
 

From 2083c2f8c769cd0ef81f261f59875b0d23b80387 Mon Sep 17 00:00:00 2001
From: Adam Nolte <ajnolte12@gmail.com>
Date: Wed, 25 Jun 2025 15:51:18 -0500
Subject: [PATCH 7/9] fix format

---
 autoblocks/_impl/testing/v2/api.py | 1 +
 autoblocks/_impl/testing/v2/run.py | 6 +++---
 autoblocks/_impl/util.py           | 9 +++++++++
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/autoblocks/_impl/testing/v2/api.py b/autoblocks/_impl/testing/v2/api.py
index 1a68ea9d..10b41e9d 100644
--- a/autoblocks/_impl/testing/v2/api.py
+++ b/autoblocks/_impl/testing/v2/api.py
@@ -65,6 +65,7 @@ async def send_create_human_review_job(
     app_slug: str,
     rubric_id: Optional[str] = None,
 ) -> None:
+
     await post_to_api(
         f"/apps/{app_slug}/human-review/jobs",
         json=dict(
diff --git a/autoblocks/_impl/testing/v2/run.py b/autoblocks/_impl/testing/v2/run.py
index cd2abcd1..9335a264 100644
--- a/autoblocks/_impl/testing/v2/run.py
+++ b/autoblocks/_impl/testing/v2/run.py
@@ -44,7 +44,7 @@
 from autoblocks._impl.util import AutoblocksEnvVar
 from autoblocks._impl.util import all_settled
 from autoblocks._impl.util import cuid_generator
-from autoblocks._impl.util import now_iso_8601
+from autoblocks._impl.util import now_rfc3339
 from autoblocks._impl.util import parse_autoblocks_overrides
 from autoblocks._impl.util import serialize_to_string
 
@@ -357,7 +357,7 @@ async def run_test_suite_for_grid_combo(
     human_review_job: Optional[CreateHumanReviewJob],
 ) -> None:
     run_id = cuid_generator()
-    start_timestamp = now_iso_8601() + "Z"  # add Z to be compatible with the API
+    start_timestamp = now_rfc3339()
 
     log.info(f"Running test suite '{test_id}' with {len(test_cases)} test cases")
     # Determine message with priority: unified overrides > legacy env var
@@ -396,7 +396,7 @@ async def run_test_suite_for_grid_combo(
         if test_run_reset_token:
             test_run_context_var.reset(test_run_reset_token)
 
-    end_timestamp = now_iso_8601() + "Z"  # add Z to be compatible with the API
+    end_timestamp = now_rfc3339()
     if human_review_job is not None:
         try:
             assignee_email_addresses = human_review_job.get_assignee_email_addresses()
diff --git a/autoblocks/_impl/util.py b/autoblocks/_impl/util.py
index 8c69f782..b74f9736 100644
--- a/autoblocks/_impl/util.py
+++ b/autoblocks/_impl/util.py
@@ -145,6 +145,15 @@ def now_iso_8601() -> str:
     return datetime.now(timezone.utc).isoformat()
 
 
+def now_rfc3339() -> str:
+    """
+    Returns the current UTC timestamp in format.
+    RFC 3339 is a profile of ISO 8601 commonly used in APIs.
+    Format: 2025-06-25T20:47:46.429Z
+    """
+    return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
 def is_cli_running() -> bool:
     return AutoblocksEnvVar.CLI_SERVER_ADDRESS.get() is not None
 

From 9a3b41236cc50d5c25c8e5d5c5f50065a630e331 Mon Sep 17 00:00:00 2001
From: Adam Nolte <ajnolte12@gmail.com>
Date: Wed, 25 Jun 2025 15:53:43 -0500
Subject: [PATCH 8/9] fix format

---
 autoblocks/_impl/testing/v2/api.py |  5 +++--
 autoblocks/_impl/testing/v2/run.py | 25 ++++++++-----------------
 2 files changed, 11 insertions(+), 19 deletions(-)

diff --git a/autoblocks/_impl/testing/v2/api.py b/autoblocks/_impl/testing/v2/api.py
index 10b41e9d..3260efb9 100644
--- a/autoblocks/_impl/testing/v2/api.py
+++ b/autoblocks/_impl/testing/v2/api.py
@@ -1,5 +1,6 @@
 import logging
 from typing import Any
+from typing import List
 from typing import Optional
 
 from httpx import Response
@@ -60,7 +61,7 @@ async def send_create_human_review_job(
     run_id: str,
     start_timestamp: str,
     end_timestamp: str,
-    assignee_email_address: str,
+    assignee_email_addresses: List[str],
     name: str,
     app_slug: str,
     rubric_id: Optional[str] = None,
@@ -73,7 +74,7 @@ async def send_create_human_review_job(
             startTimestamp=start_timestamp,
             endTimestamp=end_timestamp,
             rubricId=rubric_id,
-            assigneeEmailAddress=assignee_email_address,
+            assigneeEmailAddresses=assignee_email_addresses,
             name=name,
         ),
     )
diff --git a/autoblocks/_impl/testing/v2/run.py b/autoblocks/_impl/testing/v2/run.py
index 9335a264..c14cb1a9 100644
--- a/autoblocks/_impl/testing/v2/run.py
+++ b/autoblocks/_impl/testing/v2/run.py
@@ -399,24 +399,15 @@ async def run_test_suite_for_grid_combo(
     end_timestamp = now_rfc3339()
     if human_review_job is not None:
         try:
-            assignee_email_addresses = human_review_job.get_assignee_email_addresses()
-            results = await all_settled(
-                [
-                    send_create_human_review_job(
-                        run_id=run_id,
-                        app_slug=app_slug,
-                        start_timestamp=start_timestamp,
-                        end_timestamp=end_timestamp,
-                        assignee_email_address=assignee_email_address,
-                        name=human_review_job.name,
-                        rubric_id=human_review_job.rubric_id,
-                    )
-                    for assignee_email_address in assignee_email_addresses
-                ]
+            await send_create_human_review_job(
+                run_id=run_id,
+                app_slug=app_slug,
+                start_timestamp=start_timestamp,
+                end_timestamp=end_timestamp,
+                assignee_email_addresses=human_review_job.get_assignee_email_addresses(),
+                name=human_review_job.name,
+                rubric_id=human_review_job.rubric_id,
             )
-            for result in results:
-                if isinstance(result, Exception):
-                    log.warn(f"Failed to create human review job for test run '{run_id}'", exc_info=result)
         except Exception as err:
             log.warn(f"Failed to create human review job for test run '{run_id}'", exc_info=err)
 

From 3adbabca7d181c95c16cd40a435dc1d24b5a84ac Mon Sep 17 00:00:00 2001
From: Adam Nolte <ajnolte12@gmail.com>
Date: Wed, 25 Jun 2025 16:35:51 -0500
Subject: [PATCH 9/9] PR feedback

---
 autoblocks/_impl/util.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/autoblocks/_impl/util.py b/autoblocks/_impl/util.py
index b74f9736..34bc4b37 100644
--- a/autoblocks/_impl/util.py
+++ b/autoblocks/_impl/util.py
@@ -147,7 +147,7 @@ def now_iso_8601() -> str:
 
 def now_rfc3339() -> str:
     """
-    Returns the current UTC timestamp in format.
+    Returns the current UTC timestamp in RFC 3339 format.
     RFC 3339 is a profile of ISO 8601 commonly used in APIs.
     Format: 2025-06-25T20:47:46.429Z
     """