diff --git a/autoblocks/_impl/testing/api.py b/autoblocks/_impl/testing/api.py index 0be14996..36200855 100644 --- a/autoblocks/_impl/testing/api.py +++ b/autoblocks/_impl/testing/api.py @@ -211,7 +211,7 @@ async def send_test_events( # Remove the events from the test_events dict after they have been sent del test_events[(run_id, test_case_hash)] except Exception as e: - log.warn(f"Failed to send test events for run '{run_id}' and test case hash '{test_case_hash}'", exc_info=e) + log.warning(f"Failed to send test events for run '{run_id}' and test case hash '{test_case_hash}'", exc_info=e) async def send_test_case_result( @@ -280,7 +280,7 @@ async def send_test_case_result( ) for result in results: if isinstance(result, Exception): - log.warn( + log.warning( "Failed to send part of the test case results to Autoblocks\n" f"test case hash: {test_case_ctx.hash()}\n" f"{result}", @@ -296,7 +296,7 @@ async def send_test_case_result( ), ) except Exception as e: - log.warn( + log.warning( "Failed to send human review fields to Autoblocks\n" f"test case hash: {test_case_ctx.hash()}\n", exc_info=e, ) @@ -304,7 +304,7 @@ async def send_test_case_result( try: await post_to_api(f"/runs/{run_id}/results/{result_id}/ui-based-evaluations", json={}) except Exception as e: - log.warn("Failed to run ui based evaluations\n" f"test case hash: {test_case_ctx.hash()}\n", exc_info=e) + log.warning("Failed to run ui based evaluations\n" f"test case hash: {test_case_ctx.hash()}\n", exc_info=e) return result_id @@ -379,7 +379,7 @@ async def send_slack_notification( json=dict(slackWebhookUrl=slack_webhook_url), ) except Exception as e: - log.warn(f"Failed to send slack notification for test run '{run_id}'", exc_info=e) + log.warning(f"Failed to send slack notification for test run '{run_id}'", exc_info=e) async def send_github_comment() -> None: @@ -396,7 +396,7 @@ async def send_github_comment() -> None: json=dict(githubToken=github_token), ) except Exception as e: - log.warn( + log.warning( "Could not create GitHub comment for build '{build_id}'." "For more information on how to set up GitHub Actions permissions, see: " "https://docs.autoblocks.ai/testing/ci#git-hub-comments-github-actions-permissions", diff --git a/autoblocks/_impl/testing/run.py b/autoblocks/_impl/testing/run.py index a7c3b36e..c5947699 100644 --- a/autoblocks/_impl/testing/run.py +++ b/autoblocks/_impl/testing/run.py @@ -481,7 +481,7 @@ async def run_test_suite_for_grid_combo( ] ) except Exception as err: - log.warn(f"Failed to create human review job for test run '{run_id}'", exc_info=err) + log.warning(f"Failed to create human review job for test run '{run_id}'", exc_info=err) await all_settled( [ diff --git a/autoblocks/_impl/testing/v2/api.py b/autoblocks/_impl/testing/v2/api.py index 3260efb9..1a07694f 100644 --- a/autoblocks/_impl/testing/v2/api.py +++ b/autoblocks/_impl/testing/v2/api.py @@ -11,6 +11,10 @@ from autoblocks._impl import global_state from autoblocks._impl.config.constants import API_ENDPOINT_V2 from autoblocks._impl.util import AutoblocksEnvVar +from autoblocks._impl.util import ThirdPartyEnvVar +from autoblocks._impl.util import is_ci +from autoblocks._impl.util import is_cli_running +from autoblocks._impl.util import is_github_comment_disabled log = logging.getLogger(__name__) @@ -78,3 +82,47 @@ async def send_create_human_review_job( name=name, ), ) + + +async def send_v2_slack_notification(run_id: str, app_slug: str) -> None: + """ + V2 Slack notification wrapper that delegates to V1 logic for maximum code reuse. + Uses V2 app-based endpoint: /apps/{app_slug}/runs/{run_id}/slack-notification + """ + slack_webhook_url = AutoblocksEnvVar.SLACK_WEBHOOK_URL.get() + if is_cli_running() or not slack_webhook_url or not is_ci(): + return + + log.info(f"Sending slack notification for test run '{run_id}' in app '{app_slug}'.") + try: + await post_to_api( + f"/apps/{app_slug}/runs/{run_id}/slack-notification", + json=dict(slackWebhookUrl=slack_webhook_url), + ) + except Exception as e: + log.warning(f"Failed to send slack notification for test run '{run_id}' in app '{app_slug}'", exc_info=e) + + +async def send_v2_github_comment(app_slug: str, build_id: str) -> None: + """ + V2 GitHub comment wrapper that delegates to V1 logic for maximum code reuse. + Uses V2 app-based endpoint: /apps/{app_slug}/builds/{build_id}/github-comment + """ + github_token = ThirdPartyEnvVar.GITHUB_TOKEN.get() + if is_cli_running() or is_github_comment_disabled() or not github_token or not build_id or not is_ci(): + return + + log.info(f"Creating GitHub comment for build '{build_id}' in app '{app_slug}'.") + try: + async with global_state.github_comment_semaphore(): + await post_to_api( + f"/apps/{app_slug}/builds/{build_id}/github-comment", + json=dict(githubToken=github_token), + ) + except Exception as e: + log.warning( + f"Could not create GitHub comment for build '{build_id}' in app '{app_slug}'. " + "For more information on how to set up GitHub Actions permissions, see: " + "https://docs.autoblocks.ai/testing/ci#git-hub-comments-github-actions-permissions", + exc_info=e, + ) diff --git a/autoblocks/_impl/testing/v2/run.py b/autoblocks/_impl/testing/v2/run.py index 40ef8b32..859ac986 100644 --- a/autoblocks/_impl/testing/v2/run.py +++ b/autoblocks/_impl/testing/v2/run.py @@ -45,6 +45,8 @@ from autoblocks._impl.testing.util import yield_grid_search_param_combos from autoblocks._impl.testing.util import yield_test_case_contexts_from_test_cases from autoblocks._impl.testing.v2.api import send_create_human_review_job +from autoblocks._impl.testing.v2.api import send_v2_github_comment +from autoblocks._impl.testing.v2.api import send_v2_slack_notification from autoblocks._impl.tracer.util import SpanAttribute from autoblocks._impl.util import AutoblocksEnvVar from autoblocks._impl.util import all_settled @@ -452,7 +454,29 @@ async def run_test_suite_for_grid_combo( rubric_id=human_review_job.rubric_id, ) except Exception as err: - log.warn(f"Failed to create human review job for test run '{run_id}'", exc_info=err) + log.warning(f"Failed to create human review job for test run '{run_id}'", exc_info=err) + + # Send V2 notifications after test completion + # Use all_settled pattern to ensure notification failures don't affect test results + build_id = AutoblocksEnvVar.V2_CI_TEST_RUN_BUILD_ID.get() + log.debug( + f"Starting V2 notification dispatch for test run '{run_id}' with app_slug '{app_slug}', build_id '{build_id}'" + ) + + try: + notifications = [ + send_v2_slack_notification(run_id=run_id, app_slug=app_slug), + ] + if build_id: + notifications.append(send_v2_github_comment(app_slug=app_slug, build_id=build_id)) + + await all_settled(notifications) + log.debug(f"V2 notification dispatch completed for test run '{run_id}' with app_slug '{app_slug}'") + except Exception as e: + log.debug( + f"V2 notification dispatch encountered error for test run '{run_id}' with app_slug '{app_slug}': {e}", + exc_info=e, + ) async def async_run_test_suite(