Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 64 additions & 7 deletions scripts/cloud_run_runtime_guard.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ def _load_services() -> list[str]:
for target in targets:
if not isinstance(target, dict):
continue
if not _target_enabled(target):
continue
runtime_target = target.get("runtime_target") or target.get(
"runtime_target_json"
)
Expand Down Expand Up @@ -85,19 +87,43 @@ def _load_services() -> list[str]:
return unique


def _service_job_aliases(service: str) -> list[str]:
service_name = str(service or "").strip()
if not service_name:
return []
aliases = [service_name]
if service_name.endswith("-service"):
aliases.append(service_name.removesuffix("-service"))
return list(dict.fromkeys(aliases))


def _scheduler_job_pattern_for_services(services: list[str]) -> str:
candidates: list[str] = []
for service in services:
service_name = str(service or "").strip()
if not service_name:
continue
candidates.append(service_name)
if service_name.endswith("-service"):
candidates.append(service_name.removesuffix("-service"))
candidates.extend(_service_job_aliases(service))
unique = list(dict.fromkeys(candidates))
return "|".join(re.escape(candidate) for candidate in unique)


def _entry_job_name(entry: dict[str, Any]) -> str:
labels = _labels(entry)
return str(labels.get("job_id") or labels.get("job_name") or "")


def _scheduler_entry_since(
entry: dict[str, Any],
service_since_by_name: dict[str, dt.datetime],
fallback: dt.datetime,
) -> dt.datetime:
job_name = _entry_job_name(entry)
matches = [
service_since
for service, service_since in service_since_by_name.items()
if any(alias and alias in job_name for alias in _service_job_aliases(service))
]
Comment on lines +119 to +123

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Use exact scheduler job matches for revision windows

When two monitored services share a prefix, this substring check can associate a scheduler log with the wrong service window. For example, with foo-service and foo-bar-service, the foo-bar-scheduler job contains the foo alias, so max(matches) may pick foo-service's later ready-revision time and drop a real foo-bar scheduler failure as stale. Match against the expected scheduler job names or otherwise choose the service-specific alias unambiguously before filtering failures.

Useful? React with 👍 / 👎.

return max(matches) if matches else fallback


def _run_gcloud(args: list[str]) -> subprocess.CompletedProcess[str]:
return subprocess.run(args, text=True, capture_output=True, check=False)

Expand Down Expand Up @@ -184,6 +210,27 @@ def _runtime_target(target: dict[str, Any]) -> dict[str, Any]:
return runtime_target if isinstance(runtime_target, dict) else {}


def _coerce_bool(value: Any, default: bool) -> bool:
if value is None:
return default
if isinstance(value, bool):
return value
text = str(value).strip().lower()
if not text:
return default
return text in {"1", "true", "yes", "y", "on"}
Comment on lines +218 to +221

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Reject invalid enabled flags instead of disabling targets

When a target has a non-empty RUNTIME_TARGET_ENABLED value outside the accepted boolean spellings, this returns False, so a typo such as treu silently removes that Cloud Run service from runtime-guard monitoring. Other configuration paths treat this flag as enabled by default and fail invalid values (scripts/build_cloud_run_env_sync_plan.py:378-386), so this should reject or ignore unknown values instead of suppressing alerts.

Useful? React with 👍 / 👎.



def _target_enabled(target: dict[str, Any]) -> bool:
runtime_target = _runtime_target(target)
for key in ("runtime_target_enabled", "RUNTIME_TARGET_ENABLED"):
if key in target:
return _coerce_bool(target.get(key), True)
if key in runtime_target:
return _coerce_bool(runtime_target.get(key), True)
Comment on lines +226 to +230

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Honor env/defaults disabled flags

When the targets JSON uses the same shape as env sync, RUNTIME_TARGET_ENABLED may come from target.env, defaults, or defaults.env, but this helper only checks the top-level target and nested runtime target. In that case env sync deploys the service with RUNTIME_TARGET_ENABLED=false while the runtime guard still loads and alerts on it, so disabled targets are not actually ignored for that supported configuration shape.

Useful? React with 👍 / 👎.

return True


def _target_service_names(target: dict[str, Any]) -> list[str]:
runtime_target = _runtime_target(target)
for key in ("service", "service_name", "cloud_run_service"):
Expand Down Expand Up @@ -428,6 +475,7 @@ def main() -> int:
issues: list[str] = []
details: list[str] = []
success_count = 0
service_since_by_name: dict[str, dt.datetime] = {}

try:
services = _load_services()
Expand All @@ -441,6 +489,7 @@ def main() -> int:

for service in services:
service_since = _cloud_run_log_since(project, service, since) if ignore_pre_ready_logs else since
service_since_by_name[service] = service_since
service_since_text = _format_timestamp(service_since)
log_filter = (
'resource.type="cloud_run_revision" '
Expand Down Expand Up @@ -474,7 +523,15 @@ def main() -> int:
for entry in entries
if regex.search(str(_labels(entry).get("job_id") or _labels(entry).get("job_name") or ""))
]
failures = [entry for entry in entries if _is_failure(entry)]
failures = []
for entry in entries:
if not _is_failure(entry):
continue
entry_timestamp = _parse_timestamp(entry.get("timestamp"))
entry_since = _scheduler_entry_since(entry, service_since_by_name, since)
if entry_timestamp and entry_timestamp < entry_since:
continue
failures.append(entry)
if failures:
issues.append(f"{len(failures)} Cloud Scheduler failure log(s)")
details.extend(_summarize(entry) for entry in failures[:5])
Expand Down
35 changes: 35 additions & 0 deletions tests/test_cloud_run_runtime_guard.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,3 +108,38 @@ def test_region_for_service_prefers_target_region(monkeypatch):
)

assert guard._region_for_service("interactive-brokers-live-u1599-tqqq-service") == "asia-east1"


def test_load_services_ignores_disabled_runtime_targets(monkeypatch):
monkeypatch.delenv("RUNTIME_GUARD_CLOUD_RUN_SERVICES", raising=False)
monkeypatch.delenv("CLOUD_RUN_SERVICES", raising=False)
monkeypatch.delenv("CLOUD_RUN_SERVICE", raising=False)
monkeypatch.setenv(
"CLOUD_RUN_SERVICE_TARGETS_JSON",
json.dumps(
{
"targets": [
{"service": "enabled-service", "RUNTIME_TARGET_ENABLED": "true"},
{"service": "disabled-service", "RUNTIME_TARGET_ENABLED": "false"},
{"service": "disabled-lower-service", "runtime_target_enabled": "false"},
]
}
),
)

assert guard._load_services() == ["enabled-service"]


def test_scheduler_entry_since_uses_matching_service_revision_window():
fallback = dt.datetime(2026, 7, 1, 1, 0, tzinfo=dt.timezone.utc)
service_since = dt.datetime(2026, 7, 1, 2, 0, tzinfo=dt.timezone.utc)
entry = {"resource": {"labels": {"job_id": "enabled-service-scheduler"}}}

assert (
guard._scheduler_entry_since(entry, {"enabled-service": service_since}, fallback)
== service_since
)
assert (
guard._scheduler_entry_since(entry, {"other-service": service_since}, fallback)
== fallback
)
Loading