From d183e17d8a0899f7a7525752f4bb2ff6aa87e840 Mon Sep 17 00:00:00 2001
From: lokic233 <>
Date: Fri, 19 Jun 2026 23:47:27 -0700
Subject: [PATCH] Normalize GKE workload createTime to RFC 3339 before sending
 to control plane

machinelearning_run() forwards workload_details["creation-timestamp"] verbatim
into gke_workload_details["createTime"], which the control plane validates as a
google.protobuf.Timestamp. If the injected GKE_DIAGON_METADATA creation-timestamp
is not strict RFC 3339, the create request fails with HTTP 400 INVALID_ARGUMENT:
"Illegal timestamp format; timestamps must end with 'Z' or have a valid timezone
offset." This blocks MLRun creation.

Add a best-effort _normalize_rfc3339() helper and apply it at the createTime
assignment, so timestamps lacking a 'Z'/offset are normalized to UTC RFC 3339.
Unparseable values pass through unchanged (the control plane still validates).

Reproduced on google-cloud-mldiagnostics 1.0.2/1.0.3, GKE TPU, single-controller
Pathways: the 400 cleared once createTime was RFC 3339, and the MLRun was created.

Note: the SDK's own host_utils.get_identifier formats timestamps as
"%Y%m%d-%H%M%S" (no 'Z'), so the verbatim pass-through is a latent issue even for
the SDK's intended inputs.

Reported and authored via Navi on behalf of @lokic233 (dengcchi, Meta MRS-CE).
---
 .../clients/control_plane_client.py           | 41 ++++++++++++++++++-
 1 file changed, 40 insertions(+), 1 deletion(-)

diff --git a/src/google_cloud_mldiagnostics/clients/control_plane_client.py b/src/google_cloud_mldiagnostics/clients/control_plane_client.py
index 0b860a4..5885e5a 100644
--- a/src/google_cloud_mldiagnostics/clients/control_plane_client.py
+++ b/src/google_cloud_mldiagnostics/clients/control_plane_client.py
@@ -15,6 +15,7 @@
 """Client for sending requests to Diagon Control Plane."""
 
 import ast
+import datetime
 import logging
 import pprint
 import random
@@ -30,6 +31,42 @@
 
 logger = logging.getLogger(__name__)
 _MAX_RETRIES = 3
+
+
+def _normalize_rfc3339(timestamp: str) -> str:
+  """Normalizes a timestamp to RFC 3339 with a trailing 'Z' (UTC).
+
+  The GKE workload ``createTime`` is sent to the control plane as a
+  ``google.protobuf.Timestamp``, which is rejected with HTTP 400
+  INVALID_ARGUMENT ("timestamps must end with 'Z' or have a valid timezone
+  offset") if it is not strict RFC 3339. The source ``creation-timestamp`` comes
+  from the injected ``GKE_DIAGON_METADATA`` env and is not guaranteed to be
+  RFC 3339, so normalize it here before sending. Best effort: if the value
+  cannot be parsed it is returned unchanged (the control plane still validates).
+  """
+  if not timestamp:
+    return timestamp
+  try:
+    ts = timestamp.strip()
+    # Python < 3.11 fromisoformat rejects a trailing 'Z'.
+    if ts.endswith("Z"):
+      ts = ts[:-1] + "+00:00"
+    parsed = datetime.datetime.fromisoformat(ts)
+    if parsed.tzinfo is None:
+      parsed = parsed.replace(tzinfo=datetime.timezone.utc)
+    return (
+        parsed.astimezone(datetime.timezone.utc)
+        .strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3]
+        + "Z"
+    )
+  except (ValueError, TypeError):
+    logger.warning(
+        "Could not normalize createTime %r to RFC 3339; sending as-is.",
+        timestamp,
+    )
+    return timestamp
+
+
 _ERROR_CODE_ALREADY_EXISTS = 6
 
 
@@ -258,7 +295,9 @@ def create_ml_run(
           gke_workload_details["labels"] = workload_details["labels"]
         creation_timestamp = workload_details.get("creation-timestamp")
         if creation_timestamp:
-          gke_workload_details["createTime"] = creation_timestamp
+          gke_workload_details["createTime"] = _normalize_rfc3339(
+              creation_timestamp
+          )
         payload["workloadDetails"] = {"gke": gke_workload_details}
 
     # Sanitize the name for machineLearningRunId