diff --git a/deno.json b/deno.json index 715f693..d97b404 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,7 @@ { "workspace": [ "generation", + "tests", "lib" ] } diff --git a/deno.lock b/deno.lock index fae42f2..64b3631 100644 --- a/deno.lock +++ b/deno.lock @@ -3,7 +3,9 @@ "specifiers": { "jsr:@cloudydeno/kubernetes-client@0.8": "0.8.0", "jsr:@cloudydeno/stream-observables@1": "1.4.1", + "jsr:@std/assert@^1.0.18": "1.0.18", "jsr:@std/internal@^1.0.10": "1.0.10", + "jsr:@std/internal@^1.0.12": "1.0.12", "jsr:@std/path@1": "1.1.2", "jsr:@std/streams@1": "1.0.12", "jsr:@std/yaml@1": "1.0.9" @@ -20,13 +22,22 @@ "@cloudydeno/stream-observables@1.4.1": { "integrity": "555aa3d786fcfd066d096fc943ea47afc688618e95146e615705e981008f57c0" }, + "@std/assert@1.0.18": { + "integrity": "270245e9c2c13b446286de475131dc688ca9abcd94fc5db41d43a219b34d1c78", + "dependencies": [ + "jsr:@std/internal@^1.0.12" + ] + }, "@std/internal@1.0.10": { "integrity": "e3be62ce42cab0e177c27698e5d9800122f67b766a0bea6ca4867886cbde8cf7" }, + "@std/internal@1.0.12": { + "integrity": "972a634fd5bc34b242024402972cd5143eac68d8dffaca5eaa4dba30ce17b027" + }, "@std/path@1.1.2": { "integrity": "c0b13b97dfe06546d5e16bf3966b1cadf92e1cc83e56ba5476ad8b498d9e3038", "dependencies": [ - "jsr:@std/internal" + "jsr:@std/internal@^1.0.10" ] }, "@std/streams@1.0.12": { @@ -37,35 +48,7 @@ } }, "remote": { - "https://deno.land/std@0.177.0/streams/text_line_stream.ts": "a9dd2636c6b90e626e19df26c97034c5f638bdd957cbd5c531d6278fe1d08e90", - "https://deno.land/std@0.198.0/_util/diff.ts": "1a3c044aedf77647d6cac86b798c6417603361b66b54c53331b312caeb447aea", - "https://deno.land/std@0.198.0/assert/_constants.ts": "8a9da298c26750b28b326b297316cdde860bc237533b07e1337c021379e6b2a9", - "https://deno.land/std@0.198.0/assert/_format.ts": "a69126e8a469009adf4cf2a50af889aca364c349797e63174884a52ff75cf4c7", - "https://deno.land/std@0.198.0/assert/assert.ts": "9a97dad6d98c238938e7540736b826440ad8c1c1e54430ca4c4e623e585607ee", - "https://deno.land/std@0.198.0/assert/assert_almost_equals.ts": "e15ca1f34d0d5e0afae63b3f5d975cbd18335a132e42b0c747d282f62ad2cd6c", - "https://deno.land/std@0.198.0/assert/assert_array_includes.ts": "6856d7f2c3544bc6e62fb4646dfefa3d1df5ff14744d1bca19f0cbaf3b0d66c9", - "https://deno.land/std@0.198.0/assert/assert_equals.ts": "a0ee60574e437bcab2dcb79af9d48dc88845f8fd559468d9c21b15fd638ef943", - "https://deno.land/std@0.198.0/assert/assert_exists.ts": "407cb6b9fb23a835cd8d5ad804e2e2edbbbf3870e322d53f79e1c7a512e2efd7", - "https://deno.land/std@0.198.0/assert/assert_false.ts": "a9962749f4bf5844e3fa494257f1de73d69e4fe0e82c34d0099287552163a2dc", - "https://deno.land/std@0.198.0/assert/assert_instance_of.ts": "09fd297352a5b5bbb16da2b5e1a0d8c6c44da5447772648622dcc7df7af1ddb8", - "https://deno.land/std@0.198.0/assert/assert_is_error.ts": "b4eae4e5d182272efc172bf28e2e30b86bb1650cd88aea059e5d2586d4160fb9", - "https://deno.land/std@0.198.0/assert/assert_match.ts": "c4083f80600bc190309903c95e397a7c9257ff8b5ae5c7ef91e834704e672e9b", - "https://deno.land/std@0.198.0/assert/assert_not_equals.ts": "9f1acab95bd1f5fc9a1b17b8027d894509a745d91bac1718fdab51dc76831754", - "https://deno.land/std@0.198.0/assert/assert_not_instance_of.ts": "0c14d3dfd9ab7a5276ed8ed0b18c703d79a3d106102077ec437bfe7ed912bd22", - "https://deno.land/std@0.198.0/assert/assert_not_match.ts": "3796a5b0c57a1ce6c1c57883dd4286be13a26f715ea662318ab43a8491a13ab0", - "https://deno.land/std@0.198.0/assert/assert_not_strict_equals.ts": "ca6c6d645e95fbc873d25320efeb8c4c6089a9a5e09f92d7c1c4b6e935c2a6ad", - "https://deno.land/std@0.198.0/assert/assert_object_match.ts": "d8fc2867cfd92eeacf9cea621e10336b666de1874a6767b5ec48988838370b54", - "https://deno.land/std@0.198.0/assert/assert_rejects.ts": "45c59724de2701e3b1f67c391d6c71c392363635aad3f68a1b3408f9efca0057", - "https://deno.land/std@0.198.0/assert/assert_strict_equals.ts": "5cf29b38b3f8dece95287325723272aa04e04dbf158d886d662fa594fddc9ed3", - "https://deno.land/std@0.198.0/assert/assert_string_includes.ts": "b821d39ebf5cb0200a348863c86d8c4c4b398e02012ce74ad15666fc4b631b0c", - "https://deno.land/std@0.198.0/assert/assert_throws.ts": "63784e951475cb7bdfd59878cd25a0931e18f6dc32a6077c454b2cd94f4f4bcd", - "https://deno.land/std@0.198.0/assert/assertion_error.ts": "4d0bde9b374dfbcbe8ac23f54f567b77024fb67dbb1906a852d67fe050d42f56", - "https://deno.land/std@0.198.0/assert/equal.ts": "9f1a46d5993966d2596c44e5858eec821859b45f783a5ee2f7a695dfc12d8ece", - "https://deno.land/std@0.198.0/assert/fail.ts": "c36353d7ae6e1f7933d45f8ea51e358c8c4b67d7e7502028598fe1fea062e278", - "https://deno.land/std@0.198.0/assert/mod.ts": "08d55a652c22c5da0215054b21085cec25a5da47ce4a6f9de7d9ad36df35bdee", - "https://deno.land/std@0.198.0/assert/unimplemented.ts": "d56fbeecb1f108331a380f72e3e010a1f161baa6956fd0f7cf3e095ae1a4c75a", - "https://deno.land/std@0.198.0/assert/unreachable.ts": "4600dc0baf7d9c15a7f7d234f00c23bca8f3eba8b140286aaca7aa998cf9a536", - "https://deno.land/std@0.198.0/fmt/colors.ts": "a7eecffdf3d1d54db890723b303847b6e0a1ab4b528ba6958b8f2e754cf1b3bc" + "https://deno.land/std@0.177.0/streams/text_line_stream.ts": "a9dd2636c6b90e626e19df26c97034c5f638bdd957cbd5c531d6278fe1d08e90" }, "workspace": { "members": { @@ -80,6 +63,11 @@ "dependencies": [ "jsr:@cloudydeno/kubernetes-client@0.8" ] + }, + "tests": { + "dependencies": [ + "jsr:@std/assert@^1.0.18" + ] } } } diff --git a/lib/contrib/core-v1-pod-health.ts b/lib/contrib/core-v1-pod-health.ts new file mode 100644 index 0000000..5b19ab1 --- /dev/null +++ b/lib/contrib/core-v1-pod-health.ts @@ -0,0 +1,169 @@ +import type { Pod, Container, PodCondition } from "../builtin/core@v1/structs.ts"; + +/** + * Extra utility to assist with representing Pod health e.g. to a user. + * @module + */ + +/** A judgement of a Pod's latest health. */ +export type PodHealth = { + /** A string representation of ready vs. total containers e.g. "1/1", "0/2" (kubectl READY column) */ + readyCount: string; + /** A flexible string summarizing pod status e.g. "Running", "CrashLoopBackOff" (kubectl STATUS column) */ + reason: string; + /** Number of observed container restarts (main part of kubectl RESTARTS column) */ + restarts: number; + /** Date of latest container restart, if any (added on to kubectl RESTARTS column) */ + lastRestartDate: Date | undefined; +}; + +/** + * Determines the overall health of a given Pod. + * This function evaluates health using the same logic as `kubectl get pods`. + * @param pod A Pod object retrieved from Kubernetes API. + * @returns Values for kubectl's READY, STATUS, and RESTARTS columns. + */ +export function calculatePodHealth(pod: Pod): PodHealth { + + // Grabs a particular condition off the pod status + function getCondition(conditionType: string): PodCondition | undefined { + return pod.status?.conditions?.find( + condition => condition.type == conditionType); + } + + // Following function body is generally a port of: + // https://github.com/kubernetes/kubernetes/blob/v1.35.1/pkg/printers/internalversion/printers.go#L899 + + let restarts = 0; + let restartableInitContainerRestarts = 0; + let totalContainers = pod.spec?.containers.length ?? 0; + let readyContainers = 0; + let lastRestartDate = new Date(0); + let lastRestartableInitContainerRestartDate = new Date(0); + + const podPhase = pod.status?.phase; + let reason = pod.status?.reason || podPhase || 'Unknown'; + + // If the Pod carries {type:PodScheduled, reason:SchedulingGated}, set reason to 'SchedulingGated'. + if (getCondition('PodScheduled')?.reason == 'SchedulingGated') { + reason = 'SchedulingGated'; + } + + const initContainers = new Map() + for (const container of pod.spec?.initContainers ?? []) { + initContainers.set(container.name, container); + if (container.restartPolicy == 'Always') { + totalContainers++; + } + } + + let initializing = false; + const statuses = pod.status?.initContainerStatuses ?? []; + initContainerLoop: for (const [i, container] of statuses.entries()) { + restarts += container.restartCount ?? 0; + if (container.lastState?.terminated) { + const terminatedDate = container.lastState.terminated.finishedAt; + if (terminatedDate && lastRestartDate < terminatedDate) { + lastRestartDate = terminatedDate; + } + } + if (initContainers.get(container.name)?.restartPolicy == 'Always') { + restartableInitContainerRestarts += container.restartCount + if (container.lastState?.terminated) { + const terminatedDate = container.lastState.terminated.finishedAt; + if (terminatedDate && lastRestartableInitContainerRestartDate < terminatedDate) { + lastRestartableInitContainerRestartDate = terminatedDate; + } + } + } + switch (true) { + case container.state?.terminated?.exitCode == 0: + continue initContainerLoop; + case initContainers.get(container.name)?.restartPolicy == 'Always' && container.started: + if (container.ready) { + readyContainers++; + } + continue initContainerLoop; + case container.state?.terminated != null: + // initialization is failed + if (!container.state?.terminated.reason) { + if (container.state?.terminated.signal != 0) { + reason = `Init:Signal:${container.state?.terminated.signal}`; + } else { + reason = `Init:ExitCode:${container.state?.terminated.exitCode}`; + } + } else { + reason = `Init:${container.state?.terminated.reason}`; + } + initializing = true; + break; + case container.state?.waiting?.reason && container.state.waiting.reason != "PodInitializing": + reason = `Init:${container.state.waiting.reason}`; + initializing = true; + break; + default: + reason = `Init:${i}/${pod.spec?.initContainers?.length}`; + initializing = true; + } + break; + } + + if (!initializing || getCondition('Initialized')?.status == 'True') { + restarts = restartableInitContainerRestarts; + lastRestartDate = lastRestartableInitContainerRestartDate; + let hasRunning = false; + let errorReason = ""; + for (const container of pod.status?.containerStatuses?.toReversed() ?? []) { + restarts += container.restartCount; + if (container.lastState?.terminated != null) { + const terminatedDate = container.lastState?.terminated.finishedAt; + if (terminatedDate && lastRestartDate < terminatedDate) { + lastRestartDate = terminatedDate; + } + } + if (container.state?.waiting?.reason) { + reason = container.state.waiting.reason; + } else if (container.state?.terminated) { + if (container.state.terminated.reason) { + reason = container.state.terminated.reason; + } else if (container.state.terminated.signal != 0) { + reason = `Signal:${container.state.terminated.signal}`; + } else { + reason = `ExitCode:${container.state.terminated.exitCode}`; + } + if (container.state.terminated.exitCode != 0) { + errorReason = reason; + } + } else if (container.ready && container.state?.running) { + hasRunning = true; + readyContainers++; + } + } + + // change pod status back to "Running" if there is at least one container still reporting as "Running" status + if (reason == "Completed") { + if (hasRunning && getCondition('Ready')?.status == 'True') { + reason = "Running"; + } else if (errorReason) { + reason = errorReason; + } else if (hasRunning) { + reason = "NotReady"; + } + } + } + + if (pod.metadata?.deletionTimestamp) { + if (pod.status?.reason == 'NodeLost') { + reason = "Unknown"; + } else if (!(podPhase == 'Failed' || podPhase == 'Succeeded')) { + reason = "Terminating"; + } + } + + return { + readyCount: `${readyContainers}/${totalContainers}`, + reason, + restarts, + lastRestartDate: lastRestartDate.valueOf() ? lastRestartDate : void 0, + }; +} diff --git a/lib/deno.json b/lib/deno.json index 1cd21e6..f8a98e1 100644 --- a/lib/deno.json +++ b/lib/deno.json @@ -1,6 +1,6 @@ { "name": "@cloudydeno/kubernetes-apis", - "version": "0.7.0", + "version": "0.7.1", "imports": { "@cloudydeno/kubernetes-client": "jsr:@cloudydeno/kubernetes-client@^0.8" }, @@ -53,6 +53,8 @@ "./autoscaling.k8s.io/v1": "./vpa/autoscaling.k8s.io@v1/mod.ts", + "./core/v1/pod-health": "./contrib/core-v1-pod-health.ts", + "./common.ts": "./common.ts", "./deps.ts": "./deps.ts", "./operations.ts": "./operations.ts", @@ -60,7 +62,7 @@ }, "publish": { "exclude": [ - "examples", + "examples" ] } } diff --git a/tests/core-v1-pod-health_test.ts b/tests/core-v1-pod-health_test.ts new file mode 100644 index 0000000..4b1d221 --- /dev/null +++ b/tests/core-v1-pod-health_test.ts @@ -0,0 +1,130 @@ +import { calculatePodHealth } from "@cloudydeno/kubernetes-apis/core/v1/pod-health"; +import { assertObjectMatch } from "@std/assert/object-match"; + +// Some contrived tests of the pod health routine to make sure it generally functions + +Deno.test('pod health helper', () => { + const oneContainerSpec = { + containers: [{ + name: 'app', + }], + }; + const emptyStatusFields = { + name: 'app', + image: '', + imageID: '', + ready: false, + restartCount: 0, + }; + + // Null pod + assertObjectMatch(calculatePodHealth({}), { + reason: 'Unknown', + readyCount: '0/0', + restarts: 0, + lastRestartDate: undefined, + }); + + // Containers not started yet + assertObjectMatch(calculatePodHealth({ + spec: oneContainerSpec, + status: { + phase: 'Running', + containerStatuses: [{ + ...emptyStatusFields, + state: { + waiting: { + reason: 'Initializing', + }, + }, + }], + }, + }), { + reason: 'Initializing', + readyCount: '0/1', + restarts: 0, + lastRestartDate: undefined, + }); + + // Container running, not ready + assertObjectMatch(calculatePodHealth({ + spec: oneContainerSpec, + status: { + phase: 'Running', + containerStatuses: [{ + ...emptyStatusFields, + state: { + running: { + startedAt: new Date, + }, + }, + }], + }, + }), { + reason: 'Running', + readyCount: '0/1', + restarts: 0, + lastRestartDate: undefined, + }); + + // Container passing readiness checks + assertObjectMatch(calculatePodHealth({ + spec: oneContainerSpec, + status: { + phase: 'Running', + containerStatuses: [{ + ...emptyStatusFields, + ready: true, + state: { + running: { + startedAt: new Date, + }, + }, + }], + }, + }), { + reason: 'Running', + readyCount: '1/1', + restarts: 0, + lastRestartDate: undefined, + }); + + // Container with liveness issues + assertObjectMatch(calculatePodHealth({ + spec: oneContainerSpec, + status: { + phase: 'Running', + containerStatuses: [{ + ...emptyStatusFields, + ready: true, + restartCount: 5, + lastState: { + terminated: { + exitCode: 5, + finishedAt: new Date(5), + }, + }, + state: { + waiting: { + reason: 'CrashLoopBackOff', + }, + }, + }], + }, + }), { + reason: 'CrashLoopBackOff', + readyCount: '0/1', + restarts: 5, + lastRestartDate: new Date(5), + }); + + // Pod pending deletion + assertObjectMatch(calculatePodHealth({ + metadata: { + deletionTimestamp: new Date, + }, + }), { + reason: 'Terminating', + }); + +}); diff --git a/tests/deno.json b/tests/deno.json new file mode 100644 index 0000000..62f266d --- /dev/null +++ b/tests/deno.json @@ -0,0 +1,5 @@ +{ + "imports": { + "@std/assert": "jsr:@std/assert@^1.0.18" + } +} diff --git a/lib/tunnels_test.ts b/tests/tunnels_test.ts similarity index 97% rename from lib/tunnels_test.ts rename to tests/tunnels_test.ts index 6168a47..7e9ed63 100644 --- a/lib/tunnels_test.ts +++ b/tests/tunnels_test.ts @@ -1,5 +1,5 @@ -import { assertEquals } from "https://deno.land/std@0.198.0/assert/mod.ts"; -import { StdioTunnel, type ExecStatus, PortforwardTunnel } from "./tunnels.ts"; +import { assertEquals } from "@std/assert/equals"; +import { StdioTunnel, type ExecStatus, PortforwardTunnel } from "@cloudydeno/kubernetes-apis/tunnels.ts"; Deno.test('stdiotunnel output buffering', async () => { const intendedStdout = 'hello world';